Pisilinux3/pisi
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* optimization: use a single index for multiple repositories, no performance

Browse Source 
degradation observed, scalable in the number of repos.
* generalize decode/encode method in dbshelve
* fix: user interface init must come first
* fix: do not try to read autoxml file if xmlfile_support not available
* various other small fixes
This commit is contained in:
Eray Özkural
2006-03-22 15:39:47 +00:00
parent 418e8c1d69
commit ca94151d64
18 changed files with 362 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pisi.e3p
+7 -3
View File
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE Project SYSTEM "Project-3.8.dtd">
<!-- Project file for project pisi -->
<!-- Saved: 2006-03-09, 15:08:35 -->
<!-- Saved: 2006-03-21, 15:04:22 -->
<!-- Copyright (C) 2006 PiSi Development Team, -->
<Project version="3.8">
<ProgLanguage mixed="0">Python</ProgLanguage>
@@ -474,6 +474,10 @@
<Dir>tests</Dir>
<Name>repodbtests.py</Name>
</Source>
<Source>
<Dir>pisi</Dir>
<Name>itembyrepodb.py</Name>
</Source>
</Sources>
<Forms>
</Forms>
@@ -515,10 +519,10 @@
<VcsOtherData>{'standardLayout': 1}</VcsOtherData>
</Vcs>
<FiletypeAssociations>
<FiletypeAssociation pattern="*.py" type="SOURCES" />
<FiletypeAssociation pattern="*.ui.h" type="FORMS" />
<FiletypeAssociation pattern="*.idl" type="INTERFACES" />
<FiletypeAssociation pattern="*.ui" type="FORMS" />
<FiletypeAssociation pattern="*.idl" type="INTERFACES" />
<FiletypeAssociation pattern="*.py" type="SOURCES" />
<FiletypeAssociation pattern="*.ptl" type="SOURCES" />
</FiletypeAssociations>
</Project>
pisi/__init__.py
+2 -2
View File
@@ -12,9 +12,9 @@
# PiSi version
__version__ = "1.0.5"
__version__ = "1.0.6"
__dbversion__ = "1.0.5"
__dbversion__ = "1.0.6"
__filesdbversion__ = "1.0.5" # yes, this is the real bottleneck
__all__ = [ 'api', 'config', 'packagedb', 'installdb', 'search' ]
pisi/api.py
+31 -26
View File
@@ -51,20 +51,23 @@ class Error(pisi.Error):
def init(database = True, options = None, ui = None, comar = True):
"""Initialize PiSi subsystem"""
# UI comes first
if ui is None:
from pisi.cli import CLI
if options:
ctx.ui = CLI(options.debug)
else:
ctx.ui = CLI()
else:
ctx.ui = ui
import pisi.config
ctx.config = pisi.config.Config(options)
# TODO: this is definitely not dynamic beyond this point!
ctx.comar = comar and not ctx.config.get_option('ignore_comar')
if ui is None:
if options:
pisi.context.ui = pisi.cli.CLI(options.debug)
else:
pisi.context.ui = pisi.cli.CLI()
else:
pisi.context.ui = ui
# initialize repository databases
ctx.database = database
if database:
@@ -73,7 +76,7 @@ def init(database = True, options = None, ui = None, comar = True):
ctx.installdb = pisi.installdb.init()
ctx.filesdb = pisi.files.FilesDB()
ctx.componentdb = pisi.component.ComponentDB()
packagedb.init_db()
ctx.packagedb = packagedb.init_db()
ctx.sourcedb = pisi.sourcedb.init()
pisi.search.init(['summary', 'description'], ['en', 'tr'])
else:
@@ -81,6 +84,7 @@ def init(database = True, options = None, ui = None, comar = True):
ctx.installdb = None
ctx.filesdb = None
ctx.componentdb = None
ctx.packagedb = None
ctx.sourcedb = None
ctx.ui.debug('PISI API initialized')
ctx.initialized = True
@@ -89,11 +93,13 @@ def finalize():
if ctx.initialized:
pisi.repodb.finalize()
pisi.installdb.finalize()
if ctx.filesdb != None:
if ctx.filesdb:
ctx.filesdb.close()
if ctx.componentdb != None:
if ctx.componentdb:
ctx.componentdb.close()
packagedb.finalize_db()
if ctx.packagedb:
packagedb.finalize_db()
ctx.packagedb = None
pisi.sourcedb.finalize()
pisi.search.finalize()
if ctx.dbenv:
@@ -107,8 +113,7 @@ def list_available():
available = set()
for repo in pisi.context.repodb.list():
pkg_db = pisi.packagedb.get_db(repo)
available.update(pkg_db.list_packages())
available.update(ctx.packagedb.list_packages())
return available
def list_upgradable():
@@ -119,7 +124,7 @@ def list_upgradable():
Ap = []
for x in A:
(version, release, build) = ctx.installdb.get_version(x)
pkg = packagedb.get_package(x)
pkg = ctx.packagedb.get_package(x)
if ignore_build or (not build):
if release < pkg.release:
Ap.append(x)
@@ -142,7 +147,7 @@ def package_graph(A, ignore_installed = False):
# try to construct a pisi graph of packages to
# install / reinstall
G_f = pgraph.PGraph(packagedb) # construct G_f
G_f = pgraph.PGraph(ctx.packagedb) # construct G_f
# find the "install closure" graph of G_f by package
# set A using packagedb
@@ -153,7 +158,7 @@ def package_graph(A, ignore_installed = False):
while len(B) > 0:
Bp = set()
for x in B:
pkg = packagedb.get_package(x)
pkg = ctx.packagedb.get_package(x)
#print pkg
for dep in pkg.runtimeDependencies():
if ignore_installed:
@@ -169,14 +174,14 @@ def configure_pending():
# start with pending packages
# configure them in reverse topological order of dependency
A = ctx.installdb.list_pending()
G_f = pgraph.PGraph(packagedb) # construct G_f
G_f = pgraph.PGraph(ctx.packagedb) # construct G_f
for x in A.keys():
G_f.add_package(x)
B = A
while len(B) > 0:
Bp = set()
for x in B.keys():
pkg = packagedb.get_package(x)
pkg = ctx.packagedb.get_package(x)
for dep in pkg.runtimeDependencies():
if dep.package in G_f.vertices():
G_f.add_dep(x, dep)
@@ -233,12 +238,12 @@ def info_file(package_fn):
def info_name(package_name, installed=False):
"""fetch package information for a package"""
if installed:
pkgdb = packagedb.inst_packagedb
else:
pkgdb = packagedb
if pkgdb.has_package(package_name):
package = pkgdb.get_package(package_name)
if ctx.packagedb.has_package(package_name):
package, repo = pkgdb.get_package(package_name)
if (not installed) and (repo==pisi.itembyrepodb.installed):
raise Error(_('Package %s not found') % package_name)
from pisi.metadata import MetaData
metadata = MetaData()
metadata.package = package
@@ -313,7 +318,7 @@ def update_repo(repo):
index.read_uri(ctx.repodb.get_repo(repo).indexuri.get_uri(), repo)
else:
raise Error(_('No repository named %s found.') % repo)
index.update_db(repo)
ctx.txn_proc(lambda txn : index.update_db(repo, txn=txn))
ctx.ui.info(_('\n* Package database updated.'))
def delete_cache():
pisi/atomicoperations.py
+17 -20
View File
@@ -28,7 +28,7 @@ import pisi.packagedb as packagedb
import pisi.dependency as dependency
import pisi.util as util
from pisi.specfile import *
from pisi.package import Package
#from pisi.package import Package
from pisi.metadata import MetaData
from pisi.files import Files
from pisi.uri import URI
@@ -38,6 +38,9 @@ import pisi.ui
class Error(pisi.Error):
pass
class NotfoundError(pisi.Error):
pass
# single package operations
class AtomicOperation(object):
@@ -61,10 +64,10 @@ class Install(AtomicOperation):
def from_name(name):
# download package and return an installer object
# find package in repository
repo = packagedb.which_repo(name)
repo = ctx.packagedb.which_repo(name)
if repo:
repo = ctx.repodb.get_repo(repo)
pkg = packagedb.get_package(name)
pkg = ctx.packagedb.get_package(name)
# FIXME: let pkg.packageURI be stored as URI type rather than string
pkg_uri = URI(pkg.packageURI)
@@ -83,7 +86,7 @@ class Install(AtomicOperation):
def __init__(self, package_fname, ignore_dep = None):
"initialize from a file name"
super(Install, self).__init__(ignore_dep)
self.package = Package(package_fname)
self.package = pisi.package.Package(package_fname)
self.package.read()
self.metadata = self.package.metadata
self.files = self.package.files
@@ -146,9 +149,8 @@ class Install(AtomicOperation):
# check if package is in database
# If it is not, put it into 3rd party packagedb
if not packagedb.has_package(self.pkginfo.name):
db = packagedb.thirdparty_packagedb
db.add_package(self.pkginfo)
if not ctx.packagedb.has_package(self.pkginfo.name):
ctx.packagedb.add_package(self.pkginfo, pisi.itembyrepodb.thirdparty)
# check file conflicts
for file in self.files.list:
@@ -299,10 +301,10 @@ class Install(AtomicOperation):
txn)
# filesdb
ctx.filesdb.add_files(self.metadata.package.name, self.files, txn)
ctx.filesdb.add_files(self.metadata.package.name, self.files, txn=txn)
# installed packages
packagedb.inst_packagedb.add_package(self.pkginfo, txn)
ctx.packagedb.add_package(self.pkginfo, pisi.itembyrepodb.installed, txn=txn)
def update_environment(self):
# check if we have any shared objects or anything under
@@ -345,7 +347,7 @@ class Remove(AtomicOperation):
def __init__(self, package_name, ignore_dep = None):
super(Remove, self).__init__(ignore_dep)
self.package_name = package_name
self.package = packagedb.get_package(self.package_name)
self.package = ctx.packagedb.get_package(self.package_name)
try:
self.files = ctx.installdb.files(self.package_name)
except pisi.Error, e:
@@ -356,8 +358,7 @@ class Remove(AtomicOperation):
def run(self):
"""Remove a single package"""
inst_packagedb = packagedb.inst_packagedb
ctx.ui.status(_('Removing package %s') % self.package_name)
ctx.ui.notify(pisi.ui.removing, package = self.package, files = self.files)
if not ctx.installdb.is_installed(self.package_name):
@@ -426,11 +427,7 @@ class Remove(AtomicOperation):
def remove_db(self, txn):
ctx.installdb.remove(self.package_name, txn)
ctx.filesdb.remove_files(self.files, txn)
if packagedb.thirdparty_packagedb.has_package(self.package_name, txn):
packagedb.thirdparty_packagedb.remove_package(self.package_name, txn)
if packagedb.inst_packagedb.has_package(self.package_name, txn):
packagedb.inst_packagedb.remove_package(self.package_name, txn)
ctx.packagedb.remove_tracking_package(self.package_name, txn)
def remove_single(package_name):
Remove(package_name).run()
@@ -466,8 +463,8 @@ def virtual_install(metadata, files, txn):
if ctx.installdb.is_installed(pkg.name, txn):
if __is_virtual_upgrade(metadata):
ctx.installdb.remove(pkg.name, txn)
packagedb.remove_package(pkg.name, txn)
ctx.filesdb.remove_files(ctx.installdb.files(pkg.name), txn)
ctx.packagedb.remove_package(pkg.name, txn=txn)
ctx.filesdb.remove_files(ctx.installdb.files(pkg.name), txn=txn)
else:
return
@@ -488,7 +485,7 @@ def virtual_install(metadata, files, txn):
ctx.filesdb.add_files(metadata.package.name, files, txn)
# installed packages
packagedb.inst_packagedb.add_package(pkginfo, txn)
ctx.packagedb.add_package(pkginfo, pisi.itembyrepodb.installed, txn=txn)
def resurrect_package(package_fn, write_files, txn = None):
"""Resurrect the package from xml files"""
pisi/cli/commands.py
+8 -10
View File
@@ -283,8 +283,7 @@ conflicts relations starting from given packages.
a = set()
from pisi import packagedb
for repo in ctx.repodb.list():
pkg_db = packagedb.get_db(repo)
a = a.union(pkg_db.list_packages())
a = a.union(ctx.packagedb.list_packages(repo))
return a
def run(self):
@@ -610,7 +609,7 @@ Usage: info <package1> <package2> ... <packagen>
metadata, files = pisi.api.info_name(arg, True)
ctx.ui.info(_('Installed package:'))
self.print_pkginfo(metadata, files)
if packagedb.has_package(arg):
elif ctx.packagedb.has_package(arg):
metadata, files = pisi.api.info_name(arg, False)
ctx.ui.info(_('Package found in repository:'))
self.print_pkginfo(metadata, files)
@@ -619,7 +618,7 @@ Usage: info <package1> <package2> ... <packagen>
import os.path
ctx.ui.info(unicode(metadata.package))
revdeps = [x[0] for x in packagedb.get_rev_deps(metadata.package.name)]
revdeps = [x[0] for x in ctx.packagedb.get_rev_deps(metadata.package.name)]
print _('Reverse Dependencies:'), util.strlist(revdeps)
if self.options.files or self.options.files_path:
if files:
@@ -722,7 +721,7 @@ Usage: list-installed
ctx.ui.info(_('Package Name |St| Version| Rel.| Build| Distro| Date'))
print '========================================================================'
for pkg in list:
package = pisi.packagedb.inst_packagedb.get_package(pkg)
package = ctx.packagedb.get_package(pkg, pisi.itembyrepodb.installed)
inst_info = ctx.installdb.get_info(pkg)
if self.options.long:
ctx.ui.info(unicode(package))
@@ -730,7 +729,7 @@ Usage: list-installed
elif self.options.install_info:
ctx.ui.info('%-15s |%s' % (package.name, inst_info.one_liner()))
else:
ctx.ui.info('%15s - %s' % (package.name, package.summary))
ctx.ui.info('%15s - %s' % (package.name, unicode(package.summary)))
self.finalize()
class RebuildDb(Command):
@@ -924,12 +923,11 @@ Gives a brief list of PiSi packages published in the repository.
from pisi import packagedb
from colors import colorize
pkg_db = packagedb.get_db(repo)
list = pkg_db.list_packages()
list = ctx.packagedb.list_packages(repo)
installed_list = ctx.installdb.list_installed()
list.sort()
for p in list:
package = pkg_db.get_package(p)
package = ctx.packagedb.get_package(p)
if self.options.long:
ctx.ui.info(unicode(package))
else:
@@ -1045,7 +1043,7 @@ Usage: list-upgrades [ <repo1> <repo2> ... repon ]
ctx.ui.info(_('Package Name |St| Version| Rel.| Build| Distro| Date'))
print '========================================================================'
for pkg in list:
package = pisi.packagedb.inst_packagedb.get_package(pkg)
package = ctx.packagedb.get_package(pkg, pisi.itembyrepodb.installed)
inst_info = ctx.installdb.get_info(pkg)
if self.options.long:
ctx.ui.info(package)
pisi/context.py
+1
View File
@@ -33,6 +33,7 @@ ui = None # not now
dbenv = None
installdb = None
packagedb = None
repodb = None
invidx = None
pisi/dbshelve.py
+31 -16
View File
@@ -21,6 +21,7 @@
# 13-Dec-2000: Updated to be used with the new bsddb3 package.
# Added DBShelfCursor class.
#
# 13-Dec-2005: Minor hacking by exa to make it work better with PISI
#------------------------------------------------------------------------
"""Manage shelves of pickled objects using bsddb database files for the
@@ -29,6 +30,7 @@ storage.
Add transaction processing by default to dictionary ops also
Also other minor improvements -- exa
Now added support for overriding the marshalling method
"""
#------------------------------------------------------------------------
@@ -37,6 +39,11 @@ import cPickle
import bsddb3.db as db
import bsddb3.dbobj as dbobj
import pisi
class CodingError(pisi.Error):
pass
class DBShelf:
"""A shelf to hold pickled objects, built upon a bsddb DB object. It
automatically pickles/unpickles data objects going to/from the DB.
@@ -48,7 +55,6 @@ class DBShelf:
self.db = dbobj.DB(dbenv)
else:
self.db = db.DB(None)
self.binary = 1
# it is better to explicitly close a shelf
#def __del__(self):
@@ -76,6 +82,15 @@ class DBShelf:
return retval
else:
return proc(txn)
def decode(self, data):
try:
return cPickle.loads(data)
except:
raise CodingError()
def encode(self, obj):
return cPickle.dumps(obj, 1)
def clear(self, txn = None):
def proc(txn):
@@ -105,13 +120,13 @@ class DBShelf:
def __getitem__(self, key):
def proc(txn):
data = self.db.get(key)
return cPickle.loads(data)
return self.decode(data)
return self.txn_proc(proc, None)
def __setitem__(self, key, value):
# hyperdandik transactions
def proc(txn):
data = cPickle.dumps(value, self.binary)
data = self.encode(value)
self.db.put(key,data,txn)
return self.txn_proc(proc, None)
@@ -138,7 +153,7 @@ class DBShelf:
newitems = []
for k, v in items:
newitems.append( (k, cPickle.loads(v)) )
newitems.append( (k, self.decode(v) ) )
return newitems
def values(self, txn=None):
@@ -147,13 +162,13 @@ class DBShelf:
else:
values = self.db.values()
return map(cPickle.loads, values)
return map(lambda x : self.decode(x), values)
#-----------------------------------
# Other methods
def __append(self, value, txn=None):
data = cPickle.dumps(value, self.binary)
data = self.encode(value)
return self.db.append(data, txn)
def append(self, value, txn=None):
@@ -165,7 +180,7 @@ class DBShelf:
def associate(self, secondaryDB, callback, flags=0):
def _shelf_callback(priKey, priData, realCallback=callback):
data = cPickle.loads(priData)
data = self.decode(priData)
return realCallback(priKey, data)
return self.db.associate(secondaryDB, _shelf_callback, flags)
@@ -178,15 +193,15 @@ class DBShelf:
# off.
data = apply(self.db.get, args, kw)
try:
return cPickle.loads(data)
except (TypeError, cPickle.UnpicklingError):
return self.decode(data)
except (TypeError, CodingError):
return data # we may be getting the default value, or None,
# so it doesn't need unpickled.
def get_both(self, key, value, txn=None, flags=0):
data = cPickle.dumps(value, self.binary)
data = self.encode(value)
data = self.db.get(key, data, txn, flags)
return cPickle.loads(data)
return self.decode(data)
def cursor(self, txn=None, flags=0):
c = DBShelfCursor(self.db.cursor(txn, flags))
@@ -194,7 +209,7 @@ class DBShelf:
return c
def put(self, key, value, txn=None, flags=0):
data = cPickle.dumps(value, self.binary)
data = self.encode(value)
return self.db.put(key, data, txn, flags)
def join(self, cursorList, flags=0):
@@ -229,7 +244,7 @@ class DBShelfCursor:
return DBShelfCursor(self.dbc.dup(flags))
def put(self, key, value, flags=0):
data = cPickle.dumps(value, self.binary)
data = self.encode(value)
return self.dbc.put(key, data, flags)
@@ -247,7 +262,7 @@ class DBShelfCursor:
return self._extract(rec)
def get_3(self, key, value, flags):
data = cPickle.dumps(value, self.binary)
data = self.encode(value)
rec = self.dbc.get(key, flags)
return self._extract(rec)
@@ -263,7 +278,7 @@ class DBShelfCursor:
def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP)
def get_both(self, key, value, flags=0):
data = cPickle.dumps(value, self.binary)
data = self.encode(value)
rec = self.dbc.get_both(key, flags)
return self._extract(rec)
@@ -286,7 +301,7 @@ class DBShelfCursor:
return None
else:
key, data = rec
return key, cPickle.loads(data)
return key, self.decode(data)
#----------------------------------------------
# Methods allowed to pass-through to self.dbc
pisi/dependency.py
+6 -7
View File
@@ -17,9 +17,8 @@ import gettext
__trans = gettext.translation('pisi', fallback=True)
_ = __trans.ugettext
#import pisi.db as db
import pisi.context as ctx
import pisi.packagedb as packagedb
#import pisi.packagedb as packagedb
from pisi.version import Version
import pisi.pxml.autoxml as autoxml
from pisi.util import Checks
@@ -84,7 +83,7 @@ dependency spec"""
if not ctx.installdb.is_installed(pkg_name):
return False
else:
pkg = packagedb.inst_packagedb.get_package(pkg_name)
pkg = ctx.packagedb.get_package(pkg_name)
(version, release) = (pkg.version, pkg.release)
return depinfo.satisfies(pkg_name, version, release)
@@ -92,10 +91,10 @@ def repo_satisfies_dep(depinfo):
"""determine if a package in *repository* satisfies given
dependency spec"""
pkg_name = depinfo.package
if not packagedb.has_package(pkg_name):
if not ctx.packagedb.has_package(pkg_name):
return False
else:
pkg = packagedb.get_package(pkg_name)
pkg = ctx.packagedb.get_package(pkg_name)
(version, release) = (pkg.version, pkg.release)
return depinfo.satisfies(pkg_name, version, release)
@@ -108,13 +107,13 @@ def satisfies_dependencies(pkg, deps, sat = installed_satisfies_dep):
return True
def satisfies_runtime_deps(pkg):
deps = packagedb.get_package(pkg).runtimeDependencies()
deps = ctx.packagedb.get_package(pkg).runtimeDependencies()
return satisfies_dependencies(pkg, deps)
def installable(pkg):
"""calculate if pkg name is installable currently
which means it has to satisfy both install and runtime dependencies"""
if not packagedb.has_package(pkg):
if not ctx.packagedb.has_package(pkg):
ctx.ui.info(_("Package %s is not present in the package database") % pkg);
return False
elif satisfies_runtime_deps(pkg):
pisi/index.py
+5 -6
View File
@@ -79,15 +79,14 @@ class Index(XmlFile):
if fn == 'pspec.xml':
self.add_spec(os.path.join(root, fn), repo_uri)
def update_db(self, repo):
pkgdb = packagedb.get_db(repo)
pkgdb.clear()
def update_db(self, repo, txn = None):
for comp in self.components:
ctx.componentdb.update_component(comp)
ctx.componentdb.update_component(comp, txn=txn)
ctx.packagedb.remove_repo(repo, txn=txn)
for pkg in self.packages:
pkgdb.add_package(pkg)
ctx.packagedb.add_package(pkg, repo, txn=txn)
for sf in self.specs:
ctx.sourcedb.add_spec(sf, repo)
ctx.sourcedb.add_spec(sf, repo, txn=txn)
def add_package(self, path, repo_uri):
package = Package(path, 'r')
pisi/installdb.py
+1 -1
View File
@@ -36,7 +36,7 @@ class InstallDBError(pisi.Error):
class InstallInfo:
# some data is replicated from packagedb.inst_packagedb
# some data is replicated from packagedb
# we store as an object, hey, we can waste O(1) space.
# this is also easier to modify in the future, without
# requiring database upgrades! wow!
pisi/itembyrepodb.py
+162
View File
@@ -0,0 +1,162 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2005, TUBITAK/UEKAE
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#
# Author: Eray Ozkural <eray@uludag.org.tr>
import os
import fcntl
import types
import gettext
__trans = gettext.translation('pisi', fallback=True)
_ = __trans.ugettext
import pisi
import pisi.util as util
import pisi.context as ctx
import pisi.lockeddbshelve as shelve
installed, thirdparty = range(2)
class Error(pisi.Error):
pass
class NotfoundError(pisi.Error):
pass
# the above are special databases to keep track of already installed stuff and
# third party stuff not in any real repository
class ItemByRepoDB(object):
def __init__(self, name):
self.d = shelve.LockedDBShelf(name)
#self.dbyrepo = shelve.LockedDBShelf(name + '-byrepo')
def close(self):
self.d.close()
def clear(self, txn = None):
self.d.clear(txn=txn)
def txn_proc(self, proc, txn):
self.d.txn_proc(proc, txn)
def items(self):
return self.d.items()
def list(self, repo = None):
if repo:
return [ k for k,data in self.d.items() if data.has_key(self.repo_str(repo))]
else:
return [ pkg for pkg in self.d.keys() ]
# TODO: carry this to repodb, really :/
def order(self):
import pisi.repodb
order = [ 'repo-' + x for x in ctx.repodb.list() ] + ['trdparty', 'inst']
return order
# def list_repo(self, repo):
# return self.dbyrepo[repo]
def repo_str(self, repo):
if repo==thirdparty:
repo='trdparty'
elif repo==installed:
repo='inst'
else:
repo='repo-'+repo
return repo
def str_repo(self, str):
if str.startswith('repo-'):
return str[5:]
elif str=='trdparty':
return thirdparty
elif str=='inst':
return installed
else:
raise Error(_('Invalid repository string'))
def has_key(self, name, repo = None, txn = None):
name = str(name)
if not repo:
return self.d.has_key(name, txn)
else:
return self.d.has_key(name, txn).has_key(repo)
def get_item_repo(self, name, repo = None, txn = None):
name = str(name)
def proc(txn):
if not self.d.has_key(name, txn=txn):
raise NotfoundError(_('Key %s not found') % name)
s = self.d.get(name, txn=txn)
if not repo:
for repostr in self.order():
if s.has_key(repostr):
return (s[repostr], self.str_repo(repostr))
else:
repostr = self.repo_str(repo)
if s.has_key(repostr):
return (s[repostr], repo)
return None
return self.d.txn_proc(proc, txn)
def get_item(self, name, repo = None, txn = None):
x = self.get_item_repo(name, repo, txn)
if x:
item, repo = x
# discard repo, not always needed
return item
else:
return None
def which_repo(self, name, repo = None, txn = None):
x = self.get_item_repo(name, repo, txn)
if x:
item, repo = x
return repo
else:
return None
def add_item(self, name, obj, repo, txn = None):
repostr = self.repo_str(repo)
def proc(txn):
if not self.d.has_key(name):
s = dict()
else:
s = self.d.get(name, txn)
s[ repostr ] = obj
self.d.put(name, s, txn)
self.d.txn_proc(proc, txn)
def remove_item(self, name, repo, txn = None):
name = str(name)
repo = str(repo)
def proc(txn):
s = self.d.get(name, txn)
del s[self.repo_str(repo)]
if (len(s)==0):
self.d.delete(name, txn)
else:
self.d.put(name, s, txn)
self.d.txn_proc(proc, txn)
def remove_repo(self, repo, txn = None):
def proc(txn):
repostr = self.repo_str(repo)
for (key, s) in self.d.items():
if s.has_key(repostr):
del s[repostr]
self.d.put(repo, s, txn)
self.d.txn_proc(proc, txn)
pisi/lockeddbshelve.py
+19 -4
View File
@@ -9,16 +9,22 @@
#
# Please read the COPYING file.
#
#
# Authors: Eray Ozkural <eray@uludag.org.tr>
"""
By default, cPickle + 7zip compression is used
"""
import os
import fcntl
import types
import cPickle
import bsddb3.db as db
import bsddb3.dbobj as dbobj
#import bsddb3.dbshelve as shelve
import pisi.dbshelve as shelve
import os
import fcntl
import types
import pylzma
import gettext
__trans = gettext.translation('pisi', fallback=True)
@@ -83,6 +89,15 @@ def init_dbenv():
class LockedDBShelf(shelve.DBShelf):
"""A simple wrapper to implement locking for bsddb's dbshelf"""
# def decode(self, data):
# try:
# return cPickle.loads(pylzma.decompress(data))
# except:
# raise shelve.CodingError()
# def encode(self, obj):
# return pylzma.compress(cPickle.dumps(obj, 1), algorithm=0)
def __init__(self, dbname, mode=0644,
filetype=db.DB_BTREE, dbenv = None):
if dbenv == None:
pisi/operations.py
+9 -9
View File
@@ -168,7 +168,7 @@ def check_conflicts(order):
while len(B) > 0:
Bp = set()
for x in B:
pkg = packagedb.get_package(x) # get latest version!
pkg = ctx.packagedb.get_package(x) # get latest version!
#TODO: read conflicts from a conflicts db...
for conflict in pkg.conflicts:
if ctx.installdb.is_installed(self.pkginfo):
@@ -249,7 +249,7 @@ def plan_install_pkg_names(A):
# try to construct a pisi graph of packages to
# install / reinstall
G_f = pgraph.PGraph(packagedb) # construct G_f
G_f = pgraph.PGraph(ctx.packagedb) # construct G_f
# find the "install closure" graph of G_f by package
# set A using packagedb
@@ -260,7 +260,7 @@ def plan_install_pkg_names(A):
while len(B) > 0:
Bp = set()
for x in B:
pkg = packagedb.get_package(x)
pkg = ctx.packagedb.get_package(x)
for dep in pkg.runtimeDependencies():
ctx.ui.debug('checking %s' % str(dep))
# we don't deal with already *satisfied* dependencies
@@ -299,7 +299,7 @@ def upgrade_pkg_names(A = []):
ctx.ui.info(_('Package %s is not installed.') % x)
continue
(version, release, build) = ctx.installdb.get_version(x)
pkg = packagedb.get_package(x)
pkg = ctx.packagedb.get_package(x)
if ignore_build or (not build) or (not pkg.build):
if release < pkg.release:
@@ -348,7 +348,7 @@ def plan_upgrade(A):
# try to construct a pisi graph of packages to
# install / reinstall
G_f = pgraph.PGraph(packagedb) # construct G_f
G_f = pgraph.PGraph(ctx.packagedb) # construct G_f
# find the "install closure" graph of G_f by package
# set A using packagedb
@@ -481,7 +481,7 @@ def plan_remove(A):
# try to construct a pisi graph of packages to
# install / reinstall
G_f = pgraph.PGraph(packagedb) # construct G_f
G_f = pgraph.PGraph(ctx.packagedb) # construct G_f
# find the (install closure) graph of G_f by package
# set A using packagedb
@@ -491,12 +491,12 @@ def plan_remove(A):
while len(B) > 0:
Bp = set()
for x in B:
pkg = packagedb.get_package(x)
rev_deps = packagedb.get_rev_deps(x)
pkg = ctx.packagedb.get_package(x)
rev_deps = ctx.packagedb.get_rev_deps(x)
for (rev_dep, depinfo) in rev_deps:
# we don't deal with uninstalled rev deps
# and unsatisfied dependencies (this is important, too)
if packagedb.inst_packagedb.has_package(rev_dep) and \
if ctx.packagedb.has_package(rev_dep, pisi.itembyrepodb.installed) and \
dependency.installed_satisfies_dep(depinfo):
if not rev_dep in G_f.vertices():
Bp.add(rev_dep)
pisi/package.py
-1
View File
@@ -24,7 +24,6 @@ _ = __trans.ugettext
import pisi
import pisi.context as ctx
import pisi.archive as archive
from pisi.packagedb import get_package
from pisi.uri import URI
from pisi.metadata import MetaData
from pisi.files import Files
pisi/packagedb.py
+54 -112
View File
@@ -29,6 +29,7 @@ import pisi
import pisi.util as util
import pisi.context as ctx
import pisi.lockeddbshelve as shelve
from pisi.itembyrepodb import ItemByRepoDB
class Error(pisi.Error):
pass
@@ -36,10 +37,10 @@ class Error(pisi.Error):
class PackageDB(object):
"""PackageDB class provides an interface to the package database
using shelf objects"""
def __init__(self, id):
self.id = id
self.d = shelve.LockedDBShelf('package-%s' % id )
self.dr = shelve.LockedDBShelf('revdep-%s' % id )
def __init__(self):
self.d = ItemByRepoDB('package')
self.dr = ItemByRepoDB('revdep')
def close(self):
self.d.close()
@@ -49,43 +50,43 @@ class PackageDB(object):
self.d.destroy()
self.dr.destroy()
def has_package(self, name, txn = None):
name = str(name)
return self.d.has_key(name, txn)
def has_package(self, name, repo=None, txn = None):
return self.d.has_key(name, repo, txn)
def get_package(self, name):
name = str(name)
return self.d[name]
def get_package(self, name, repo=None, txn = None):
try:
return self.d.get_item(name, repo, txn)
except pisi.itembyrepodb.NotfoundError, e:
raise Error(_('Package %s not found') % name)
def get_package_repo(self, name, repo=None, txn = None):
return self.d.get_item_repo(name, repo, txn)
def which_repo(self, name, repo=None, txn = None):
return self.d.which_repo(name, repo, txn)
def get_rev_deps(self, name):
name = str(name)
if self.dr.has_key(name):
return self.dr[name]
else:
return []
x = self.dr.get_item(self, name)
if not x:
x = []
def list_packages(self):
list = []
for (pkg, x) in self.d.items():
list.append(pkg)
return list
def list_packages(self, repo=None):
return self.d.list(repo)
#TODO: list_upgrades?
def add_package(self, package_info, txn = None):
def add_package(self, package_info, repo, txn = None):
name = str(package_info.name)
def proc(txn):
self.d.put(name, package_info, txn)
self.d.add_item(name, package_info, repo, txn)
for dep in package_info.runtimeDependencies():
dep_name = str(dep.package)
if self.dr.has_key(dep_name, txn):
revdep = self.dr.get(dep_name, txn)
if self.dr.has_key(dep_name, None, txn):
revdep = self.dr.get_item(dep_name, None, txn)
revdep = filter(lambda (n,d):n!=name, revdep)
revdep.append( (name, dep) )
self.dr.put(dep_name, revdep, txn)
self.dr.add_item(dep_name, revdep, repo, txn)
else:
self.dr.put(dep_name, [ (name, dep) ], txn)
self.dr.add_item(dep_name, [ (name, dep) ], repo, txn)
# add component
ctx.componentdb.add_package(package_info.partOf, package_info.name, txn)
# index summary and description
@@ -96,103 +97,44 @@ class PackageDB(object):
if lang in ['en', 'tr']:
pisi.search.add_doc('description', lang, package_info.name, doc, txn)
self.d.txn_proc(proc, txn)
ctx.txn_proc(proc, txn)
def clear(self):
def clear(self, txn = None):
self.d.clear()
self.dr.clear()
def remove_package(self, name, txn = None):
def remove_package(self, name, repo = None, txn = None):
name = str(name)
def proc(txn):
package_info = self.d.get(name, txn)
self.d.delete(name, txn)
package_info = self.d.get_item_repo(name, repo, txn)
self.d.remove_item(name, repo, txn)
#FIXME: what's happening to dr?
#WORKAROUND: do not remove component if it is not in repo
if self.id.startswith('repo'):
if type(repo)==types.StringType:
ctx.componentdb.remove_package(package_info.partOf, package_info.name, txn)
self.d.txn_proc(proc, txn)
def remove_repo(self, repo, txn = None):
def proc(txn):
self.d.remove_repo(repo, txn=txn)
self.dr.remove_repo(repo, txn=txn)
self.d.txn_proc(proc, txn)
packagedbs = {}
pkgdb = None
def add_db(name):
pisi.packagedb.packagedbs[name] = PackageDB('repo-' + name)
def get_db(name):
return pisi.packagedb.packagedbs[name]
def remove_db(name):
pisi.packagedb.packagedbs[name].close()
pisi.packagedb.packagedbs[name].destroy()
del pisi.packagedb.packagedbs[name]
def has_package(name, txn = None):
def proc(txn):
repo = which_repo(name)
if repo or thirdparty_packagedb.has_package(name, txn) or inst_packagedb.has_package(name, txn):
return True
return False
return ctx.txn_proc(proc, txn)
def which_repo(name, txn = None):
import pisi.repodb
def proc(txn):
for repo in pisi.repodb.db.list():
if get_db(repo).has_package(name, txn):
return repo
return None
return ctx.txn_proc(proc, txn)
def get_package(name):
repo = which_repo(name)
if repo:
return get_db(repo).get_package(name)
if thirdparty_packagedb.has_package(name):
return thirdparty_packagedb.get_package(name)
if inst_packagedb.has_package(name):
return inst_packagedb.get_package(name)
raise Error(_('get_package: package %s not found') % name)
def get_rev_deps(name):
repo = which_repo(name)
if repo:
return get_db(repo).get_rev_deps(name)
if thirdparty_packagedb.has_package(name):
return thirdparty_packagedb.get_rev_deps(name)
if inst_packagedb.has_package(name):
return inst_packagedb.get_rev_deps(name)
return []
def remove_package(name, txn = None):
def remove_tracking_package(name, txn = None):
# remove the guy from the tracking databases
inst_packagedb.remove_package(name, txn)
if thirdparty_packagedb.has_package(name, txn):
thirdparty_packagedb.remove_package(name, txn)
# tracking databases for non-repository information
thirdparty_packagedb = inst_packagedb = None
if pkgdb.has_package(name, itembyrepodb.installed, txn):
pkgdb.remove_package(name, itembyrepodb.installed, txn=txn)
if pkgdb.has_package(name, itembyrepodb.thirdparty, txn):
pkgdb.remove_package(name, itembyrepodb.thirdparty, txn=txn)
def init_db():
global thirdparty_packagedb
global inst_packagedb
if not thirdparty_packagedb:
thirdparty_packagedb = PackageDB('thirdparty')
if not pisi.packagedb.inst_packagedb:
inst_packagedb = PackageDB('installed')
global pkgdb
pkgdb = PackageDB()
return pkgdb
def finalize_db():
global thirdparty_packagedb
global inst_packagedb
global packagedbs
if thirdparty_packagedb:
thirdparty_packagedb.close()
thirdparty_packagedb = None
if inst_packagedb:
inst_packagedb.close()
inst_packagedb = None
packagedbs.clear()
global pkgdb
if pkgdb:
pkgdb.close()
pisi/pxml/autoxml.py
+1 -1
View File
@@ -345,7 +345,7 @@ class autoxml(oo.autosuper, oo.autoprop):
# init hook
if hasattr(self, 'init'):
self.init(tag)
if uri:
if xmlfile_support and uri:
self.read(uri, keepDoc, tmpDir)
cls.__init__ = initialize
pisi/repodb.py
+1 -8
View File
@@ -49,11 +49,6 @@ class RepoDB(object):
self.d.put("order", [], txn)
self.d.txn_proc(proc, txn)
def init_dbs(self):
# initialize package/source dbs
for x in self.list():
packagedb.add_db(x)
def close(self):
self.d.close()
@@ -88,7 +83,6 @@ class RepoDB(object):
order = self.d.get("order", txn)
order.append(name)
self.d.put("order", order, txn)
packagedb.add_db(name)
self.d.txn_proc(proc, txn)
def list(self):
@@ -104,7 +98,7 @@ class RepoDB(object):
list = self.d.get("order", txn)
list.remove(name)
self.d.put("order", list, txn)
packagedb.remove_db(name)
ctx.packagedb.remove_repo(name, txn=txn)
self.d.txn_proc(proc, txn)
db = None
@@ -116,7 +110,6 @@ def init():
return db
db = RepoDB()
db.init_dbs()
return db
def finalize():
tests/packagedbtests.py
+7 -7
View File
@@ -24,18 +24,18 @@ class PackageDBTestCase(testcase.TestCase):
testcase.TestCase.setUp(self)
self.spec = SpecFile()
self.spec.read('tests/popt/pspec.xml')
self.pdb = PackageDB('testdb')
self.spec.check()
def testAdd(self):
self.pdb.add_package(self.spec.packages[0])
self.assert_(self.pdb.has_package('popt-libs'))
ctx.repodb.add_repo('test', pisi.repodb.Repo(pisi.uri.URI('fakerepo.xml')) )
ctx.packagedb.add_package(self.spec.packages[0], 'test')
self.assert_(ctx.packagedb.has_package('popt-libs'))
# close the database and remove lock
self.pdb.close()
#self.pdb.close()
def testRemove(self):
self.pdb.remove_package('popt-libs')
self.assert_(not self.pdb.has_package('popt-libs'))
ctx.packagedb.remove_package('popt-libs', 'test')
self.assert_(not ctx.packagedb.has_package('popt-libs', 'test'))
self.pdb.close()
suite = unittest.makeSuite(PackageDBTestCase)