diff --git a/pisi.e3p b/pisi.e3p index 751716ce..8b52d72e 100644 --- a/pisi.e3p +++ b/pisi.e3p @@ -1,7 +1,7 @@ - + Python @@ -474,6 +474,10 @@ tests repodbtests.py + + pisi + itembyrepodb.py + @@ -515,10 +519,10 @@ {'standardLayout': 1} - - + + diff --git a/pisi/__init__.py b/pisi/__init__.py index 6ae512df..e044b4ae 100644 --- a/pisi/__init__.py +++ b/pisi/__init__.py @@ -12,9 +12,9 @@ # PiSi version -__version__ = "1.0.5" +__version__ = "1.0.6" -__dbversion__ = "1.0.5" +__dbversion__ = "1.0.6" __filesdbversion__ = "1.0.5" # yes, this is the real bottleneck __all__ = [ 'api', 'config', 'packagedb', 'installdb', 'search' ] diff --git a/pisi/api.py b/pisi/api.py index 878fd372..1d7ee9fa 100644 --- a/pisi/api.py +++ b/pisi/api.py @@ -51,20 +51,23 @@ class Error(pisi.Error): def init(database = True, options = None, ui = None, comar = True): """Initialize PiSi subsystem""" + # UI comes first + + if ui is None: + from pisi.cli import CLI + if options: + ctx.ui = CLI(options.debug) + else: + ctx.ui = CLI() + else: + ctx.ui = ui + import pisi.config ctx.config = pisi.config.Config(options) # TODO: this is definitely not dynamic beyond this point! ctx.comar = comar and not ctx.config.get_option('ignore_comar') - if ui is None: - if options: - pisi.context.ui = pisi.cli.CLI(options.debug) - else: - pisi.context.ui = pisi.cli.CLI() - else: - pisi.context.ui = ui - # initialize repository databases ctx.database = database if database: @@ -73,7 +76,7 @@ def init(database = True, options = None, ui = None, comar = True): ctx.installdb = pisi.installdb.init() ctx.filesdb = pisi.files.FilesDB() ctx.componentdb = pisi.component.ComponentDB() - packagedb.init_db() + ctx.packagedb = packagedb.init_db() ctx.sourcedb = pisi.sourcedb.init() pisi.search.init(['summary', 'description'], ['en', 'tr']) else: @@ -81,6 +84,7 @@ def init(database = True, options = None, ui = None, comar = True): ctx.installdb = None ctx.filesdb = None ctx.componentdb = None + ctx.packagedb = None ctx.sourcedb = None ctx.ui.debug('PISI API initialized') ctx.initialized = True @@ -89,11 +93,13 @@ def finalize(): if ctx.initialized: pisi.repodb.finalize() pisi.installdb.finalize() - if ctx.filesdb != None: + if ctx.filesdb: ctx.filesdb.close() - if ctx.componentdb != None: + if ctx.componentdb: ctx.componentdb.close() - packagedb.finalize_db() + if ctx.packagedb: + packagedb.finalize_db() + ctx.packagedb = None pisi.sourcedb.finalize() pisi.search.finalize() if ctx.dbenv: @@ -107,8 +113,7 @@ def list_available(): available = set() for repo in pisi.context.repodb.list(): - pkg_db = pisi.packagedb.get_db(repo) - available.update(pkg_db.list_packages()) + available.update(ctx.packagedb.list_packages()) return available def list_upgradable(): @@ -119,7 +124,7 @@ def list_upgradable(): Ap = [] for x in A: (version, release, build) = ctx.installdb.get_version(x) - pkg = packagedb.get_package(x) + pkg = ctx.packagedb.get_package(x) if ignore_build or (not build): if release < pkg.release: Ap.append(x) @@ -142,7 +147,7 @@ def package_graph(A, ignore_installed = False): # try to construct a pisi graph of packages to # install / reinstall - G_f = pgraph.PGraph(packagedb) # construct G_f + G_f = pgraph.PGraph(ctx.packagedb) # construct G_f # find the "install closure" graph of G_f by package # set A using packagedb @@ -153,7 +158,7 @@ def package_graph(A, ignore_installed = False): while len(B) > 0: Bp = set() for x in B: - pkg = packagedb.get_package(x) + pkg = ctx.packagedb.get_package(x) #print pkg for dep in pkg.runtimeDependencies(): if ignore_installed: @@ -169,14 +174,14 @@ def configure_pending(): # start with pending packages # configure them in reverse topological order of dependency A = ctx.installdb.list_pending() - G_f = pgraph.PGraph(packagedb) # construct G_f + G_f = pgraph.PGraph(ctx.packagedb) # construct G_f for x in A.keys(): G_f.add_package(x) B = A while len(B) > 0: Bp = set() for x in B.keys(): - pkg = packagedb.get_package(x) + pkg = ctx.packagedb.get_package(x) for dep in pkg.runtimeDependencies(): if dep.package in G_f.vertices(): G_f.add_dep(x, dep) @@ -233,12 +238,12 @@ def info_file(package_fn): def info_name(package_name, installed=False): """fetch package information for a package""" - if installed: - pkgdb = packagedb.inst_packagedb - else: - pkgdb = packagedb - if pkgdb.has_package(package_name): - package = pkgdb.get_package(package_name) + if ctx.packagedb.has_package(package_name): + package, repo = pkgdb.get_package(package_name) + + if (not installed) and (repo==pisi.itembyrepodb.installed): + raise Error(_('Package %s not found') % package_name) + from pisi.metadata import MetaData metadata = MetaData() metadata.package = package @@ -313,7 +318,7 @@ def update_repo(repo): index.read_uri(ctx.repodb.get_repo(repo).indexuri.get_uri(), repo) else: raise Error(_('No repository named %s found.') % repo) - index.update_db(repo) + ctx.txn_proc(lambda txn : index.update_db(repo, txn=txn)) ctx.ui.info(_('\n* Package database updated.')) def delete_cache(): diff --git a/pisi/atomicoperations.py b/pisi/atomicoperations.py index 55d31bde..d3c2b759 100644 --- a/pisi/atomicoperations.py +++ b/pisi/atomicoperations.py @@ -28,7 +28,7 @@ import pisi.packagedb as packagedb import pisi.dependency as dependency import pisi.util as util from pisi.specfile import * -from pisi.package import Package +#from pisi.package import Package from pisi.metadata import MetaData from pisi.files import Files from pisi.uri import URI @@ -38,6 +38,9 @@ import pisi.ui class Error(pisi.Error): pass +class NotfoundError(pisi.Error): + pass + # single package operations class AtomicOperation(object): @@ -61,10 +64,10 @@ class Install(AtomicOperation): def from_name(name): # download package and return an installer object # find package in repository - repo = packagedb.which_repo(name) + repo = ctx.packagedb.which_repo(name) if repo: repo = ctx.repodb.get_repo(repo) - pkg = packagedb.get_package(name) + pkg = ctx.packagedb.get_package(name) # FIXME: let pkg.packageURI be stored as URI type rather than string pkg_uri = URI(pkg.packageURI) @@ -83,7 +86,7 @@ class Install(AtomicOperation): def __init__(self, package_fname, ignore_dep = None): "initialize from a file name" super(Install, self).__init__(ignore_dep) - self.package = Package(package_fname) + self.package = pisi.package.Package(package_fname) self.package.read() self.metadata = self.package.metadata self.files = self.package.files @@ -146,9 +149,8 @@ class Install(AtomicOperation): # check if package is in database # If it is not, put it into 3rd party packagedb - if not packagedb.has_package(self.pkginfo.name): - db = packagedb.thirdparty_packagedb - db.add_package(self.pkginfo) + if not ctx.packagedb.has_package(self.pkginfo.name): + ctx.packagedb.add_package(self.pkginfo, pisi.itembyrepodb.thirdparty) # check file conflicts for file in self.files.list: @@ -299,10 +301,10 @@ class Install(AtomicOperation): txn) # filesdb - ctx.filesdb.add_files(self.metadata.package.name, self.files, txn) + ctx.filesdb.add_files(self.metadata.package.name, self.files, txn=txn) # installed packages - packagedb.inst_packagedb.add_package(self.pkginfo, txn) + ctx.packagedb.add_package(self.pkginfo, pisi.itembyrepodb.installed, txn=txn) def update_environment(self): # check if we have any shared objects or anything under @@ -345,7 +347,7 @@ class Remove(AtomicOperation): def __init__(self, package_name, ignore_dep = None): super(Remove, self).__init__(ignore_dep) self.package_name = package_name - self.package = packagedb.get_package(self.package_name) + self.package = ctx.packagedb.get_package(self.package_name) try: self.files = ctx.installdb.files(self.package_name) except pisi.Error, e: @@ -356,8 +358,7 @@ class Remove(AtomicOperation): def run(self): """Remove a single package""" - inst_packagedb = packagedb.inst_packagedb - + ctx.ui.status(_('Removing package %s') % self.package_name) ctx.ui.notify(pisi.ui.removing, package = self.package, files = self.files) if not ctx.installdb.is_installed(self.package_name): @@ -426,11 +427,7 @@ class Remove(AtomicOperation): def remove_db(self, txn): ctx.installdb.remove(self.package_name, txn) ctx.filesdb.remove_files(self.files, txn) - if packagedb.thirdparty_packagedb.has_package(self.package_name, txn): - packagedb.thirdparty_packagedb.remove_package(self.package_name, txn) - if packagedb.inst_packagedb.has_package(self.package_name, txn): - packagedb.inst_packagedb.remove_package(self.package_name, txn) - + ctx.packagedb.remove_tracking_package(self.package_name, txn) def remove_single(package_name): Remove(package_name).run() @@ -466,8 +463,8 @@ def virtual_install(metadata, files, txn): if ctx.installdb.is_installed(pkg.name, txn): if __is_virtual_upgrade(metadata): ctx.installdb.remove(pkg.name, txn) - packagedb.remove_package(pkg.name, txn) - ctx.filesdb.remove_files(ctx.installdb.files(pkg.name), txn) + ctx.packagedb.remove_package(pkg.name, txn=txn) + ctx.filesdb.remove_files(ctx.installdb.files(pkg.name), txn=txn) else: return @@ -488,7 +485,7 @@ def virtual_install(metadata, files, txn): ctx.filesdb.add_files(metadata.package.name, files, txn) # installed packages - packagedb.inst_packagedb.add_package(pkginfo, txn) + ctx.packagedb.add_package(pkginfo, pisi.itembyrepodb.installed, txn=txn) def resurrect_package(package_fn, write_files, txn = None): """Resurrect the package from xml files""" diff --git a/pisi/cli/commands.py b/pisi/cli/commands.py index 9c1ff59d..22aa0c29 100644 --- a/pisi/cli/commands.py +++ b/pisi/cli/commands.py @@ -283,8 +283,7 @@ conflicts relations starting from given packages. a = set() from pisi import packagedb for repo in ctx.repodb.list(): - pkg_db = packagedb.get_db(repo) - a = a.union(pkg_db.list_packages()) + a = a.union(ctx.packagedb.list_packages(repo)) return a def run(self): @@ -610,7 +609,7 @@ Usage: info ... metadata, files = pisi.api.info_name(arg, True) ctx.ui.info(_('Installed package:')) self.print_pkginfo(metadata, files) - if packagedb.has_package(arg): + elif ctx.packagedb.has_package(arg): metadata, files = pisi.api.info_name(arg, False) ctx.ui.info(_('Package found in repository:')) self.print_pkginfo(metadata, files) @@ -619,7 +618,7 @@ Usage: info ... import os.path ctx.ui.info(unicode(metadata.package)) - revdeps = [x[0] for x in packagedb.get_rev_deps(metadata.package.name)] + revdeps = [x[0] for x in ctx.packagedb.get_rev_deps(metadata.package.name)] print _('Reverse Dependencies:'), util.strlist(revdeps) if self.options.files or self.options.files_path: if files: @@ -722,7 +721,7 @@ Usage: list-installed ctx.ui.info(_('Package Name |St| Version| Rel.| Build| Distro| Date')) print '========================================================================' for pkg in list: - package = pisi.packagedb.inst_packagedb.get_package(pkg) + package = ctx.packagedb.get_package(pkg, pisi.itembyrepodb.installed) inst_info = ctx.installdb.get_info(pkg) if self.options.long: ctx.ui.info(unicode(package)) @@ -730,7 +729,7 @@ Usage: list-installed elif self.options.install_info: ctx.ui.info('%-15s |%s' % (package.name, inst_info.one_liner())) else: - ctx.ui.info('%15s - %s' % (package.name, package.summary)) + ctx.ui.info('%15s - %s' % (package.name, unicode(package.summary))) self.finalize() class RebuildDb(Command): @@ -924,12 +923,11 @@ Gives a brief list of PiSi packages published in the repository. from pisi import packagedb from colors import colorize - pkg_db = packagedb.get_db(repo) - list = pkg_db.list_packages() + list = ctx.packagedb.list_packages(repo) installed_list = ctx.installdb.list_installed() list.sort() for p in list: - package = pkg_db.get_package(p) + package = ctx.packagedb.get_package(p) if self.options.long: ctx.ui.info(unicode(package)) else: @@ -1045,7 +1043,7 @@ Usage: list-upgrades [ ... repon ] ctx.ui.info(_('Package Name |St| Version| Rel.| Build| Distro| Date')) print '========================================================================' for pkg in list: - package = pisi.packagedb.inst_packagedb.get_package(pkg) + package = ctx.packagedb.get_package(pkg, pisi.itembyrepodb.installed) inst_info = ctx.installdb.get_info(pkg) if self.options.long: ctx.ui.info(package) diff --git a/pisi/context.py b/pisi/context.py index 7f29cc4c..5b7b987d 100644 --- a/pisi/context.py +++ b/pisi/context.py @@ -33,6 +33,7 @@ ui = None # not now dbenv = None installdb = None +packagedb = None repodb = None invidx = None diff --git a/pisi/dbshelve.py b/pisi/dbshelve.py index e5236524..2927e421 100644 --- a/pisi/dbshelve.py +++ b/pisi/dbshelve.py @@ -21,6 +21,7 @@ # 13-Dec-2000: Updated to be used with the new bsddb3 package. # Added DBShelfCursor class. # +# 13-Dec-2005: Minor hacking by exa to make it work better with PISI #------------------------------------------------------------------------ """Manage shelves of pickled objects using bsddb database files for the @@ -29,6 +30,7 @@ storage. Add transaction processing by default to dictionary ops also Also other minor improvements -- exa +Now added support for overriding the marshalling method """ #------------------------------------------------------------------------ @@ -37,6 +39,11 @@ import cPickle import bsddb3.db as db import bsddb3.dbobj as dbobj +import pisi + +class CodingError(pisi.Error): + pass + class DBShelf: """A shelf to hold pickled objects, built upon a bsddb DB object. It automatically pickles/unpickles data objects going to/from the DB. @@ -48,7 +55,6 @@ class DBShelf: self.db = dbobj.DB(dbenv) else: self.db = db.DB(None) - self.binary = 1 # it is better to explicitly close a shelf #def __del__(self): @@ -76,6 +82,15 @@ class DBShelf: return retval else: return proc(txn) + + def decode(self, data): + try: + return cPickle.loads(data) + except: + raise CodingError() + + def encode(self, obj): + return cPickle.dumps(obj, 1) def clear(self, txn = None): def proc(txn): @@ -105,13 +120,13 @@ class DBShelf: def __getitem__(self, key): def proc(txn): data = self.db.get(key) - return cPickle.loads(data) + return self.decode(data) return self.txn_proc(proc, None) def __setitem__(self, key, value): # hyperdandik transactions def proc(txn): - data = cPickle.dumps(value, self.binary) + data = self.encode(value) self.db.put(key,data,txn) return self.txn_proc(proc, None) @@ -138,7 +153,7 @@ class DBShelf: newitems = [] for k, v in items: - newitems.append( (k, cPickle.loads(v)) ) + newitems.append( (k, self.decode(v) ) ) return newitems def values(self, txn=None): @@ -147,13 +162,13 @@ class DBShelf: else: values = self.db.values() - return map(cPickle.loads, values) + return map(lambda x : self.decode(x), values) #----------------------------------- # Other methods def __append(self, value, txn=None): - data = cPickle.dumps(value, self.binary) + data = self.encode(value) return self.db.append(data, txn) def append(self, value, txn=None): @@ -165,7 +180,7 @@ class DBShelf: def associate(self, secondaryDB, callback, flags=0): def _shelf_callback(priKey, priData, realCallback=callback): - data = cPickle.loads(priData) + data = self.decode(priData) return realCallback(priKey, data) return self.db.associate(secondaryDB, _shelf_callback, flags) @@ -178,15 +193,15 @@ class DBShelf: # off. data = apply(self.db.get, args, kw) try: - return cPickle.loads(data) - except (TypeError, cPickle.UnpicklingError): + return self.decode(data) + except (TypeError, CodingError): return data # we may be getting the default value, or None, # so it doesn't need unpickled. def get_both(self, key, value, txn=None, flags=0): - data = cPickle.dumps(value, self.binary) + data = self.encode(value) data = self.db.get(key, data, txn, flags) - return cPickle.loads(data) + return self.decode(data) def cursor(self, txn=None, flags=0): c = DBShelfCursor(self.db.cursor(txn, flags)) @@ -194,7 +209,7 @@ class DBShelf: return c def put(self, key, value, txn=None, flags=0): - data = cPickle.dumps(value, self.binary) + data = self.encode(value) return self.db.put(key, data, txn, flags) def join(self, cursorList, flags=0): @@ -229,7 +244,7 @@ class DBShelfCursor: return DBShelfCursor(self.dbc.dup(flags)) def put(self, key, value, flags=0): - data = cPickle.dumps(value, self.binary) + data = self.encode(value) return self.dbc.put(key, data, flags) @@ -247,7 +262,7 @@ class DBShelfCursor: return self._extract(rec) def get_3(self, key, value, flags): - data = cPickle.dumps(value, self.binary) + data = self.encode(value) rec = self.dbc.get(key, flags) return self._extract(rec) @@ -263,7 +278,7 @@ class DBShelfCursor: def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP) def get_both(self, key, value, flags=0): - data = cPickle.dumps(value, self.binary) + data = self.encode(value) rec = self.dbc.get_both(key, flags) return self._extract(rec) @@ -286,7 +301,7 @@ class DBShelfCursor: return None else: key, data = rec - return key, cPickle.loads(data) + return key, self.decode(data) #---------------------------------------------- # Methods allowed to pass-through to self.dbc diff --git a/pisi/dependency.py b/pisi/dependency.py index ebc6f177..c6779d3b 100644 --- a/pisi/dependency.py +++ b/pisi/dependency.py @@ -17,9 +17,8 @@ import gettext __trans = gettext.translation('pisi', fallback=True) _ = __trans.ugettext -#import pisi.db as db import pisi.context as ctx -import pisi.packagedb as packagedb +#import pisi.packagedb as packagedb from pisi.version import Version import pisi.pxml.autoxml as autoxml from pisi.util import Checks @@ -84,7 +83,7 @@ dependency spec""" if not ctx.installdb.is_installed(pkg_name): return False else: - pkg = packagedb.inst_packagedb.get_package(pkg_name) + pkg = ctx.packagedb.get_package(pkg_name) (version, release) = (pkg.version, pkg.release) return depinfo.satisfies(pkg_name, version, release) @@ -92,10 +91,10 @@ def repo_satisfies_dep(depinfo): """determine if a package in *repository* satisfies given dependency spec""" pkg_name = depinfo.package - if not packagedb.has_package(pkg_name): + if not ctx.packagedb.has_package(pkg_name): return False else: - pkg = packagedb.get_package(pkg_name) + pkg = ctx.packagedb.get_package(pkg_name) (version, release) = (pkg.version, pkg.release) return depinfo.satisfies(pkg_name, version, release) @@ -108,13 +107,13 @@ def satisfies_dependencies(pkg, deps, sat = installed_satisfies_dep): return True def satisfies_runtime_deps(pkg): - deps = packagedb.get_package(pkg).runtimeDependencies() + deps = ctx.packagedb.get_package(pkg).runtimeDependencies() return satisfies_dependencies(pkg, deps) def installable(pkg): """calculate if pkg name is installable currently which means it has to satisfy both install and runtime dependencies""" - if not packagedb.has_package(pkg): + if not ctx.packagedb.has_package(pkg): ctx.ui.info(_("Package %s is not present in the package database") % pkg); return False elif satisfies_runtime_deps(pkg): diff --git a/pisi/index.py b/pisi/index.py index d6be339c..a1245518 100644 --- a/pisi/index.py +++ b/pisi/index.py @@ -79,15 +79,14 @@ class Index(XmlFile): if fn == 'pspec.xml': self.add_spec(os.path.join(root, fn), repo_uri) - def update_db(self, repo): - pkgdb = packagedb.get_db(repo) - pkgdb.clear() + def update_db(self, repo, txn = None): for comp in self.components: - ctx.componentdb.update_component(comp) + ctx.componentdb.update_component(comp, txn=txn) + ctx.packagedb.remove_repo(repo, txn=txn) for pkg in self.packages: - pkgdb.add_package(pkg) + ctx.packagedb.add_package(pkg, repo, txn=txn) for sf in self.specs: - ctx.sourcedb.add_spec(sf, repo) + ctx.sourcedb.add_spec(sf, repo, txn=txn) def add_package(self, path, repo_uri): package = Package(path, 'r') diff --git a/pisi/installdb.py b/pisi/installdb.py index 3466ee30..806b6fb3 100644 --- a/pisi/installdb.py +++ b/pisi/installdb.py @@ -36,7 +36,7 @@ class InstallDBError(pisi.Error): class InstallInfo: - # some data is replicated from packagedb.inst_packagedb + # some data is replicated from packagedb # we store as an object, hey, we can waste O(1) space. # this is also easier to modify in the future, without # requiring database upgrades! wow! diff --git a/pisi/itembyrepodb.py b/pisi/itembyrepodb.py new file mode 100644 index 00000000..509d3c82 --- /dev/null +++ b/pisi/itembyrepodb.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# +# Author: Eray Ozkural + +import os +import fcntl +import types + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.util as util +import pisi.context as ctx +import pisi.lockeddbshelve as shelve + +installed, thirdparty = range(2) + +class Error(pisi.Error): + pass + +class NotfoundError(pisi.Error): + pass + +# the above are special databases to keep track of already installed stuff and +# third party stuff not in any real repository + +class ItemByRepoDB(object): + + def __init__(self, name): + self.d = shelve.LockedDBShelf(name) + #self.dbyrepo = shelve.LockedDBShelf(name + '-byrepo') + + def close(self): + self.d.close() + + def clear(self, txn = None): + self.d.clear(txn=txn) + + def txn_proc(self, proc, txn): + self.d.txn_proc(proc, txn) + + def items(self): + return self.d.items() + + def list(self, repo = None): + if repo: + return [ k for k,data in self.d.items() if data.has_key(self.repo_str(repo))] + else: + return [ pkg for pkg in self.d.keys() ] + + # TODO: carry this to repodb, really :/ + def order(self): + import pisi.repodb + order = [ 'repo-' + x for x in ctx.repodb.list() ] + ['trdparty', 'inst'] + return order + +# def list_repo(self, repo): +# return self.dbyrepo[repo] + + def repo_str(self, repo): + if repo==thirdparty: + repo='trdparty' + elif repo==installed: + repo='inst' + else: + repo='repo-'+repo + return repo + + def str_repo(self, str): + if str.startswith('repo-'): + return str[5:] + elif str=='trdparty': + return thirdparty + elif str=='inst': + return installed + else: + raise Error(_('Invalid repository string')) + + def has_key(self, name, repo = None, txn = None): + name = str(name) + if not repo: + return self.d.has_key(name, txn) + else: + return self.d.has_key(name, txn).has_key(repo) + + def get_item_repo(self, name, repo = None, txn = None): + name = str(name) + def proc(txn): + if not self.d.has_key(name, txn=txn): + raise NotfoundError(_('Key %s not found') % name) + s = self.d.get(name, txn=txn) + if not repo: + for repostr in self.order(): + if s.has_key(repostr): + return (s[repostr], self.str_repo(repostr)) + else: + repostr = self.repo_str(repo) + if s.has_key(repostr): + return (s[repostr], repo) + return None + + return self.d.txn_proc(proc, txn) + + def get_item(self, name, repo = None, txn = None): + x = self.get_item_repo(name, repo, txn) + if x: + item, repo = x + # discard repo, not always needed + return item + else: + return None + + def which_repo(self, name, repo = None, txn = None): + x = self.get_item_repo(name, repo, txn) + if x: + item, repo = x + return repo + else: + return None + + def add_item(self, name, obj, repo, txn = None): + repostr = self.repo_str(repo) + def proc(txn): + if not self.d.has_key(name): + s = dict() + else: + s = self.d.get(name, txn) + s[ repostr ] = obj + self.d.put(name, s, txn) + self.d.txn_proc(proc, txn) + + def remove_item(self, name, repo, txn = None): + name = str(name) + repo = str(repo) + def proc(txn): + s = self.d.get(name, txn) + del s[self.repo_str(repo)] + if (len(s)==0): + self.d.delete(name, txn) + else: + self.d.put(name, s, txn) + self.d.txn_proc(proc, txn) + + def remove_repo(self, repo, txn = None): + def proc(txn): + repostr = self.repo_str(repo) + for (key, s) in self.d.items(): + if s.has_key(repostr): + del s[repostr] + self.d.put(repo, s, txn) + self.d.txn_proc(proc, txn) diff --git a/pisi/lockeddbshelve.py b/pisi/lockeddbshelve.py index 35c8ff19..0be44ea4 100644 --- a/pisi/lockeddbshelve.py +++ b/pisi/lockeddbshelve.py @@ -9,16 +9,22 @@ # # Please read the COPYING file. # -# # Authors: Eray Ozkural +""" +By default, cPickle + 7zip compression is used +""" + +import os +import fcntl +import types +import cPickle + import bsddb3.db as db import bsddb3.dbobj as dbobj #import bsddb3.dbshelve as shelve import pisi.dbshelve as shelve -import os -import fcntl -import types +import pylzma import gettext __trans = gettext.translation('pisi', fallback=True) @@ -83,6 +89,15 @@ def init_dbenv(): class LockedDBShelf(shelve.DBShelf): """A simple wrapper to implement locking for bsddb's dbshelf""" +# def decode(self, data): +# try: +# return cPickle.loads(pylzma.decompress(data)) +# except: +# raise shelve.CodingError() + +# def encode(self, obj): +# return pylzma.compress(cPickle.dumps(obj, 1), algorithm=0) + def __init__(self, dbname, mode=0644, filetype=db.DB_BTREE, dbenv = None): if dbenv == None: diff --git a/pisi/operations.py b/pisi/operations.py index 100d589a..490b3824 100644 --- a/pisi/operations.py +++ b/pisi/operations.py @@ -168,7 +168,7 @@ def check_conflicts(order): while len(B) > 0: Bp = set() for x in B: - pkg = packagedb.get_package(x) # get latest version! + pkg = ctx.packagedb.get_package(x) # get latest version! #TODO: read conflicts from a conflicts db... for conflict in pkg.conflicts: if ctx.installdb.is_installed(self.pkginfo): @@ -249,7 +249,7 @@ def plan_install_pkg_names(A): # try to construct a pisi graph of packages to # install / reinstall - G_f = pgraph.PGraph(packagedb) # construct G_f + G_f = pgraph.PGraph(ctx.packagedb) # construct G_f # find the "install closure" graph of G_f by package # set A using packagedb @@ -260,7 +260,7 @@ def plan_install_pkg_names(A): while len(B) > 0: Bp = set() for x in B: - pkg = packagedb.get_package(x) + pkg = ctx.packagedb.get_package(x) for dep in pkg.runtimeDependencies(): ctx.ui.debug('checking %s' % str(dep)) # we don't deal with already *satisfied* dependencies @@ -299,7 +299,7 @@ def upgrade_pkg_names(A = []): ctx.ui.info(_('Package %s is not installed.') % x) continue (version, release, build) = ctx.installdb.get_version(x) - pkg = packagedb.get_package(x) + pkg = ctx.packagedb.get_package(x) if ignore_build or (not build) or (not pkg.build): if release < pkg.release: @@ -348,7 +348,7 @@ def plan_upgrade(A): # try to construct a pisi graph of packages to # install / reinstall - G_f = pgraph.PGraph(packagedb) # construct G_f + G_f = pgraph.PGraph(ctx.packagedb) # construct G_f # find the "install closure" graph of G_f by package # set A using packagedb @@ -481,7 +481,7 @@ def plan_remove(A): # try to construct a pisi graph of packages to # install / reinstall - G_f = pgraph.PGraph(packagedb) # construct G_f + G_f = pgraph.PGraph(ctx.packagedb) # construct G_f # find the (install closure) graph of G_f by package # set A using packagedb @@ -491,12 +491,12 @@ def plan_remove(A): while len(B) > 0: Bp = set() for x in B: - pkg = packagedb.get_package(x) - rev_deps = packagedb.get_rev_deps(x) + pkg = ctx.packagedb.get_package(x) + rev_deps = ctx.packagedb.get_rev_deps(x) for (rev_dep, depinfo) in rev_deps: # we don't deal with uninstalled rev deps # and unsatisfied dependencies (this is important, too) - if packagedb.inst_packagedb.has_package(rev_dep) and \ + if ctx.packagedb.has_package(rev_dep, pisi.itembyrepodb.installed) and \ dependency.installed_satisfies_dep(depinfo): if not rev_dep in G_f.vertices(): Bp.add(rev_dep) diff --git a/pisi/package.py b/pisi/package.py index 7a928d1f..a04d0d35 100644 --- a/pisi/package.py +++ b/pisi/package.py @@ -24,7 +24,6 @@ _ = __trans.ugettext import pisi import pisi.context as ctx import pisi.archive as archive -from pisi.packagedb import get_package from pisi.uri import URI from pisi.metadata import MetaData from pisi.files import Files diff --git a/pisi/packagedb.py b/pisi/packagedb.py index 79861480..fbc668e7 100644 --- a/pisi/packagedb.py +++ b/pisi/packagedb.py @@ -29,6 +29,7 @@ import pisi import pisi.util as util import pisi.context as ctx import pisi.lockeddbshelve as shelve +from pisi.itembyrepodb import ItemByRepoDB class Error(pisi.Error): pass @@ -36,10 +37,10 @@ class Error(pisi.Error): class PackageDB(object): """PackageDB class provides an interface to the package database using shelf objects""" - def __init__(self, id): - self.id = id - self.d = shelve.LockedDBShelf('package-%s' % id ) - self.dr = shelve.LockedDBShelf('revdep-%s' % id ) + + def __init__(self): + self.d = ItemByRepoDB('package') + self.dr = ItemByRepoDB('revdep') def close(self): self.d.close() @@ -49,43 +50,43 @@ class PackageDB(object): self.d.destroy() self.dr.destroy() - def has_package(self, name, txn = None): - name = str(name) - return self.d.has_key(name, txn) + def has_package(self, name, repo=None, txn = None): + return self.d.has_key(name, repo, txn) - def get_package(self, name): - name = str(name) - return self.d[name] + def get_package(self, name, repo=None, txn = None): + try: + return self.d.get_item(name, repo, txn) + except pisi.itembyrepodb.NotfoundError, e: + raise Error(_('Package %s not found') % name) + + def get_package_repo(self, name, repo=None, txn = None): + return self.d.get_item_repo(name, repo, txn) + + def which_repo(self, name, repo=None, txn = None): + return self.d.which_repo(name, repo, txn) def get_rev_deps(self, name): - name = str(name) - if self.dr.has_key(name): - return self.dr[name] - else: - return [] + x = self.dr.get_item(self, name) + if not x: + x = [] - def list_packages(self): - list = [] - for (pkg, x) in self.d.items(): - list.append(pkg) - return list + def list_packages(self, repo=None): + return self.d.list(repo) - #TODO: list_upgrades? - - def add_package(self, package_info, txn = None): + def add_package(self, package_info, repo, txn = None): name = str(package_info.name) def proc(txn): - self.d.put(name, package_info, txn) + self.d.add_item(name, package_info, repo, txn) for dep in package_info.runtimeDependencies(): dep_name = str(dep.package) - if self.dr.has_key(dep_name, txn): - revdep = self.dr.get(dep_name, txn) + if self.dr.has_key(dep_name, None, txn): + revdep = self.dr.get_item(dep_name, None, txn) revdep = filter(lambda (n,d):n!=name, revdep) revdep.append( (name, dep) ) - self.dr.put(dep_name, revdep, txn) + self.dr.add_item(dep_name, revdep, repo, txn) else: - self.dr.put(dep_name, [ (name, dep) ], txn) + self.dr.add_item(dep_name, [ (name, dep) ], repo, txn) # add component ctx.componentdb.add_package(package_info.partOf, package_info.name, txn) # index summary and description @@ -96,103 +97,44 @@ class PackageDB(object): if lang in ['en', 'tr']: pisi.search.add_doc('description', lang, package_info.name, doc, txn) - self.d.txn_proc(proc, txn) + ctx.txn_proc(proc, txn) - def clear(self): + def clear(self, txn = None): self.d.clear() + self.dr.clear() - def remove_package(self, name, txn = None): + def remove_package(self, name, repo = None, txn = None): name = str(name) def proc(txn): - package_info = self.d.get(name, txn) - self.d.delete(name, txn) + package_info = self.d.get_item_repo(name, repo, txn) + self.d.remove_item(name, repo, txn) #FIXME: what's happening to dr? #WORKAROUND: do not remove component if it is not in repo - if self.id.startswith('repo'): + if type(repo)==types.StringType: ctx.componentdb.remove_package(package_info.partOf, package_info.name, txn) self.d.txn_proc(proc, txn) + + def remove_repo(self, repo, txn = None): + def proc(txn): + self.d.remove_repo(repo, txn=txn) + self.dr.remove_repo(repo, txn=txn) + self.d.txn_proc(proc, txn) -packagedbs = {} +pkgdb = None -def add_db(name): - pisi.packagedb.packagedbs[name] = PackageDB('repo-' + name) - -def get_db(name): - return pisi.packagedb.packagedbs[name] - -def remove_db(name): - pisi.packagedb.packagedbs[name].close() - pisi.packagedb.packagedbs[name].destroy() - del pisi.packagedb.packagedbs[name] - -def has_package(name, txn = None): - def proc(txn): - repo = which_repo(name) - if repo or thirdparty_packagedb.has_package(name, txn) or inst_packagedb.has_package(name, txn): - return True - return False - return ctx.txn_proc(proc, txn) - -def which_repo(name, txn = None): - import pisi.repodb - def proc(txn): - for repo in pisi.repodb.db.list(): - if get_db(repo).has_package(name, txn): - return repo - return None - return ctx.txn_proc(proc, txn) - -def get_package(name): - repo = which_repo(name) - if repo: - return get_db(repo).get_package(name) - if thirdparty_packagedb.has_package(name): - return thirdparty_packagedb.get_package(name) - if inst_packagedb.has_package(name): - return inst_packagedb.get_package(name) - raise Error(_('get_package: package %s not found') % name) - -def get_rev_deps(name): - repo = which_repo(name) - if repo: - return get_db(repo).get_rev_deps(name) - if thirdparty_packagedb.has_package(name): - return thirdparty_packagedb.get_rev_deps(name) - if inst_packagedb.has_package(name): - return inst_packagedb.get_rev_deps(name) - - return [] - -def remove_package(name, txn = None): +def remove_tracking_package(name, txn = None): # remove the guy from the tracking databases - inst_packagedb.remove_package(name, txn) - if thirdparty_packagedb.has_package(name, txn): - thirdparty_packagedb.remove_package(name, txn) - -# tracking databases for non-repository information - -thirdparty_packagedb = inst_packagedb = None + if pkgdb.has_package(name, itembyrepodb.installed, txn): + pkgdb.remove_package(name, itembyrepodb.installed, txn=txn) + if pkgdb.has_package(name, itembyrepodb.thirdparty, txn): + pkgdb.remove_package(name, itembyrepodb.thirdparty, txn=txn) def init_db(): - global thirdparty_packagedb - global inst_packagedb - - if not thirdparty_packagedb: - thirdparty_packagedb = PackageDB('thirdparty') - if not pisi.packagedb.inst_packagedb: - inst_packagedb = PackageDB('installed') + global pkgdb + pkgdb = PackageDB() + return pkgdb def finalize_db(): - global thirdparty_packagedb - global inst_packagedb - global packagedbs - - if thirdparty_packagedb: - thirdparty_packagedb.close() - thirdparty_packagedb = None - - if inst_packagedb: - inst_packagedb.close() - inst_packagedb = None - - packagedbs.clear() + global pkgdb + if pkgdb: + pkgdb.close() diff --git a/pisi/pxml/autoxml.py b/pisi/pxml/autoxml.py index e90e8323..30d898af 100644 --- a/pisi/pxml/autoxml.py +++ b/pisi/pxml/autoxml.py @@ -345,7 +345,7 @@ class autoxml(oo.autosuper, oo.autoprop): # init hook if hasattr(self, 'init'): self.init(tag) - if uri: + if xmlfile_support and uri: self.read(uri, keepDoc, tmpDir) cls.__init__ = initialize diff --git a/pisi/repodb.py b/pisi/repodb.py index 489bb02a..9b71f1a6 100644 --- a/pisi/repodb.py +++ b/pisi/repodb.py @@ -49,11 +49,6 @@ class RepoDB(object): self.d.put("order", [], txn) self.d.txn_proc(proc, txn) - def init_dbs(self): - # initialize package/source dbs - for x in self.list(): - packagedb.add_db(x) - def close(self): self.d.close() @@ -88,7 +83,6 @@ class RepoDB(object): order = self.d.get("order", txn) order.append(name) self.d.put("order", order, txn) - packagedb.add_db(name) self.d.txn_proc(proc, txn) def list(self): @@ -104,7 +98,7 @@ class RepoDB(object): list = self.d.get("order", txn) list.remove(name) self.d.put("order", list, txn) - packagedb.remove_db(name) + ctx.packagedb.remove_repo(name, txn=txn) self.d.txn_proc(proc, txn) db = None @@ -116,7 +110,6 @@ def init(): return db db = RepoDB() - db.init_dbs() return db def finalize(): diff --git a/tests/packagedbtests.py b/tests/packagedbtests.py index 7d3acf54..e3a3808e 100644 --- a/tests/packagedbtests.py +++ b/tests/packagedbtests.py @@ -24,18 +24,18 @@ class PackageDBTestCase(testcase.TestCase): testcase.TestCase.setUp(self) self.spec = SpecFile() self.spec.read('tests/popt/pspec.xml') - - self.pdb = PackageDB('testdb') + self.spec.check() def testAdd(self): - self.pdb.add_package(self.spec.packages[0]) - self.assert_(self.pdb.has_package('popt-libs')) + ctx.repodb.add_repo('test', pisi.repodb.Repo(pisi.uri.URI('fakerepo.xml')) ) + ctx.packagedb.add_package(self.spec.packages[0], 'test') + self.assert_(ctx.packagedb.has_package('popt-libs')) # close the database and remove lock - self.pdb.close() + #self.pdb.close() def testRemove(self): - self.pdb.remove_package('popt-libs') - self.assert_(not self.pdb.has_package('popt-libs')) + ctx.packagedb.remove_package('popt-libs', 'test') + self.assert_(not ctx.packagedb.has_package('popt-libs', 'test')) self.pdb.close() suite = unittest.makeSuite(PackageDBTestCase)