diff --git a/RefactorPlan b/RefactorPlan index c4b2efde..f3198733 100644 --- a/RefactorPlan +++ b/RefactorPlan @@ -141,3 +141,15 @@ currently PiSi wrotes everyting into logs which is not usefull. Also this inform * Lots of assertions used in the code with no following descriptive string information. +==> repo order + +* When trying to install a package or looking for a dependency for a package the first found package + in "repo order" is used. This design decision is not very good. When a new repo that has the latest + version of any package is added and if it is the last repo in order, pisi can not upgrade to this + package. + + We can remove this order thing and in this situation we can use the latest version of the package. + This should also lead to some other problems but between these two not very good solutions this + seems to be the better one. + + diff --git a/pisi/cli/db/__init__.py b/pisi/cli/db/__init__.py new file mode 100644 index 00000000..31121064 --- /dev/null +++ b/pisi/cli/db/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# diff --git a/pisi/cli/db/componentdb.py b/pisi/cli/db/componentdb.py new file mode 100644 index 00000000..c7b71623 --- /dev/null +++ b/pisi/cli/db/componentdb.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +import types + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.context as ctx +import pisi.db.itembyrepodb as itembyrepodb +import pisi.component + + +class ComponentDB(object): + + def __init__(self): + self.d = itembyrepodb.ItemByRepoDB('component') + + def close(self): + self.d.close() + + def destroy(self): + self.d.destroy() + + def has_component(self, name, repo = itembyrepodb.repos, txn = None): + name = str(name) + return self.d.has_key(name, repo, txn) + + def get_component(self, name, repo=None, txn = None): + try: + return self.d.get_item(name, repo, txn=txn) + except itembyrepodb.NotfoundError, e: + raise Error(_('Component %s not found') % name) + + def get_component_repo(self, name, repo=None, txn = None): + try: + return self.d.get_item_repo(name, repo, txn=txn) + except itembyrepodb.NotfoundError, e: + raise Error(_('Component %s not found') % name) + + def get_union_comp(self, name, txn = None, repo = itembyrepodb.repos ): + """get a union of all repository components packages, not just the first repo in order. + get only basic repo info from the first repo""" + def proc(txn): + s = self.d.d.get(name, txn=txn) + pkgs = set() + srcs = set() + for repostr in self.d.order(repo = repo): + if s.has_key(repostr): + pkgs |= set(s[repostr].packages) + srcs |= set(s[repostr].sources) + comp = self.get_component(name) + comp.packages = list(pkgs) + comp.sources = list(srcs) + return comp + return self.d.txn_proc(proc, txn) + + def list_components(self, repo=None): + return self.d.list(repo) + + # walk: walks through the underlying components' packages + def get_union_packages(self, component_name, walk=False, repo=pisi.db.itembyrepodb.repos, txn = None): + """returns union of all repository component's packages, not just the first repo's + component's in order""" + + component = self.get_union_comp(component_name, txn, repo) + if not walk: + return component.packages + + packages = [] + packages.extend(component.packages) + for dep in component.dependencies: + packages.extend(self.get_union_packages(dep, walk, repo, txn)) + + return packages + + # walk: walks through the underlying components' packages + def get_packages(self, component_name, walk=False, repo=None, txn = None): + """returns the given component's and underlying recursive components' packages""" + + component = self.get_component(component_name, repo, txn) + if not walk: + return component.packages + + packages = [] + packages.extend(component.packages) + for dep in component.dependencies: + packages.extend(self.get_packages(dep, walk, repo, txn)) + + return packages + + def add_child(self, component, repo, txn = None): + """update component tree""" + parent_name = ".".join(component.name.split(".")[:-1]) + if not parent_name: # root component + return + + if self.has_component(parent_name, repo, txn): + parent = self.get_component(parent_name, repo, txn) + else: + parent = pisi.component.Component(name = parent_name) + + if component.name not in parent.dependencies: + parent.dependencies.append(component.name) + self.d.add_item(parent_name, parent, repo, txn) + + def update_component(self, component, repo, txn = None): + def proc(txn): + if self.has_component(component.name, repo, txn): + # preserve list of sources, packages and dependencies + current = self.d.get_item(component.name, repo, txn) + component.packages = current.packages + component.sources = current.sources + component.dependencies = current.dependencies + self.d.add_item(component.name, component, repo, txn) + self.add_child(component, repo, txn) + self.d.txn_proc(proc, txn) + + def add_package(self, component_name, package, repo, txn = None): + def proc(txn): + assert component_name + if self.has_component(component_name, repo, txn): + component = self.get_component(component_name, repo, txn) + else: + component = pisi.component.Component( name = component_name ) + if not package in component.packages: + component.packages.append(package) + self.d.add_item(component_name, component, repo, txn) # update + self.add_child(component, repo, txn) + self.d.txn_proc(proc, txn) + + def remove_package(self, component_name, package, repo = None, txn = None): + def proc(txn, repo): + if not self.has_component(component_name, repo, txn): + raise Error(_('Information for component %s not available') % component_name) + if not repo: + repo = self.d.which_repo(component_name, txn=txn) # get default repo then + component = self.get_component(component_name, repo, txn) + if package in component.packages: + component.packages.remove(package) + self.d.add_item(component_name, component, repo, txn) # update + + ctx.txn_proc(lambda x: proc(txn, repo), txn) + + def add_spec(self, component_name, spec, repo, txn = None): + def proc(txn): + assert component_name + if self.has_component(component_name, repo, txn): + component = self.get_component(component_name, repo, txn) + else: + component = pisi.component.Component( name = component_name ) + if not spec in component.sources: + component.sources.append(spec) + self.d.add_item(component_name, component, repo, txn) # update + self.add_child(component, repo, txn) + self.d.txn_proc(proc, txn) + + def remove_spec(self, component_name, spec, repo = None, txn = None): + def proc(txn, repo): + if not self.has_component(component_name, repo, txn): + raise Error(_('Information for component %s not available') % component_name) + if not repo: + repo = self.d.which_repo(component_name, txn=txn) # get default repo then + component = self.get_component(component_name, repo, txn) + if spec in component.sources: + component.sources.remove(spec) + self.d.add_item(component_name, component, repo, txn) # update + + ctx.txn_proc(lambda x: proc(txn, repo), txn) + + def clear(self, txn = None): + self.d.clear(txn) + + def remove_component(self, name, repo = None, txn = None): + name = str(name) + self.d.remove_item(name, repo, txn) + + def remove_repo(self, repo, txn = None): + self.d.remove_repo(repo, txn=txn) diff --git a/pisi/cli/db/dbshelve.py b/pisi/cli/db/dbshelve.py new file mode 100644 index 00000000..6ddf3da4 --- /dev/null +++ b/pisi/cli/db/dbshelve.py @@ -0,0 +1,315 @@ +#------------------------------------------------------------------------ +# Copyright (c) 1997-2001 by Total Control Software +# All Rights Reserved +#------------------------------------------------------------------------ +# +# Module Name: dbShelve.py +# +# Description: A reimplementation of the standard shelve.py that +# forces the use of cPickle, and DB. +# +# Creation Date: 11/3/97 3:39:04PM +# +# License: This is free software. You may use this software for any +# purpose including modification/redistribution, so long as +# this header remains intact and that you do not claim any +# rights of ownership or authorship of this software. This +# software has been tested, but no warranty is expressed or +# implied. +# +# 13-Dec-2000: Updated to be used with the new bsddb3 package. +# Added DBShelfCursor class. +# +# 13-Dec-2005: Minor hacking by exa to make it work better with PiSi +#------------------------------------------------------------------------ + +"""Manage shelves of pickled objects using bsddb database files for the +storage. + +Add transaction processing by default to dictionary ops also +Also other minor improvements -- exa + +Now added support for overriding the marshalling method +""" + +#------------------------------------------------------------------------ + +import cPickle +import bsddb3.db as db +import bsddb3.dbobj as dbobj +import string +import pisi + +class CodingError(pisi.Error): + pass + +class DBShelf: + """A shelf to hold pickled objects, built upon a bsddb DB object. It + automatically pickles/unpickles data objects going to/from the DB. + """ + def __init__(self, dbenv = None): + self.dbenv = dbenv + # how lame is bsddb3? + if self.dbenv: + self.db = dbobj.DB(dbenv) + else: + self.db = db.DB(None) + + # it is better to explicitly close a shelf + #def __del__(self): + # self.close() + + allowed_chars = string.letters + string.digits + '-' + def check_key(key): + return pisi.util.all(lambda x: x in allowed_chars, key) + + def has_key(self, key, txn = None): + if txn: + return self.db.has_key(key, txn) + else: + return self.db.has_key(key) + + def txn_proc(self, proc, txn): + # can be used to txn protect a method automatically + if not txn: + if self.dbenv: + autotxn = self.dbenv.txn_begin() + try: + retval = proc(autotxn) + except db.DBError, e: + autotxn.abort() + raise pisi.Error, e + autotxn.commit() + else: # execute without transactions + retval = proc(None) + return retval + else: + return proc(txn) + + def decode(self, data): + try: + return cPickle.loads(data) + except cPickle.UnpicklingError: + raise CodingError() + + def encode(self, obj): + return cPickle.dumps(obj, 2) + + def clear(self, txn = None): + def proc(txn): + for x in self.keys(txn): + self.db.delete(x, txn) + self.txn_proc(proc, txn) + + def delete(self, x, txn): + def proc(txn): + self.db.delete(x, txn) + self.txn_proc(proc, txn) + + # another lame pythonic implementation method: + #def __getattr__(self, name): + # """Many methods we can just pass through to the DB object. + # (See below) + # """ + # print 'aptal bsddb3', name + # return getattr(self.db, name) + + #----------------------------------- + # Dictionary access methods + + def __len__(self): + return len(self.db) + + def __getitem__(self, key): + def proc(txn): + data = self.db.get(key) + return self.decode(data) + return self.txn_proc(proc, None) + + def __setitem__(self, key, value): + # hyperdandik transactions + def proc(txn): + data = self.encode(value) + self.db.put(key,data,txn) + return self.txn_proc(proc, None) + + def __delitem__(self, key): + txn = self.dbenv.txn_begin() + try: + self.db.delete(key, txn) + except db.DBError, e: + txn.abort() + raise e + txn.commit() + + def keys(self, txn=None): + if txn != None: + return self.db.keys(txn) + else: + return self.db.keys() + + def items(self, txn=None): + if txn != None: + items = self.db.items(txn) + else: + items = self.db.items() + newitems = [] + + for k, v in items: + newitems.append( (k, self.decode(v) ) ) + return newitems + + def values(self, txn=None): + if txn != None: + values = self.db.values(txn) + else: + values = self.db.values() + + return map(lambda x : self.decode(x), values) + + #----------------------------------- + # Other methods + + def __append(self, value, txn=None): + data = self.encode(value) + return self.db.append(data, txn) + + def append(self, value, txn=None): + if self.get_type() != db.DB_RECNO: + self.append = self.__append + return self.append(value, txn=txn) + raise db.DBError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO" + + + def associate(self, secondaryDB, callback, flags=0): + def _shelf_callback(priKey, priData, realCallback=callback): + data = self.decode(priData) + return realCallback(priKey, data) + return self.db.associate(secondaryDB, _shelf_callback, flags) + + + #def get(self, key, default=None, txn=None, flags=0): + def get(self, *args, **kw): + # We do it with *args and **kw so if the default value wasn't + # given nothing is passed to the extension module. That way + # an exception can be raised if set_get_returns_none is turned + # off. + data = apply(self.db.get, args, kw) + try: + return self.decode(data) + except (TypeError, CodingError): + return data # we may be getting the default value, or None, + # so it doesn't need unpickled. + + def get_both(self, key, value, txn=None, flags=0): + data = self.encode(value) + data = self.db.get(key, data, txn, flags) + return self.decode(data) + + def cursor(self, txn=None, flags=0): + c = DBShelfCursor(self.db.cursor(txn, flags)) + c.binary = self.binary + return c + + def put(self, key, value, txn=None, flags=0): + data = self.encode(value) + return self.db.put(key, data, txn, flags) + + def join(self, cursorList, flags=0): + raise NotImplementedError + + #---------------------------------------------- + # Methods allowed to pass-through to self.db + # + # close, delete, fd, get_byteswapped, get_type, has_key, + # key_range, open, remove, rename, stat, sync, + # upgrade, verify, and all set_* methods. + + +#--------------------------------------------------------------------------- + +class DBShelfCursor: + """ + """ + def __init__(self, cursor): + self.dbc = cursor + + def __del__(self): + self.close() + + def __getattr__(self, name): + """Some methods we can just pass through to the cursor object. (See below)""" + return getattr(self.dbc, name) + + #---------------------------------------------- + + def dup(self, flags=0): + return DBShelfCursor(self.dbc.dup(flags)) + + def put(self, key, value, flags=0): + data = self.encode(value) + return self.dbc.put(key, data, flags) + + + def get(self, *args): + count = len(args) # a method overloading hack + method = getattr(self, 'get_%d' % count) + apply(method, args) + + def get_1(self, flags): + rec = self.dbc.get(flags) + return self._extract(rec) + + def get_2(self, key, flags): + rec = self.dbc.get(key, flags) + return self._extract(rec) + + def get_3(self, key, value, flags): + data = self.encode(value) + rec = self.dbc.get(key, flags) + return self._extract(rec) + + + def current(self, flags=0): return self.get_1(flags|db.DB_CURRENT) + def first(self, flags=0): return self.get_1(flags|db.DB_FIRST) + def last(self, flags=0): return self.get_1(flags|db.DB_LAST) + def next(self, flags=0): return self.get_1(flags|db.DB_NEXT) + def prev(self, flags=0): return self.get_1(flags|db.DB_PREV) + def consume(self, flags=0): return self.get_1(flags|db.DB_CONSUME) + def next_dup(self, flags=0): return self.get_1(flags|db.DB_NEXT_DUP) + def next_nodup(self, flags=0): return self.get_1(flags|db.DB_NEXT_NODUP) + def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP) + + def get_both(self, key, value, flags=0): + data = self.encode(value) + rec = self.dbc.get_both(key, flags) + return self._extract(rec) + + def set(self, key, flags=0): + rec = self.dbc.set(key, flags) + return self._extract(rec) + + def set_range(self, key, flags=0): + rec = self.dbc.set_range(key, flags) + return self._extract(rec) + + def set_recno(self, recno, flags=0): + rec = self.dbc.set_recno(recno, flags) + return self._extract(rec) + + set_both = get_both + + def _extract(self, rec): + if rec is None: + return None + else: + key, data = rec + return key, self.decode(data) + + #---------------------------------------------- + # Methods allowed to pass-through to self.dbc + # + # close, count, delete, get_recno, join_item + + +#--------------------------------------------------------------------------- diff --git a/pisi/cli/db/filesdb.py b/pisi/cli/db/filesdb.py new file mode 100644 index 00000000..4480797c --- /dev/null +++ b/pisi/cli/db/filesdb.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +import pisi.db.lockeddbshelve as shelve +import pisi.files + +class FilesDB(shelve.LockedDBShelf): + ''' + FilesDB provides db access operations for Files objects (files.Files) + ''' + def __init__(self): + shelve.LockedDBShelf.__init__(self, 'files') + + def add_files(self, pkg_name, files, txn = None): + def proc(txn): + for x in files.list: + path = x.path + del x.path # don't store redundant attribute in db + self.put(path, (pkg_name, x), txn) + x.path = path # store it back in + self.txn_proc(proc, txn) + + def remove_files(self, files, txn = None): + def proc(txn): + for x in files.list: + if self.has_key(x.path): + self.delete(x.path, txn) + self.txn_proc(proc, txn) + + def has_file(self, path, txn = None): + return self.has_key(str(path), txn) + + def get_file(self, path, txn = None): + path = str(path) + def proc(txn): + if not self.has_key(path, txn): + return None + else: + (name, fileinfo) = self.get(path, txn) + fileinfo.path = path + return (name, fileinfo) + return self.txn_proc(proc, txn) + + def match_files(self, glob): + # NB: avoid using, this reads the entire db + import fnmatch + glob = str(glob) + infos = [] + for key in self.keys(): + if fnmatch.fnmatch(key, glob): + + # FIXME: Why should we assign path attribute manually + # in fileinfo? This is also done in get_file(), seems + # like a dirty workaround... - baris + name = self[key][0] + fileinfo = self[key][1] + fileinfo.path = key + infos.append((name, fileinfo)) + return infos diff --git a/pisi/cli/db/installdb.py b/pisi/cli/db/installdb.py new file mode 100644 index 00000000..08468a68 --- /dev/null +++ b/pisi/cli/db/installdb.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# +# +# installation database +# + +# System +import os +import fcntl + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +# PiSi +import pisi +import pisi.context as ctx +import pisi.db.lockeddbshelve as dbshelve +from pisi.files import Files +import pisi.util as util +from pisi.util import join_path + + +class InstallDBError(pisi.Error): + pass + + +class InstallInfo: + # some data is replicated from packagedb + # we store as an object, hey, we can waste O(1) space. + # this is also easier to modify in the future, without + # requiring database upgrades! wow! + def __init__(self, state, version, release, build, distribution, time): + self.state = state + self.version = version + self.release = release + self.build = build + self.distribution = distribution + self.time = time + + def one_liner(self): + import time + time_str = time.strftime("%d %b %Y %H:%M", self.time) + s = '%2s|%10s|%6s|%6s|%8s|%12s' % (self.state, self.version, self.release, + self.build, self.distribution, + time_str) + return s + + state_map = { 'i': _('installed'), 'ip':_('installed-pending'), + 'r':_('removed'), 'p': _('purged') } + + def __str__(self): + s = _("State: %s\nVersion: %s, Release: %s, Build: %s\n") % \ + (InstallInfo.state_map[self.state], self.version, + self.release, self.build) + import time + time_str = time.strftime("%d %b %Y %H:%M", self.time) + s += _('Distribution: %s, Install Time: %s\n') % (self.distribution, + time_str) + return s + + +class InstallDB: + + def __init__(self): + self.d = dbshelve.LockedDBShelf('install') + self.dp = dbshelve.LockedDBShelf('configpending') + self.files_dir = join_path(ctx.config.db_dir(), 'files') + + def close(self): + self.d.close() + self.dp.close() + + def files_name(self, pkg, version, release): + pkg_dir = self.pkg_dir(pkg, version, release) + return join_path(pkg_dir, ctx.const.files_xml) + + def files(self, pkg): + pkg = str(pkg) + pkginfo = self.d[pkg] + files = Files() + files.read(self.files_name(pkg,pkginfo.version,pkginfo.release)) + return files + + def pkg_dir(self, pkg, version, release): + return join_path(ctx.config.lib_dir(), 'package', + pkg + '-' + version + '-' + release) + + def is_recorded(self, pkg, txn = None): + pkg = str(pkg) + def proc(txn): + return self.d.has_key(pkg) + return self.d.txn_proc(proc, txn) + + def is_installed(self, pkg, txn = None): + pkg = str(pkg) + def proc(txn): + if self.is_recorded(pkg, txn): + info = self.d.get(pkg, txn) + return info.state=='i' or info.state=='ip' + else: + return False + return self.d.txn_proc(proc, txn) + + def list_installed(self, txn = None): + def proc(txn): + list = [] + for (pkg, info) in self.d.items(txn): + if info.state=='i' or info.state=='ip': + list.append(pkg) + return list + return self.d.txn_proc(proc, txn) + + def list_pending(self): + # warning: reads the entire db + dict = {} + for (pkg, x) in self.dp.items(): + pkginfo = self.d[pkg] + dict[pkg] = pkginfo + return dict + + def get_info(self, pkg): + pkg = str(pkg) + return self.d[pkg] + + def get_version(self, pkg): + pkg = str(pkg) + info = self.d[pkg] + return (info.version, info.release, info.build) + + def is_removed(self, pkg): + pkg = str(pkg) + if self.is_recorded(pkg): + info = self.d[pkg] + return info.state=='r' + else: + return False + + def install(self, pkg, version, release, build, distro = "", + config_later = False, rebuild=False, txn = None): + """install package with specific version, release, build""" + pkg = str(pkg) + def proc(txn): + if self.is_installed(pkg, txn): + raise InstallDBError(_("Already installed")) + if config_later: + state = 'ip' + self.dp.put(pkg, True, txn) + else: + state = 'i' + + # FIXME: it might be more appropriate to pass date + # as an argument, or installation data afterwards + # to do this -- exa + if not rebuild: + import time + ctime = time.localtime() + else: + files_xml = self.files_name(pkg, version, release) + ctime = util.creation_time(files_xml) + + self.d.put(pkg, InstallInfo(state, version, release, build, distro, ctime), txn) + + self.d.txn_proc(proc,txn) + + def clear_pending(self, pkg, txn = None): + pkg = str(pkg) + def proc(txn): + info = self.d.get(pkg, txn) + if self.is_installed(pkg, txn): + assert info.state == 'ip' + info.state = 'i' + self.d.put(pkg, info, txn) + self.dp.delete(pkg, txn) + self.d.txn_proc(proc,txn) + + def remove(self, pkg, txn = None): + pkg = str(pkg) + def proc(txn): + info = self.d.get(pkg, txn) + info.state = 'r' + self.d.put(pkg, info, txn) + if self.dp.has_key(pkg): + self.dp.delete(pkg, txn) + self.d.txn_proc(proc, txn) + + def purge(self, pkg, txn = None): + pkg = str(pkg) + def proc(txn): + if self.d.has_key(pkg, txn): + self.d.delete(pkg, txn) + self.d.txn_proc(proc, txn) + +db = None + +def init(): + global db + if db: + return db + + db = InstallDB() + return db + +def finalize(): + global db + if db: + db.close() + db = None diff --git a/pisi/cli/db/itembyrepodb.py b/pisi/cli/db/itembyrepodb.py new file mode 100644 index 00000000..84d12195 --- /dev/null +++ b/pisi/cli/db/itembyrepodb.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +import os +import fcntl +import types + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.util as util +import pisi.context as ctx +import pisi.db.lockeddbshelve as shelve + +installed, thirdparty, repos, all = range(1, 5) + +"""installed and thirdparty are special databases to keep track +of already installed stuff and third party stuff not in any real repository. +repos means search in repositories only, and all means search in +repositories and special databases (called tracking databases) +""" + +class Error(pisi.Error): + pass + +class NotfoundError(pisi.Error): + pass + +class ItemByRepoDB(object): + + def __init__(self, name): + self.d = shelve.LockedDBShelf(name) + #self.dbyrepo = shelve.LockedDBShelf(name + '-byrepo') + + def close(self): + self.d.close() + + def clear(self, txn = None): + self.d.clear(txn=txn) + + def txn_proc(self, proc, txn): + return self.d.txn_proc(proc, txn) + + def items(self): + return self.d.items() + + @staticmethod + def not_just_tracking(data): + keys = data.keys() + if len(keys)==1: + if 'trdparty' in keys or 'inst' in keys: + return False + elif len(keys)==2: + if 'trdparty' in keys and 'inst' in keys: + return False + return True + #below is a slower way + #for x in data.keys(): + # if x.startsWith('repo-'): + # return True + #return False + + def list_if(self, pred): + return [ k for k,data in self.d.items() if pred(k, data)] + + def list(self, repo = None): + if repo == None: + repo = repos + if repo not in [repos, all]: + return [ k for k,data in self.d.items() if data.has_key(self.repo_str(repo))] + else: + if repo == all: + return [ pkg for pkg in self.d.keys() ] + else: + return self.list_if( lambda k,data: ItemByRepoDB.not_just_tracking(data) ) + + # TODO: carry this to repodb, really :/ + def order(self, repo = None): + if repo == None: + repo = repos + assert repo in [all, repos] + order = [ 'repo-' + x for x in ctx.repodb.list() ] + if repo == all: + order += ['trdparty', 'inst'] + return order + +# def list_repo(self, repo): +# return self.dbyrepo[repo] + + def repo_str(self, repo): + if repo==thirdparty: + repo='trdparty' + elif repo==installed: + repo='inst' + else: + assert type(repo) == type("") + repo='repo-'+repo + return repo + + def str_repo(self, str): + if str.startswith('repo-'): + return str[5:] + elif str=='trdparty': + return thirdparty + elif str=='inst': + return installed + else: + raise Error(_('Invalid repository string')) + + def has_key(self, name, repo = None, txn = None): + name = str(name) + if repo == None: + repo = repos + haskey = self.d.has_key(name, txn) + if repo == all: + return haskey + elif repo == repos: + data = self.d.get(name, txn) + return haskey and ItemByRepoDB.not_just_tracking(data) + else: + repostr = self.repo_str(repo) + return haskey and self.d.get(name, txn).has_key(repostr) + + def get_item_repo(self, name, repo = None, txn = None): + name = str(name) + if repo == None: + repo = repos + def proc(txn): + if not self.d.has_key(name, txn=txn): + raise NotfoundError(_('Key %s not found') % name) + s = self.d.get(name, txn=txn) + if repo in [repos, all]: + for repostr in self.order(repo): + if s.has_key(repostr): + return (s[repostr], self.str_repo(repostr)) + else: + repostr = self.repo_str(repo) + if s.has_key(repostr): + return (s[repostr], repo) + raise NotfoundError(_('Key %s in repo %s not found') % (name, repo)) + #return None + + return self.d.txn_proc(proc, txn) + + def get_item(self, name, repo = None, txn = None): + if repo == None: + repo = repos + x = self.get_item_repo(name, repo, txn) + if x: + item, repo = x + # discard repo, not always needed + return item + else: + return None + + def which_repo(self, name, txn = None): + x = self.get_item_repo(name, txn=txn) + if x: + item, repo = x + return repo + else: + return None + + def add_item(self, name, obj, repo, txn = None): + assert not repo in [all, repos] + repostr = self.repo_str(repo) + def proc(txn): + if not self.d.has_key(name): + s = dict() + else: + s = self.d.get(name, txn) + s[ repostr ] = obj + self.d.put(name, s, txn) + self.d.txn_proc(proc, txn) + + def remove_item_repo(self, name, repo, txn = None): + assert not repo in [all, repos] + name = str(name) + def p(txn): + s = self.d.get(name, txn) + repostr = self.repo_str(repo) + if s.has_key(repostr): + del s[repostr] + if not len(s): + self.d.delete(name, txn) + else: + self.d.put(name, s, txn) + self.d.txn_proc(p, txn) + + def remove_item_only(self, name, txn = None): + def p(txn): + repo = self.which_repo(name, txn=txn) + self.remove_item_repo(name, repo, txn=txn) + self.d.txn_proc(p, txn) + + def remove_item(self, name, repo=None, txn=None): + if repo == None: + repo = repos + if not repo in [all, repos]: + self.remove_item_repo(name, repo,txn=txn) + else: + self.remove_item_only(name,txn=txn) + + def remove_repo(self, repo, txn = None): + def proc(txn): + for key in self.d.keys(): + self.remove_item_repo(key, repo, txn=txn) + self.d.txn_proc(proc, txn) diff --git a/pisi/cli/db/lockeddbshelve.py b/pisi/cli/db/lockeddbshelve.py new file mode 100644 index 00000000..2b3c0e9e --- /dev/null +++ b/pisi/cli/db/lockeddbshelve.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +import os +import fcntl +import types + +import bsddb3.db as db +import bsddb3.dbobj as dbobj +import pisi.db.dbshelve as dbshelve + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.context as ctx +import pisi.util as util + +from pisi.version import Version + +class Error(pisi.Error): + pass + +# check database version +# if write is given it knows it has write access +# if force is given it updates the specified db version +def check_dbversion(versionfile, ver, write=False, update=False): + verfn = util.join_path(ctx.config.db_dir(), versionfile) + firsttime = False + if os.path.exists(verfn): + verfile = file(verfn, 'r') + ls = verfile.readlines() + currver = Version(ls[0]) + dbver = Version(ver) + if currver < dbver: + if not update: + raise Error(_('Database version for %s insufficient. Please run rebuild-db command.') % versionfile) + else: + pass # continue to update, then + elif currver > dbver: + raise Error(_('Database version for %s greater than PiSi version. You need a newer PiSi.') % versionfile) + elif not update: + return True # db version is OK + else: + firsttime = True + if write and (update or firsttime): + if os.access(pisi.context.config.db_dir(), os.W_OK): + verfile = file(verfn, 'w') + verfile.write(ver) + verfile.close() + else: + raise Error(_('Cannot attain write access to database environment')) + else: + raise Error(_('Database version %s not present.') % versionfile) + +def lock_dbenv(): + ctx.dbenv_lock = file(util.join_path(pisi.context.config.db_dir(), 'dbenv.lock'), 'w') + try: + fcntl.flock(ctx.dbenv_lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + raise Error(_("Another instance of PiSi is running. Only one instance is allowed to modify the PiSi database at a time.")) + +# write: write access to database environment +# writeversion: would you like to be able +def init_dbenv(write=False, writeversion=False): + if os.access(pisi.context.config.db_dir(), os.R_OK): + # try to read version + check_dbversion('dbversion', pisi.__dbversion__, write=write, update=writeversion) + check_dbversion('filesdbversion', pisi.__filesdbversion__, write=write, update=writeversion) + else: + raise Error(_('Cannot attain read access to database environment')) + if write: + if os.access(pisi.context.config.db_dir(), os.W_OK): + lock_dbenv() + ctx.dbenv = dbobj.DBEnv() + flags = (db.DB_INIT_MPOOL | # cache + db.DB_INIT_TXN | # transaction subsystem + db.DB_INIT_LOG | # logging subsystem + db.DB_RECOVER | # run normal recovery + db.DB_CREATE) # allow db to create files + ctx.dbenv.set_cachesize(0, 4*1024*1024) + ctx.dbenv.open(pisi.context.config.db_dir(), flags) + ctx.dbenv.set_flags(db.DB_LOG_AUTOREMOVE, 1) # clear inactive logs automatically + else: + raise Error(_("Cannot attain write access to PiSi database. You have to be root for this operation.")) + else: + ctx.dbenv = None # read-only access to database + +class LockedDBShelf(dbshelve.DBShelf): + """A simple wrapper to implement locking for bsddb's dbshelf""" + + def __init__(self, dbname, mode=0644, + filetype=db.DB_BTREE, dbenv = None): + if dbenv == None: + dbenv = ctx.dbenv + dbshelve.DBShelf.__init__(self, dbenv) + filename = util.join_path(pisi.context.config.db_dir(), dbname + '.bdb') + if dbenv and os.access(os.path.dirname(filename), os.W_OK): + flags = 'w' + elif os.access(filename, os.R_OK): + flags = 'r' + else: + raise Error(_('Cannot attain read or write access to database %s') % dbname) + self.open(filename, dbname, filetype, flags, mode) + + def destroy(self): + os.unlink(self.filename) + + def __del__(self): + # superclass does something funky, we don't need that + pass + + def open(self, filename, dbname, filetype, flags=db.DB_CREATE, mode=0644): + self.filename = filename + self.closed = False + if type(flags) == type(''): + sflag = flags + if sflag == 'r': + flags = db.DB_RDONLY + elif sflag == 'rw': + flags = 0 + elif sflag == 'w': + flags = db.DB_CREATE + elif sflag == 'c': + flags = db.DB_CREATE + elif sflag == 'n': + flags = db.DB_TRUNCATE | db.DB_CREATE + else: + raise Error, _("Flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags") + self.flags = flags + if self.flags & db.DB_RDONLY == 0: + flags |= db.DB_AUTO_COMMIT # use txn subsystem in write mode + self.lock() + filename = os.path.realpath(filename) # we give absolute path due to dbenv + #print 'opening', filename, filetype, flags, mode + return self.db.open(filename, None, filetype, flags, mode) + + def lock(self): + self.lockfile = file(self.filename + '.lock', 'w') + try: + fcntl.flock(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + raise Error(_("Another instance of PiSi is running. Only one instance is allowed to modify the PiSi database at a time.")) + + def close(self): + if self.closed: + return + self.db.close() + if self.flags & db.DB_RDONLY == 0: + self.unlock() + self.closed = True + + def unlock(self): + self.lockfile.close() + os.unlink(self.filename + '.lock') + + @staticmethod + def encodekey(key): + '''utility method for dbs that must store unicodes in keys''' + if type(key)==types.UnicodeType: + return key.encode('utf-8') + elif type(key)==types.StringType: + return key + else: + raise Error('Key must be either string or unicode') diff --git a/pisi/cli/db/packagedb.py b/pisi/cli/db/packagedb.py new file mode 100644 index 00000000..a90ee0e1 --- /dev/null +++ b/pisi/cli/db/packagedb.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +"""package database +interface for update/query to local package repository + +we basically store everything in PackageInfo class +yes, we are cheap +""" + +import os +import fcntl +import types + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.util as util +import pisi.context as ctx +import pisi.db.itembyrepodb as itembyrepodb + + +class Error(pisi.Error): + pass + +class NotfoundError(pisi.Error): + def __init__(self, pkg): + pisi.Error.__init__("Package %s not found" % pkg) + self.pkg = pkg + +class PackageDB(object): + """PackageDB class provides an interface to the package database + using shelf objects""" + + def __init__(self): + self.d = itembyrepodb.ItemByRepoDB('package') + self.dbHistory = itembyrepodb.ItemByRepoDB('package_history') + self.dr = itembyrepodb.ItemByRepoDB('revdep') + + def close(self): + self.d.close() + self.dbHistory.close() + self.dr.close() + + def destroy(self): + self.d.destroy() + self.dbHistory.destroy() + self.dr.destroy() + + def clear(self, txn = None): + self.d.clear() + self.dbHistory.clear() + self.dr.clear() + + def has_package(self, name, repo=None, txn = None): + return self.d.has_key(name, repo, txn=txn) + + def get_package(self, name, repo=None, txn = None): + try: + return self.d.get_item(name, repo, txn=txn) + except pisi.db.itembyrepodb.NotfoundError, e: + raise Error(_('Package %s not found') % name) + + def get_history(self, name, repo=None, txn = None): + try: + return self.dbHistory.get_item(name, repo, txn=txn) + except pisi.db.itembyrepodb.NotfoundError, e: + raise Error(_('History for Package %s not found') % name) + + def get_package_repo(self, name, repo=None, txn = None): + return self.d.get_item_repo(name, repo, txn=txn) + + def which_repo(self, name, txn = None): + return self.d.which_repo(name, txn=txn) + + def get_rev_deps(self, name, repo = None, txn = None): + if self.dr.has_key(name, repo, txn=txn): + return self.dr.get_item(name, repo, txn=txn) + else: + return [] + + def get_deps(self, name, repo = None, txn = None): + if self.d.has_key(name, repo, txn=txn): + pinfo = self.d.get_item(name, repo, txn=txn) + return pinfo.packageDependencies + else: + return [] + + def list_packages(self, repo=None): + return self.d.list(repo) + + def add_package(self, package_info, repo, txn = None): + name = str(package_info.name) + + def proc(txn): + self.dbHistory.add_item(name, package_info.history, repo, txn) + package_info.history = None + self.d.add_item(name, package_info, repo, txn) + for dep in package_info.runtimeDependencies(): + dep_name = str(dep.package) + if self.dr.has_key(dep_name, repo, txn): + revdep = self.dr.get_item(dep_name, repo, txn) + revdep = filter(lambda (n,d):n!=name, revdep) + revdep.append( (name, dep) ) + self.dr.add_item(dep_name, revdep, repo, txn) + else: + self.dr.add_item(dep_name, [ (name, dep) ], repo, txn) + # add component + ctx.componentdb.add_package(package_info.partOf, package_info.name, repo, txn) + ctx.txn_proc(proc, txn) + + def remove_package(self, name, repo = None, txn = None): + name = str(name) + def proc(txn): + package_info = self.d.get_item(name, repo, txn=txn) + self.d.remove_item(name, repo, txn=txn) + self.dbHistory.remove_item(name, repo, txn=txn) + for dep in package_info.runtimeDependencies(): + dep_name = str(dep.package) + if self.dr.has_key(dep_name, repo, txn): + revdep = self.dr.get_item(dep_name, repo, txn) + revdep = filter(lambda (n,d):n!=name, revdep) + if revdep: + self.dr.add_item(dep_name, revdep, repo, txn) + else: + # Bug 3558: removal of revdep list of a package from revdepdb + # should only be done by the list members (dep. packages), not + # the package itself. So if a package is removed, it is removed + # from packagedb but its revdepdb part may still exist, until + # all the list members are removed. + self.dr.remove_item(dep_name, repo, txn=txn) + + # remove from component + ctx.componentdb.remove_package(package_info.partOf, package_info.name, repo, txn) + + self.d.txn_proc(proc, txn) + + def remove_repo(self, repo, txn = None): + def proc(txn): + self.d.remove_repo(repo, txn=txn) + self.dr.remove_repo(repo, txn=txn) + self.d.txn_proc(proc, txn) + +pkgdb = None + +def remove_tracking_package(name, txn = None): + # remove the guy from the tracking databases + if pkgdb.has_package(name, itembyrepodb.installed, txn=txn): + pkgdb.remove_package(name, itembyrepodb.installed, txn=txn) + if pkgdb.has_package(name, itembyrepodb.thirdparty, txn=txn): + pkgdb.remove_package(name, itembyrepodb.thirdparty, txn=txn) + +def init_db(): + global pkgdb + pkgdb = PackageDB() + return pkgdb + +def finalize_db(): + global pkgdb + if pkgdb: + pkgdb.close() diff --git a/pisi/cli/db/repodb.py b/pisi/cli/db/repodb.py new file mode 100644 index 00000000..dbea2ab9 --- /dev/null +++ b/pisi/cli/db/repodb.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.db.lockeddbshelve as shelve +import pisi.context as ctx +import pisi.db.packagedb as packagedb + +class Error(pisi.Error): + pass + +class Repo: + def __init__(self, indexuri): + self.indexuri = indexuri + +#class HttpRepo + +#class FtpRepo + +#class RemovableRepo + + +class RepoDB(object): + """RepoDB maps repo ids to repository information""" + + def __init__(self, txn = None): + self.d = shelve.LockedDBShelf("repo") + def proc(txn): + if not self.d.has_key("order", txn): + self.d.put("order", [], txn) + self.d.txn_proc(proc, txn) + + def close(self): + self.d.close() + + def repo_name(self, ix): + l = self.list() + return l[ix] + + def has_repo(self, name): + name = str(name) + return self.d.has_key("repo-" + name) + + def get_repo(self, name): + name = str(name) + return self.d["repo-" + name] + + def set_default_repo(self, name, txn = None): + name = str(name) + def proc(txn): + order = self.d.get("order", txn) + try: + index = order.index(name) + order[0], order[index] = order[index], order[0] + self.d.put("order", order, txn) + except ValueError: + raise Error(_('No repository named %s exists') % name) + self.d.txn_proc(proc, txn) + + def add_repo(self, name, repo_info, txn = None, at = None): + """add repository with name and repo_info at a given optional position""" + name = str(name) + assert (isinstance(repo_info,Repo)) + def proc(txn): + if self.d.has_key("repo-" + name, txn): + raise Error(_('Repository %s already exists') % name) + self.d.put("repo-" + name, repo_info, txn) + order = self.d.get("order", txn) + if at == None: + order.append(name) + else: + if at<0 or at>len(order): + raise Error(_("Cannot add repository at position %s") % at) + order.insert(at, name) + self.d.put("order", order, txn) + self.d.txn_proc(proc, txn) + + def list(self): + return self.d["order"] + + def clear(self): + self.d.clear() + + def remove_repo(self, name, txn = None): + name = str(name) + def proc(txn): + self.d.delete("repo-" + name, txn) + list = self.d.get("order", txn) + list.remove(name) + self.d.put("order", list, txn) + ctx.packagedb.remove_repo(name, txn=txn) + ctx.sourcedb.remove_repo(name, txn=txn) + ctx.componentdb.remove_repo(name, txn=txn) + self.d.txn_proc(proc, txn) + +db = None + +def init(): + global db + + if db: + return db + + db = RepoDB() + return db + +def finalize(): + global db + + if db: + db.close() + db = None diff --git a/pisi/cli/db/sourcedb.py b/pisi/cli/db/sourcedb.py new file mode 100644 index 00000000..e692772b --- /dev/null +++ b/pisi/cli/db/sourcedb.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +""" +package source database +interface for update/query to local package repository +we basically store everything in sourceinfo class +yes, we are cheap +to handle multiple repositories, for sources, we +store a set of repositories in which the source appears. +the actual guy to take is determined from the repo order. +""" + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi.context as ctx +import pisi.db.repodb +import pisi.db.itembyrepodb as itembyrepodb + +class NotfoundError(pisi.Error): + pass + +class SourceDB(object): + + def __init__(self): + self.d = itembyrepodb.ItemByRepoDB('source') + self.dpkgtosrc = itembyrepodb.ItemByRepoDB('pkgtosrc') + + def close(self): + self.d.close() + self.dpkgtosrc.close() + + def list(self): + return self.d.list() + + def has_spec(self, name, repo=None, txn=None): + return self.d.has_key(name, repo, txn) + + def get_spec(self, name, repo=None, txn = None): + try: + return self.d.get_item(name, repo, txn) + except pisi.db.itembyrepodb.NotfoundError, e: + raise NotfoundError(_("Source package %s not found") % name) + + def get_spec_repo(self, name, repo=None, txn = None): + try: + return self.d.get_item_repo(name, repo, txn) + except pisi.db.itembyrepodb.NotfoundError, e: + raise NotfoundError(_("Source package %s not found") % name) + + def pkgtosrc(self, name, txn = None): + return self.dpkgtosrc.get_item(name, txn=txn) + + def add_spec(self, spec, repo, txn = None): + assert not spec.errors() + name = str(spec.source.name) + def proc(txn): + self.d.add_item(name, spec, repo, txn) + for pkg in spec.packages: + self.dpkgtosrc.add_item(pkg.name, name, repo, txn) + ctx.componentdb.add_spec(spec.source.partOf, spec.source.name, repo, txn) + self.d.txn_proc(proc, txn) + + def remove_spec(self, name, repo, txn = None): + name = str(name) + def proc(txn): + assert self.has_spec(name, txn=txn) + spec = self.d.get_item(name, repo, txn) + self.d.remove_item(name, txn=txn) + for pkg in spec.packages: + self.dpkgtosrc.remove_item_repo(pkg.name, repo, txn) + ctx.componentdb.remove_spec(spec.source.partOf, spec.source.name, repo, txn) + + self.d.txn_proc(proc, txn) + + def remove_repo(self, repo, txn = None): + def proc(txn): + self.d.remove_repo(repo, txn=txn) + self.dpkgtosrc.remove_repo(repo, txn=txn) + self.d.txn_proc(proc, txn) + +sourcedb = None + +def init(): + global sourcedb + if sourcedb: + return sourcedb + + sourcedb = SourceDB() + return sourcedb + +def finalize(): + global sourcedb + if sourcedb: + sourcedb.close() + sourcedb = None