diff --git a/TODO b/TODO index fbe82cc0..58436838 100644 --- a/TODO +++ b/TODO @@ -172,7 +172,8 @@ Legend: * high priority - / transaction stuff for database (eray) + + transaction stuff for database (eray) + + implement a simple (dandik) version for better reliability * medium priority @@ -197,10 +198,11 @@ Legend: - date type - enum type - make autoxml dependent on only xmlext and xmlfile interfaces - ? non-interactive use 5. Post Release + * high priority + - high-level build commands - build command enhancements: - aware of repo sources @@ -215,9 +217,10 @@ Legend: - sourceb - improve url support: consider adding "other" protocols - removable media: media:// - + * medium priority + - non-interactive use - multi-architecture support (baris, caglar) / design decisions / extend XML specs to support that diff --git a/pisi-cli b/pisi-cli index 526c0098..7cc6b750 100755 --- a/pisi-cli +++ b/pisi-cli @@ -23,7 +23,6 @@ import gettext __trans = gettext.translation('pisi', fallback=True) _ = __trans.ugettext - def exit(): pisi.api.finalize() sys.exit(1) diff --git a/pisi/__init__.py b/pisi/__init__.py index 20f069e9..19d32990 100644 --- a/pisi/__init__.py +++ b/pisi/__init__.py @@ -12,9 +12,9 @@ # PiSi version -__version__ = "1.0_beta3" +__version__ = "1.0_beta4" -__all__ = [ 'api', 'packagedb', 'installdb' ] +__all__ = [ 'api', 'config', 'packagedb', 'installdb' ] class Exception(Exception): """Class of exceptions that must be caught and handled within PISI""" diff --git a/pisi/api.py b/pisi/api.py index e0c2aab7..66120647 100644 --- a/pisi/api.py +++ b/pisi/api.py @@ -47,6 +47,7 @@ from pisi.atomicoperations import resurrect_package from pisi.metadata import MetaData from pisi.files import Files import pisi.search +import pisi.lockeddbshelve as shelve class Error(pisi.Error): pass @@ -70,6 +71,7 @@ def init(database = True, options = None, ui = None, comar = True): # initialize repository databases if database: + shelve.init_dbenv() ctx.repodb = pisi.repodb.init() ctx.installdb = pisi.installdb.init() ctx.filesdb = pisi.files.FilesDB() @@ -96,6 +98,8 @@ def finalize(): packagedb.finalize_db() pisi.sourcedb.finalize() pisi.search.finalize() + if ctx.dbenv: + ctx.dbenv.close() ctx.ui.debug('PISI API finalized') ctx.ui.close() ctx.initialized = False diff --git a/pisi/cli/commands.py b/pisi/cli/commands.py index 97f39f7a..30335c4d 100644 --- a/pisi/cli/commands.py +++ b/pisi/cli/commands.py @@ -628,9 +628,9 @@ Usage: list-installed ctx.ui.info(unicode(package)) ctx.ui.info(unicode(inst_info)) elif self.options.install_info: - ctx.ui.info('%-15s | %s ' % (package.name, inst_info.one_liner())) + ctx.ui.info('%-15s |%s' % (package.name, inst_info.one_liner())) else: - ctx.ui.info('%15s - %s ' % (package.name, package.summary)) + ctx.ui.info('%15s - %s' % (package.name, package.summary)) self.finalize() class RebuildDb(Command): diff --git a/pisi/component.py b/pisi/component.py index 63b10808..8ec4960d 100644 --- a/pisi/component.py +++ b/pisi/component.py @@ -97,7 +97,7 @@ class ComponentDB(object): def list_components(self): list = [] - for (pkg, x) in self.d.iteritems(): + for (pkg, x) in self.d.items(): list.append(pkg) return list diff --git a/pisi/dbshelve.py b/pisi/dbshelve.py new file mode 100644 index 00000000..f3d14529 --- /dev/null +++ b/pisi/dbshelve.py @@ -0,0 +1,284 @@ +#!/bin/env python +#------------------------------------------------------------------------ +# Copyright (c) 1997-2001 by Total Control Software +# All Rights Reserved +#------------------------------------------------------------------------ +# +# Module Name: dbShelve.py +# +# Description: A reimplementation of the standard shelve.py that +# forces the use of cPickle, and DB. +# +# Creation Date: 11/3/97 3:39:04PM +# +# License: This is free software. You may use this software for any +# purpose including modification/redistribution, so long as +# this header remains intact and that you do not claim any +# rights of ownership or authorship of this software. This +# software has been tested, but no warranty is expressed or +# implied. +# +# 13-Dec-2000: Updated to be used with the new bsddb3 package. +# Added DBShelfCursor class. +# +#------------------------------------------------------------------------ + +"""Manage shelves of pickled objects using bsddb database files for the +storage. +""" + +#------------------------------------------------------------------------ + +import cPickle +import bsddb3.db as db +import bsddb3.dbobj as dbobj + +class DBShelf: + """A shelf to hold pickled objects, built upon a bsddb DB object. It + automatically pickles/unpickles data objects going to/from the DB. + """ + def __init__(self, dbenv = None): + self.dbenv = dbenv + self.db = dbobj.DB(dbenv) + self.binary = 1 + + # it is better to explicitly close a shelf + #def __del__(self): + # self.close() + + def has_key(self, key): + return self.db.has_key(key) + + def clear(self, txn = None): + if not txn: + txn = self.dbenv.txn_begin() + try: + for x in self.keys(txn): + del self[x] + except db.DBError, e: + txn.abort() + raise e + txn.commit() + else: + for x in self.keys(txn): + del self[x] + + #def delete(self, txn = None): + # self.db.delete(self, txn + + # another lame pythonic implementation method: + #def __getattr__(self, name): + # """Many methods we can just pass through to the DB object. + # (See below) + # """ + # print 'aptal bsddb3', name + # return getattr(self.db, name) + + #----------------------------------- + # Dictionary access methods + + def __len__(self): + return len(self.db) + + def __getitem__(self, key): + txn = self.dbenv.txn_begin() + try: + data = self.db.get(key) + except db.DBError, e: + txn.abort() + raise e + txn.commit() + return cPickle.loads(data) + + def __setitem__(self, key, value): + # hyperdandik transactions + txn = self.dbenv.txn_begin() + data = cPickle.dumps(value, self.binary) + try: + self.db.put(key,data,txn) + #print 'put', value + except db.DBError, e: + txn.abort() + raise e + txn.commit() + + def __delitem__(self, key): + txn = self.dbenv.txn_begin() + try: + self.db.delete(key, txn) + except db.DBError, e: + txn.abort() + raise e + txn.commit() + + def keys(self, txn=None): + if txn != None: + return self.db.keys(txn) + else: + return self.db.keys() + + def items(self, txn=None): + if txn != None: + items = self.db.items(txn) + else: + items = self.db.items() + newitems = [] + + for k, v in items: + newitems.append( (k, cPickle.loads(v)) ) + return newitems + + def values(self, txn=None): + if txn != None: + values = self.db.values(txn) + else: + values = self.db.values() + + return map(cPickle.loads, values) + + #----------------------------------- + # Other methods + + def __append(self, value, txn=None): + data = cPickle.dumps(value, self.binary) + return self.db.append(data, txn) + + def append(self, value, txn=None): + if self.get_type() != db.DB_RECNO: + self.append = self.__append + return self.append(value, txn=txn) + raise db.DBError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO" + + + def associate(self, secondaryDB, callback, flags=0): + def _shelf_callback(priKey, priData, realCallback=callback): + data = cPickle.loads(priData) + return realCallback(priKey, data) + return self.db.associate(secondaryDB, _shelf_callback, flags) + + + #def get(self, key, default=None, txn=None, flags=0): + def get(self, *args, **kw): + # We do it with *args and **kw so if the default value wasn't + # given nothing is passed to the extension module. That way + # an exception can be raised if set_get_returns_none is turned + # off. + data = apply(self.db.get, args, kw) + try: + return cPickle.loads(data) + except (TypeError, cPickle.UnpicklingError): + return data # we may be getting the default value, or None, + # so it doesn't need unpickled. + + def get_both(self, key, value, txn=None, flags=0): + data = cPickle.dumps(value, self.binary) + data = self.db.get(key, data, txn, flags) + return cPickle.loads(data) + + def cursor(self, txn=None, flags=0): + c = DBShelfCursor(self.db.cursor(txn, flags)) + c.binary = self.binary + return c + + def put(self, key, value, txn=None, flags=0): + data = cPickle.dumps(value, self.binary) + return self.db.put(key, data, txn, flags) + + def join(self, cursorList, flags=0): + raise NotImplementedError + + #---------------------------------------------- + # Methods allowed to pass-through to self.db + # + # close, delete, fd, get_byteswapped, get_type, has_key, + # key_range, open, remove, rename, stat, sync, + # upgrade, verify, and all set_* methods. + + +#--------------------------------------------------------------------------- + +class DBShelfCursor: + """ + """ + def __init__(self, cursor): + self.dbc = cursor + + def __del__(self): + self.close() + + def __getattr__(self, name): + """Some methods we can just pass through to the cursor object. (See below)""" + return getattr(self.dbc, name) + + #---------------------------------------------- + + def dup(self, flags=0): + return DBShelfCursor(self.dbc.dup(flags)) + + def put(self, key, value, flags=0): + data = cPickle.dumps(value, self.binary) + return self.dbc.put(key, data, flags) + + + def get(self, *args): + count = len(args) # a method overloading hack + method = getattr(self, 'get_%d' % count) + apply(method, args) + + def get_1(self, flags): + rec = self.dbc.get(flags) + return self._extract(rec) + + def get_2(self, key, flags): + rec = self.dbc.get(key, flags) + return self._extract(rec) + + def get_3(self, key, value, flags): + data = cPickle.dumps(value, self.binary) + rec = self.dbc.get(key, flags) + return self._extract(rec) + + + def current(self, flags=0): return self.get_1(flags|db.DB_CURRENT) + def first(self, flags=0): return self.get_1(flags|db.DB_FIRST) + def last(self, flags=0): return self.get_1(flags|db.DB_LAST) + def next(self, flags=0): return self.get_1(flags|db.DB_NEXT) + def prev(self, flags=0): return self.get_1(flags|db.DB_PREV) + def consume(self, flags=0): return self.get_1(flags|db.DB_CONSUME) + def next_dup(self, flags=0): return self.get_1(flags|db.DB_NEXT_DUP) + def next_nodup(self, flags=0): return self.get_1(flags|db.DB_NEXT_NODUP) + def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP) + + def get_both(self, key, value, flags=0): + data = cPickle.dumps(value, self.binary) + rec = self.dbc.get_both(key, flags) + return self._extract(rec) + + def set(self, key, flags=0): + rec = self.dbc.set(key, flags) + return self._extract(rec) + + def set_range(self, key, flags=0): + rec = self.dbc.set_range(key, flags) + return self._extract(rec) + + def set_recno(self, recno, flags=0): + rec = self.dbc.set_recno(recno, flags) + return self._extract(rec) + + set_both = get_both + + def _extract(self, rec): + if rec is None: + return None + else: + key, data = rec + return key, cPickle.loads(data) + + #---------------------------------------------- + # Methods allowed to pass-through to self.dbc + # + # close, count, delete, get_recno, join_item + + +#--------------------------------------------------------------------------- diff --git a/pisi/files.py b/pisi/files.py index f186c734..f0b912e8 100644 --- a/pisi/files.py +++ b/pisi/files.py @@ -56,7 +56,7 @@ class FilesDB(shelve.LockedDBShelf): def remove_files(self, files): for x in files.list: - self.delete(str(x.path)) + del self[str(x.path)] def has_file(self, path): return self.has_key(str(path)) diff --git a/pisi/installdb.py b/pisi/installdb.py index dc6e1f1f..e5792982 100644 --- a/pisi/installdb.py +++ b/pisi/installdb.py @@ -110,14 +110,14 @@ class InstallDB: def list_installed(self): list = [] - for (pkg, info) in self.d.iteritems(): + for (pkg, info) in self.d.items(): if info.state=='i' or info.state=='ip': list.append(pkg) return list def list_pending(self): dict = {} - for (pkg, x) in self.dp.iteritems(): + for (pkg, x) in self.dp.items(): pkginfo = self.d[pkg] dict[pkg] = pkginfo return dict diff --git a/pisi/lockeddb.py b/pisi/lockeddb.py new file mode 100644 index 00000000..baf2be92 --- /dev/null +++ b/pisi/lockeddb.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# +# +# Authors: Eray Ozkural + +import bsddb3.db as db +import os +import fcntl +import types + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.context + +class Error(pisi.Error): + pass + +class LockedDB: + + def __init__(self, dbname, mode=0644, + filetype=db.DB_BTREE, dbenv=None): + if not dbenv: + dbenv = ctx.dbenv + filename = os.path.join( pisi.context.config.db_dir(), dbname + '.bdb') + if os.access(os.path.dirname(filename), os.W_OK): + flags = 'w' + elif os.access(filename, os.R_OK): + flags = 'r' + else: + raise Error(_('Cannot attain read or write access to database %s') % dbname) + self.open(filename, dbname, filetype, flags, mode) + + def __del__(self): + # superclass does something funky, we don't need that + pass + + def open(self, filename, dbname, filetype, flags=db.DB_CREATE, mode=0644): + self.filename = filename + self.closed = False + #print 'open', filename + if type(flags) == type(''): + sflag = flags + if sflag == 'r': + flags = db.DB_RDONLY + elif sflag == 'rw': + flags = 0 + elif sflag == 'w': + flags = db.DB_CREATE + elif sflag == 'c': + flags = db.DB_CREATE + elif sflag == 'n': + flags = db.DB_TRUNCATE | db.DB_CREATE + else: + raise Error, _("Flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags") + self.flags = flags + pisi.util.check_dir(pisi.context.config.db_dir()) + if self.flags != db.DB_RDONLY: + self.lock() + return self.db.open(filename, dbname, filetype, flags, mode) + + def lock(self): + self.lockfile = file(self.filename + '.lock', 'w') + try: + fcntl.flock(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + raise Error(_("Another instance of PISI is running. Try later!")) + + def close(self): + if self.closed: + return + self.db.close() + if self.flags != db.DB_RDONLY: + self.unlock() + self.closed = True + + def unlock(self): + self.lockfile.close() + os.unlink(self.filename + '.lock') + + @staticmethod + def encodekey(key): + '''utility method for dbs that must store unicodes in keys''' + if type(key)==types.UnicodeType: + return key.encode('utf-8') + elif type(key)==types.StringType: + return key + else: + raise Error('Key must be either string or unicode') diff --git a/pisi/lockeddbshelve.py b/pisi/lockeddbshelve.py index 8da1073b..41d9ddfa 100644 --- a/pisi/lockeddbshelve.py +++ b/pisi/lockeddbshelve.py @@ -12,8 +12,10 @@ # # Authors: Eray Ozkural -import bsddb3.dbshelve as shelve import bsddb3.db as db +import bsddb3.dbobj as dbobj +#import bsddb3.dbshelve as shelve +import pisi.dbshelve as shelve import os import fcntl import types @@ -23,18 +25,33 @@ __trans = gettext.translation('pisi', fallback=True) _ = __trans.ugettext import pisi -import pisi.context +import pisi.context as ctx +from pisi.util import join_path + class Error(pisi.Error): pass + +def init_dbenv(): + ctx.dbenv = dbobj.DBEnv() + ctx.dbenv.open(pisi.context.config.db_dir(), + db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_TXN | db.DB_INIT_LOG ) + +#def open(filename, flags='r', mode = 0644, filetype = db.DB_BTREE): +# db = LockedDBShelf(None, mode, filetype, None, True) +# db.open(filename, filename, filetype, flags, mode) +# return db + class LockedDBShelf(shelve.DBShelf): """A simple wrapper to implement locking for bsddb's dbshelf""" def __init__(self, dbname, mode=0644, - filetype=db.DB_BTREE, dbenv=None): + filetype=db.DB_BTREE, dbenv = None): + if dbenv == None: + dbenv = ctx.dbenv shelve.DBShelf.__init__(self, dbenv) - filename = os.path.join( pisi.context.config.db_dir(), dbname + '.bdb') + filename = join_path(pisi.context.config.db_dir(), dbname + '.bdb') if os.access(os.path.dirname(filename), os.W_OK): flags = 'w' elif os.access(filename, os.R_OK): @@ -46,11 +63,10 @@ class LockedDBShelf(shelve.DBShelf): def __del__(self): # superclass does something funky, we don't need that pass - + def open(self, filename, dbname, filetype, flags=db.DB_CREATE, mode=0644): self.filename = filename self.closed = False - #print 'open', filename if type(flags) == type(''): sflag = flags if sflag == 'r': @@ -65,24 +81,26 @@ class LockedDBShelf(shelve.DBShelf): flags = db.DB_TRUNCATE | db.DB_CREATE else: raise Error, _("Flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags") + flags |= db.DB_AUTO_COMMIT self.flags = flags - pisi.util.check_dir(pisi.context.config.db_dir()) - if self.flags != db.DB_RDONLY: + if self.flags & db.DB_RDONLY == 0: self.lock() - return self.db.open(filename, dbname, filetype, flags, mode) + filename = os.path.realpath(filename) # we give absolute path due to dbenv + #print 'opening', filename, filetype, flags, mode + return self.db.open(filename, None, filetype, flags, mode) def lock(self): self.lockfile = file(self.filename + '.lock', 'w') try: fcntl.flock(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: - raise Error(_("Another instance of PISI is running. Try later!")) + raise Error(_("Another instance of PISI is running. Only one instance is allowed to modify the PISI database at a time.")) def close(self): if self.closed: return self.db.close() - if self.flags != db.DB_RDONLY: + if self.flags & db.DB_RDONLY == 0: self.unlock() self.closed = True diff --git a/pisi/packagedb.py b/pisi/packagedb.py index 3c0434f8..27505cfb 100644 --- a/pisi/packagedb.py +++ b/pisi/packagedb.py @@ -61,7 +61,7 @@ class PackageDB(object): def list_packages(self): list = [] - for (pkg, x) in self.d.iteritems(): + for (pkg, x) in self.d.items(): list.append(pkg) return list diff --git a/tests/installdbtests.py b/tests/installdbtests.py index 7d4f9eee..ce66809f 100644 --- a/tests/installdbtests.py +++ b/tests/installdbtests.py @@ -26,6 +26,8 @@ class InstallDBTestCase(testcase.TestCase): def testInstall(self): ctx.installdb.purge('installtest') ctx.installdb.install('installtest', '0.1', '2', '3') + self.assert_(ctx.installdb.is_recorded('installtest')) + self.assert_(ctx.installdb.is_installed('installtest')) def testRemovePurge(self): ctx.installdb.install('installtest', '0.1', '2', '3') diff --git a/tools/cat-db.py b/tools/cat-db.py index 4e430d0a..16a98e76 100755 --- a/tools/cat-db.py +++ b/tools/cat-db.py @@ -13,14 +13,15 @@ import sys import bsddb3.dbshelve as shelve +import bsddb3.db as db sys.path.append('.') import pisi -d = shelve.open( sys.argv[1], flags='r' ) +d = shelve.open( sys.argv[1], 'r', 0660, filetype = db.DB_BTREE ) -for (k, data) in d.iteritems(): - print k, data +for key, data in d.items(): + print key, data d.close()