* maintain a new version string for filesdb only, it is the bottleneck

* optimize rebuild-db so that it skips rebuilding filesdb unless necessary

pisi/__init__.py

@@ -15,6 +15,7 @@
 __version__ = "1.0.5"
 
 __dbversion__ = "1.0.5"
+__filesdbversion__ = "1.0.5"         # yes, this is the real bottleneck
 
 __all__ = [ 'api', 'config', 'packagedb', 'installdb', 'search' ]
 

pisi/api.py

@@ -17,13 +17,13 @@
 import os
 import sys
 from os.path import exists
+import bsddb3.db as db
 
 import gettext
 __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
 import pisi
-
 import pisi.context as ctx
 from pisi.uri import URI
 import pisi.util as util
@@ -321,29 +321,36 @@ def delete_cache():
     util.clean_dir(ctx.config.archives_dir())
     util.clean_dir(ctx.config.tmp_dir())
 
-    
+def rebuild_db(files=False):
-    
-def rebuild_db():
 
-    def destroy():
+    def destroy(files):
-        #FIXME: how good is deleting *all* databases?
         for db in os.listdir(ctx.config.db_dir()):
-            os.unlink(pisi.util.join_path(ctx.config.db_dir(), db))
+            if db.startswith('files') or db.startswith('filesdbversion'):
+                clean = files
+            else:
+                clean = True
+            if clean:
+                os.unlink(pisi.util.join_path(ctx.config.db_dir(), db))
 
-    def reload():
+    def reload(files, txn):
-        import os
         for package_fn in os.listdir( pisi.util.join_path( ctx.config.lib_dir(),
                                                            'package' ) ):
             if not package_fn == "scripts":
                 ctx.ui.debug('Resurrecting %s' % package_fn)
-                pisi.api.resurrect_package(package_fn)
+                pisi.api.resurrect_package(package_fn, files, txn)
 
     # save parameters and shutdown pisi
     options = ctx.config.options
     ui = ctx.ui
     comar = ctx.comar
     finalize()
-    destroy() # bye bye
+
+    try:
+        pisi.lockeddbshelve.check_dbversion('filesdbversion', pisi.__filesdbversion__, write=False)
+    except:
+        files = True # exception means the files db version was wrong
+    destroy(files) # bye bye
     # construct new database version
     init(database=True, options=options, ui=ui, comar=comar)
-    reload()
+    #ctx.txn_proc(reload)
+    reload(files, None)

pisi/atomicoperations.py

@@ -484,13 +484,14 @@ def virtual_install(metadata, files, txn):
                           txn)
 
     # filesdb
-    ctx.filesdb.add_files(metadata.package.name, files, txn)
+    if files:
-    
+        ctx.filesdb.add_files(metadata.package.name, files, txn)
+
     # installed packages
     packagedb.inst_packagedb.add_package(pkginfo, txn)
 
-def resurrect_package(package_fn):
+def resurrect_package(package_fn, write_files, txn = None):
-    """Resurrect the package in the PiSi databases"""
+    """Resurrect the package from xml files"""
 
     from os.path import exists
 
@@ -519,20 +520,23 @@ def resurrect_package(package_fn):
     if not passed:
         ctx.ui.info(_('* Adding \'%s\' to db... ') % (metadata.package.name), noln=True)
     
-    files_xml = util.join_path(ctx.config.lib_dir(), 'package',
+    if write_files:
-                               package_fn, ctx.const.files_xml)
+        files_xml = util.join_path(ctx.config.lib_dir(), 'package',
-    if not exists(files_xml):
+                                   package_fn, ctx.const.files_xml)
-       raise Error, _("Files XML '%s' cannot be found") % files_xml
+        if not exists(files_xml):
+           raise Error, _("Files XML '%s' cannot be found") % files_xml
+    
+        files = Files()
+        files.read(files_xml)
+        if files.errors():
+            raise Error, _("Invalid %s") % ctx.const.files_xml
+    else:
+        files = None
 
-    files = Files()
+    #import pisi.atomicoperations
-    files.read(files_xml)
+    def f(t):
+        pisi.atomicoperations.virtual_install(metadata, files, t)
+    ctx.txn_proc(f, txn)
 
-    if files.errors():
-       raise Error, _("Invalid %s") % ctx.const.files_xml
-
-    import pisi.atomicoperations
-    def f(txn):
-        pisi.atomicoperations.virtual_install(metadata, files, txn)
-    ctx.txn_proc(f)
     if not passed:
         ctx.ui.info(_('OK.'))

pisi/lockeddbshelve.py

@@ -33,27 +33,33 @@ from pisi.version import Version
 class Error(pisi.Error):
     pass
 
+def check_dbversion(versionfile, ver, write=True):
+    verfn = join_path(pisi.context.config.db_dir(), versionfile)
+    if os.path.exists(verfn):
+        verfile = file(verfn, 'r')
+        ls = verfile.readlines()
+        currver = Version(ls[0])
+        dbver = Version(ver)
+        if currver < dbver:
+            raise Error(_('Database version for %s insufficient. Please run rebuild-db command.') % versionfile)
+        elif currver > dbver:
+            raise Error(_('Database version for %s greater than PiSi version. You need a newer PiSi.') % versionfile)
+        else:
+            return True
+    elif write:
+        if os.access(pisi.context.config.db_dir(), os.W_OK):
+            ctx.ui.warning(_('Writing current database version'))
+            verfile = file(verfn, 'w')
+            verfile.write(pisi.__dbversion__)
+            verfile.close()
+        else:
+            raise Error(_('Cannot attain write access to database environment'))
+    
 def init_dbenv():
     if os.access(pisi.context.config.db_dir(), os.R_OK):
         # try to read version
-        verfn = join_path(pisi.context.config.db_dir(), 'dbversion')
+        check_dbversion('dbversion', pisi.__dbversion__)
-        if os.path.exists(verfn):
+        check_dbversion('filesdbversion', pisi.__filesdbversion__)
-            verfile = file(verfn, 'r')
-            ls = verfile.readlines()
-            currver = Version(ls[0])
-            dbver = Version(pisi.__dbversion__)
-            if currver < dbver:
-                raise Error(_('Database version insufficient. Please run rebuild-db command.'))
-            elif currver > dbver:
-                raise Error(_('Database version greater than PiSi version. You need a newer PiSi.'))
-        else:
-            if os.access(pisi.context.config.db_dir(), os.W_OK):
-                ctx.ui.warning(_('Writing current database version'))
-                verfile = file(verfn, 'w')
-                verfile.write(pisi.__dbversion__)
-                verfile.close()
-            else:
-                raise Error(_('Cannot attain write access to database environment'))
     else:
         raise Error(_('Cannot attain read access to database environment'))
     if os.access(pisi.context.config.db_dir(), os.W_OK):