* maintain a new version string for filesdb only, it is the bottleneck

* optimize rebuild-db so that it skips rebuilding filesdb unless necessary

pisi/__init__.py

@@ -15,6 +15,7 @@
 __version__ = "1.0.5"
 
 __dbversion__ = "1.0.5"
+__filesdbversion__ = "1.0.5"         # yes, this is the real bottleneck
 
 __all__ = [ 'api', 'config', 'packagedb', 'installdb', 'search' ]
 

pisi/api.py

@@ -17,13 +17,13 @@
 import os
 import sys
 from os.path import exists
+import bsddb3.db as db
 
 import gettext
 __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
 import pisi
-
 import pisi.context as ctx
 from pisi.uri import URI
 import pisi.util as util
@@ -321,29 +321,36 @@ def delete_cache():
     util.clean_dir(ctx.config.archives_dir())
     util.clean_dir(ctx.config.tmp_dir())
 
+def rebuild_db(files=False):
 
-    
+    def destroy(files):
-def rebuild_db():
-
-    def destroy():
-        #FIXME: how good is deleting *all* databases?
         for db in os.listdir(ctx.config.db_dir()):
+            if db.startswith('files') or db.startswith('filesdbversion'):
+                clean = files
+            else:
+                clean = True
+            if clean:
                 os.unlink(pisi.util.join_path(ctx.config.db_dir(), db))
 
-    def reload():
+    def reload(files, txn):
-        import os
         for package_fn in os.listdir( pisi.util.join_path( ctx.config.lib_dir(),
                                                            'package' ) ):
             if not package_fn == "scripts":
                 ctx.ui.debug('Resurrecting %s' % package_fn)
-                pisi.api.resurrect_package(package_fn)
+                pisi.api.resurrect_package(package_fn, files, txn)
 
     # save parameters and shutdown pisi
     options = ctx.config.options
     ui = ctx.ui
     comar = ctx.comar
     finalize()
-    destroy() # bye bye
+
+    try:
+        pisi.lockeddbshelve.check_dbversion('filesdbversion', pisi.__filesdbversion__, write=False)
+    except:
+        files = True # exception means the files db version was wrong
+    destroy(files) # bye bye
     # construct new database version
     init(database=True, options=options, ui=ui, comar=comar)
-    reload()
+    #ctx.txn_proc(reload)
+    reload(files, None)

pisi/atomicoperations.py

@@ -484,13 +484,14 @@ def virtual_install(metadata, files, txn):
                           txn)
 
     # filesdb
+    if files:
         ctx.filesdb.add_files(metadata.package.name, files, txn)
 
     # installed packages
     packagedb.inst_packagedb.add_package(pkginfo, txn)
 
-def resurrect_package(package_fn):
+def resurrect_package(package_fn, write_files, txn = None):
-    """Resurrect the package in the PiSi databases"""
+    """Resurrect the package from xml files"""
 
     from os.path import exists
 
@@ -519,6 +520,7 @@ def resurrect_package(package_fn):
     if not passed:
         ctx.ui.info(_('* Adding \'%s\' to db... ') % (metadata.package.name), noln=True)
     
+    if write_files:
         files_xml = util.join_path(ctx.config.lib_dir(), 'package',
                                    package_fn, ctx.const.files_xml)
         if not exists(files_xml):
@@ -526,13 +528,15 @@ def resurrect_package(package_fn):
     
         files = Files()
         files.read(files_xml)
-
         if files.errors():
             raise Error, _("Invalid %s") % ctx.const.files_xml
+    else:
+        files = None
+
+    #import pisi.atomicoperations
+    def f(t):
+        pisi.atomicoperations.virtual_install(metadata, files, t)
+    ctx.txn_proc(f, txn)
 
-    import pisi.atomicoperations
-    def f(txn):
-        pisi.atomicoperations.virtual_install(metadata, files, txn)
-    ctx.txn_proc(f)
     if not passed:
         ctx.ui.info(_('OK.'))

pisi/lockeddbshelve.py

@@ -33,20 +33,20 @@ from pisi.version import Version
 class Error(pisi.Error):
     pass
 
-def init_dbenv():
+def check_dbversion(versionfile, ver, write=True):
-    if os.access(pisi.context.config.db_dir(), os.R_OK):
+    verfn = join_path(pisi.context.config.db_dir(), versionfile)
-        # try to read version
-        verfn = join_path(pisi.context.config.db_dir(), 'dbversion')
     if os.path.exists(verfn):
         verfile = file(verfn, 'r')
         ls = verfile.readlines()
         currver = Version(ls[0])
-            dbver = Version(pisi.__dbversion__)
+        dbver = Version(ver)
         if currver < dbver:
-                raise Error(_('Database version insufficient. Please run rebuild-db command.'))
+            raise Error(_('Database version for %s insufficient. Please run rebuild-db command.') % versionfile)
         elif currver > dbver:
-                raise Error(_('Database version greater than PiSi version. You need a newer PiSi.'))
+            raise Error(_('Database version for %s greater than PiSi version. You need a newer PiSi.') % versionfile)
         else:
+            return True
+    elif write:
         if os.access(pisi.context.config.db_dir(), os.W_OK):
             ctx.ui.warning(_('Writing current database version'))
             verfile = file(verfn, 'w')
@@ -54,6 +54,12 @@ def init_dbenv():
             verfile.close()
         else:
             raise Error(_('Cannot attain write access to database environment'))
+    
+def init_dbenv():
+    if os.access(pisi.context.config.db_dir(), os.R_OK):
+        # try to read version
+        check_dbversion('dbversion', pisi.__dbversion__)
+        check_dbversion('filesdbversion', pisi.__filesdbversion__)
     else:
         raise Error(_('Cannot attain read access to database environment'))
     if os.access(pisi.context.config.db_dir(), os.W_OK):