- pisi db sınıfları db paketine alındı, bazı import ve isimlendirmeler düzeltildi.

- Component ve Files db erişim sınıfları veri sınıflarından ayrıldı
- testler kısmen düzenlendi.
Uyarılar:
- rebuild-db gerekli
- Bazı testler kırılmış, düzeltilmesi gerekiyor
- Package manager GUI'si çalışmıyor

RefactorPlan

@@ -78,6 +78,9 @@ writing our own simple serialization routines for some objects like Conflicts or
 
 * context singleton is a mess. Maybe we should consider another approach to access common db objects 
 
+* ComponentDB should not hold lists of Packages that belongs to it. A new index for components to package DB 
+should be enough.
+  
 ==> class attributes
 * In some classes there are some attributes assigned but never
 used. (see remove_unused_attributes.patch)

pisi/api.py

@@ -21,16 +21,20 @@ __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
 import pisi
+import pisi.db
 import pisi.context as ctx
 from pisi.uri import URI
 import pisi.util as util
 import pisi.dependency as dependency
 import pisi.pgraph as pgraph
 import pisi.operations as operations
-import pisi.packagedb as packagedb
+import pisi.db.packagedb as packagedb
-import pisi.repodb
+import pisi.db.repodb
-import pisi.installdb
+import pisi.db.installdb
-import pisi.sourcedb
+import pisi.db.filesdb
+import pisi.db.componentdb
+import pisi.db.sourcedb
+import pisi.db.lockeddbshelve as lockeddbshelve
 import pisi.component as component
 from pisi.index import Index
 import pisi.cli
@@ -43,7 +47,7 @@ from pisi.atomicoperations import resurrect_package, build
 from pisi.metadata import MetaData
 from pisi.files import Files
 from pisi.file import File
-import pisi.lockeddbshelve as shelve
+import pisi.db.lockeddbshelve as shelve
 from pisi.version import Version
 
 class Error(pisi.Error):
@@ -106,12 +110,12 @@ def init(database = True, write = True,
     ctx.database = database
     if database:
         shelve.init_dbenv(write=write)
-        ctx.repodb = pisi.repodb.init()
+        ctx.repodb = pisi.db.repodb.init()
-        ctx.installdb = pisi.installdb.init()
+        ctx.installdb = pisi.db.installdb.init()
-        ctx.filesdb = pisi.files.FilesDB()
+        ctx.filesdb = pisi.db.filesdb.FilesDB()
-        ctx.componentdb = pisi.component.ComponentDB()
+        ctx.componentdb = pisi.db.componentdb.ComponentDB()
         ctx.packagedb = packagedb.init_db()
-        ctx.sourcedb = pisi.sourcedb.init()
+        ctx.sourcedb = pisi.db.sourcedb.init()
     else:
         ctx.repodb = None
         ctx.installdb = None
@@ -130,8 +134,8 @@ def finalize():
             ctx.loghandler.flush()
             ctx.log.removeHandler(ctx.loghandler)
 
-        pisi.repodb.finalize()
+        pisi.db.repodb.finalize()
-        pisi.installdb.finalize()
+        pisi.db.installdb.finalize()
         if ctx.filesdb != None:
             ctx.filesdb.close()
             ctx.filesdb = None
@@ -142,7 +146,7 @@ def finalize():
             packagedb.finalize_db()
             ctx.packagedb = None
         if ctx.sourcedb:
-            pisi.sourcedb.finalize()
+            pisi.db.sourcedb.finalize()
             ctx.sourcedb = None
         if ctx.dbenv:
             ctx.dbenv.close()
@@ -168,7 +172,7 @@ def list_upgradable():
 
     return filter(pisi.operations.is_upgradable, ctx.installdb.list_installed())
 
-def package_graph(A, repo = pisi.itembyrepodb.installed, ignore_installed = False):
+def package_graph(A, repo = pisi.db.itembyrepodb.installed, ignore_installed = False):
     """Construct a package relations graph.
     
     Graph will contain all dependencies of packages A, if ignore_installed
@@ -241,14 +245,14 @@ def generate_conflicts(A):
 
 def generate_pending_order(A):
     # returns pending package list in reverse topological order of dependency
-    G_f = pgraph.PGraph(ctx.packagedb, pisi.itembyrepodb.installed) # construct G_f
+    G_f = pgraph.PGraph(ctx.packagedb, pisi.db.itembyrepodb.installed) # construct G_f
     for x in A.keys():
         G_f.add_package(x)
     B = A
     while len(B) > 0:
         Bp = set()
         for x in B.keys():
-            pkg = ctx.packagedb.get_package(x, pisi.itembyrepodb.installed)
+            pkg = ctx.packagedb.get_package(x, pisi.db.itembyrepodb.installed)
             for dep in pkg.runtimeDependencies():
                 if dep.package in G_f.vertices():
                     G_f.add_dep(x, dep)
@@ -317,9 +321,9 @@ def info_file(package_fn):
 def info_name(package_name, installed=False):
     """Fetch package information for the given package."""
     if installed:
-        package = ctx.packagedb.get_package(package_name, pisi.itembyrepodb.installed)
+        package = ctx.packagedb.get_package(package_name, pisi.db.itembyrepodb.installed)
     else:
-        package, repo = ctx.packagedb.get_package_repo(package_name, pisi.itembyrepodb.repos)
+        package, repo = ctx.packagedb.get_package_repo(package_name, pisi.db.itembyrepodb.repos)
         repostr = repo
 
     from pisi.metadata import MetaData
@@ -338,7 +342,7 @@ def info_name(package_name, installed=False):
         files = None
     return metadata, files
 
-def search_package_terms(terms, repo = pisi.itembyrepodb.all):
+def search_package_terms(terms, repo = pisi.db.itembyrepodb.all):
 
     def search(package, term):
         term = unicode(term).lower()
@@ -393,7 +397,7 @@ def add_repo(name, indexuri, at = None):
     if ctx.repodb.has_repo(name):
         raise Error(_('Repo %s already present.') % name)
     else:
-        repo = pisi.repodb.Repo(URI(indexuri))
+        repo = pisi.db.repodb.Repo(URI(indexuri))
         ctx.repodb.add_repo(name, repo, at = at)
         ctx.ui.info(_('Repo %s added to system.') % name)
 
@@ -529,7 +533,7 @@ def rebuild_db(files=False):
         raise
     except Exception, e: #FIXME: what exception could we catch here, replace with that.
         files = True # exception means the files db version was wrong
-    pisi.lockeddbshelve.init_dbenv(write=True, writeversion=True)
+    lockeddbshelve.init_dbenv(write=True, writeversion=True)
     destroy(files) # bye bye
 
     # save parameters and shutdown pisi

pisi/atomicoperations.py

@@ -22,7 +22,7 @@ import shutil
 
 import pisi
 import pisi.context as ctx
-import pisi.packagedb as packagedb
+import pisi.db.packagedb as packagedb
 import pisi.dependency as dependency
 import pisi.conflict
 import pisi.util as util
@@ -158,7 +158,7 @@ class Install(AtomicOperation):
         # check if package is in database
         # If it is not, put it into 3rd party packagedb
         if not ctx.packagedb.has_package(self.pkginfo.name):
-            ctx.packagedb.add_package(self.pkginfo, pisi.itembyrepodb.thirdparty)
+            ctx.packagedb.add_package(self.pkginfo, pisi.db.itembyrepodb.thirdparty)
 
         # check file conflicts
         file_conflicts = []
@@ -188,7 +188,7 @@ class Install(AtomicOperation):
         if ctx.installdb.is_installed(pkg.name): # is this a reinstallation?
 
             #FIXME: consider REPOSITORY instead of DISTRIBUTION -- exa
-            #ipackage = ctx.packagedb.get_package(pkg.name, pisi.itembyrepodb.installed)
+            #ipackage = ctx.packagedb.get_package(pkg.name, pisi.db.itembyrepodb.installed)
             ipkg = ctx.installdb.get_info(pkg.name)
             repomismatch = ipkg.distribution != pkg.distribution
 
@@ -398,7 +398,7 @@ class Install(AtomicOperation):
         ctx.filesdb.add_files(self.metadata.package.name, self.files, txn=txn)
 
         # installed packages
-        ctx.packagedb.add_package(self.pkginfo, pisi.itembyrepodb.installed, txn=txn)
+        ctx.packagedb.add_package(self.pkginfo, pisi.db.itembyrepodb.installed, txn=txn)
 
 
 def install_single(pkg, upgrade = False):
@@ -427,7 +427,7 @@ class Remove(AtomicOperation):
     def __init__(self, package_name):
         super(Remove, self).__init__()
         self.package_name = package_name
-        self.package = ctx.packagedb.get_package(self.package_name, pisi.itembyrepodb.installed)
+        self.package = ctx.packagedb.get_package(self.package_name, pisi.db.itembyrepodb.installed)
         try:
             self.files = ctx.installdb.files(self.package_name)
         except pisi.Error, e:
@@ -525,7 +525,7 @@ class Remove(AtomicOperation):
     def remove_db(self, txn):
         ctx.installdb.remove(self.package_name, txn)
         ctx.filesdb.remove_files(self.files, txn)
-        pisi.packagedb.remove_tracking_package(self.package_name, txn)
+        pisi.db.packagedb.remove_tracking_package(self.package_name, txn)
 
 
 def remove_single(package_name):
@@ -552,7 +552,7 @@ def virtual_install(metadata, files, txn):
         ctx.filesdb.add_files(metadata.package.name, files, txn=txn)
 
     # installed packages
-    ctx.packagedb.add_package(metadata.package, pisi.itembyrepodb.installed, txn=txn)
+    ctx.packagedb.add_package(metadata.package, pisi.db.itembyrepodb.installed, txn=txn)
 
 def resurrect_package(package_fn, write_files, txn = None):
     """Resurrect the package from xml files"""

pisi/cli/commands.py

@@ -376,7 +376,7 @@ the package in graphviz format to 'pgraph.dot'.
                 repo = ctx.get_option('repository')
                 ctx.ui.info(_('Plotting packages in repository %s') % repo)
             else:
-                repo = pisi.itembyrepodb.repos
+                repo = pisi.db.itembyrepodb.repos
             if self.args:
                 a = self.args
             else:
@@ -389,7 +389,7 @@ the package in graphviz format to 'pgraph.dot'.
                 # if A is empty, then graph all packages
                 ctx.ui.info(_('Plotting a graph of relations among all installed packages'))
                 a = ctx.installdb.list_installed()
-            repo = pisi.itembyrepodb.installed
+            repo = pisi.db.itembyrepodb.installed
         g = pisi.api.package_graph(a, repo = repo,
                                    ignore_installed = ctx.get_option('ignore_installed'))
         g.write_graphviz(file(ctx.get_option('output'), 'w'))
@@ -944,7 +944,7 @@ Usage: info <package1> <package2> ... <packagen>
                     ctx.ui.info(_('[inst] '), noln=True)
                 else:
                     ctx.ui.info(_('Installed package:'))
-                self.print_pkginfo(metadata, files,pisi.itembyrepodb.installed)
+                self.print_pkginfo(metadata, files,pisi.db.itembyrepodb.installed)
 
             if ctx.packagedb.has_package(arg):
                 metadata, files = pisi.api.info_name(arg, False)
@@ -952,7 +952,7 @@ Usage: info <package1> <package2> ... <packagen>
                     ctx.ui.info(_('[repo] '), noln=True)
                 else:
                     ctx.ui.info(_('Package found in repository:'))
-                self.print_pkginfo(metadata, files, pisi.itembyrepodb.repos)
+                self.print_pkginfo(metadata, files, pisi.db.itembyrepodb.repos)
 
             if not ctx.packagedb.has_package(arg):
                 ctx.ui.info(_("%s is not found in repositories") % arg)
@@ -1124,7 +1124,7 @@ Usage: list-installed
             ctx.ui.info(_('Package Name     |St|   Version|  Rel.| Build|  Distro|             Date'))
             print         '========================================================================'
         for pkg in installed:
-            package = ctx.packagedb.get_package(pkg, pisi.itembyrepodb.installed)
+            package = ctx.packagedb.get_package(pkg, pisi.db.itembyrepodb.installed)
             inst_info = ctx.installdb.get_info(pkg)
             if self.options.long:
                 ctx.ui.info(unicode(package))
@@ -1502,7 +1502,7 @@ Lists the packages that will be upgraded.
             ctx.ui.info(_('Package Name     |St|   Version|  Rel.| Build|  Distro|             Date'))
             print         '========================================================================'
         for pkg in upgradable_pkgs:
-            package = ctx.packagedb.get_package(pkg, pisi.itembyrepodb.installed)
+            package = ctx.packagedb.get_package(pkg, pisi.db.itembyrepodb.installed)
             inst_info = ctx.installdb.get_info(pkg)
             if self.options.long:
                 ctx.ui.info(package)

pisi/component.py

@@ -15,12 +15,8 @@ __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
 import pisi
-import pisi.context as ctx
 import pisi.pxml.xmlfile as xmlfile
 import pisi.pxml.autoxml as autoxml
-import pisi.lockeddbshelve as shelve
-from pisi.itembyrepodb import ItemByRepoDB
-import pisi.itembyrepodb as itembyrepodb
 
 class Error(pisi.Error):
     pass
@@ -80,173 +76,4 @@ class Component(xmlfile.XmlFile):
     # the parts of this component.
     # to be filled by the component database, thus it is optional.
     t_Packages = [ [autoxml.String], autoxml.optional, "Parts/Package"]
-
     t_Sources = [ [autoxml.String], autoxml.optional, "Parts/Source"]
-
-class ComponentDB(object):
-    """a database of components"""
-
-    def __init__(self):
-        self.d = ItemByRepoDB('component')
-
-    def close(self):
-        self.d.close()
-
-    def destroy(self):
-        self.d.destroy()
-
-    def has_component(self, name, repo = pisi.itembyrepodb.repos, txn = None):
-        name = str(name)
-        return self.d.has_key(name, repo, txn)
-
-    def get_component(self, name, repo=None, txn = None):
-        try:
-            return self.d.get_item(name, repo, txn=txn)
-        except pisi.itembyrepodb.NotfoundError, e:
-            raise Error(_('Component %s not found') % name)
-
-    def get_component_repo(self, name, repo=None, txn = None):
-        try:
-            return self.d.get_item_repo(name, repo, txn=txn)
-        except pisi.itembyrepodb.NotfoundError, e:
-            raise Error(_('Component %s not found') % name)
-
-    def get_union_comp(self, name, txn = None, repo = pisi.itembyrepodb.repos ):
-        """get a union of all repository components packages, not just the first repo in order.
-        get only basic repo info from the first repo"""
-        def proc(txn):
-            s = self.d.d.get(name, txn=txn)
-            pkgs = set()
-            srcs = set()
-            for repostr in self.d.order(repo = repo):
-                if s.has_key(repostr):
-                    pkgs |= set(s[repostr].packages)
-                    srcs |= set(s[repostr].sources)
-            comp = self.get_component(name)
-            comp.packages = list(pkgs)
-            comp.sources = list(srcs)
-            return comp
-        return self.d.txn_proc(proc, txn)
-
-    def list_components(self, repo=None):
-        return self.d.list(repo)
-
-    # walk: walks through the underlying  components' packages
-    def get_union_packages(self, component_name, walk=False, repo=pisi.itembyrepodb.repos, txn = None):
-        """returns union of all repository component's packages, not just the first repo's 
-        component's in order"""
-        
-        component = self.get_union_comp(component_name, txn, repo)
-        if not walk:
-            return component.packages
-
-        packages = []
-        packages.extend(component.packages)
-        for dep in component.dependencies:
-            packages.extend(self.get_union_packages(dep, walk, repo, txn))
-
-        return packages
-
-    # walk: walks through the underlying  components' packages
-    def get_packages(self, component_name, walk=False, repo=None, txn = None):
-        """returns the given component's and underlying recursive components' packages"""
-        
-        component = self.get_component(component_name, repo, txn)
-        if not walk:
-            return component.packages
-
-        packages = []
-        packages.extend(component.packages)
-        for dep in component.dependencies:
-            packages.extend(self.get_packages(dep, walk, repo, txn))
-
-        return packages
-
-    def add_child(self, component, repo, txn = None):
-        """update component tree"""
-        parent_name = ".".join(component.name.split(".")[:-1])
-        if not parent_name: # root component
-            return
-
-        if self.has_component(parent_name, repo, txn):
-            parent = self.get_component(parent_name, repo, txn)
-        else:
-            parent = Component(name = parent_name)
-
-        if component.name not in parent.dependencies:
-            parent.dependencies.append(component.name)
-            self.d.add_item(parent_name, parent, repo, txn)
-
-    def update_component(self, component, repo, txn = None):
-        def proc(txn):
-            if self.has_component(component.name, repo, txn):
-                # preserve list of sources, packages and dependencies
-                current = self.d.get_item(component.name, repo, txn)
-                component.packages = current.packages
-                component.sources = current.sources
-                component.dependencies = current.dependencies
-            self.d.add_item(component.name, component, repo, txn)
-            self.add_child(component, repo, txn)
-        self.d.txn_proc(proc, txn)
-
-    def add_package(self, component_name, package, repo, txn = None):
-        def proc(txn):
-            assert component_name
-            if self.has_component(component_name, repo, txn):
-                component = self.get_component(component_name, repo, txn)
-            else:
-                component = Component( name = component_name )
-            if not package in component.packages:
-                component.packages.append(package)
-            self.d.add_item(component_name, component, repo, txn) # update
-            self.add_child(component, repo, txn)
-        self.d.txn_proc(proc, txn)
-
-    def remove_package(self, component_name, package, repo = None, txn = None):
-        def proc(txn, repo):
-            if not self.has_component(component_name, repo, txn):
-                raise Error(_('Information for component %s not available') % component_name)
-            if not repo:
-                repo = self.d.which_repo(component_name, txn=txn) # get default repo then
-            component = self.get_component(component_name, repo, txn)
-            if package in component.packages:
-                component.packages.remove(package)
-            self.d.add_item(component_name, component, repo, txn) # update
-
-        ctx.txn_proc(lambda x: proc(txn, repo), txn)
-
-    def add_spec(self, component_name, spec, repo, txn = None):
-        def proc(txn):
-            assert component_name
-            if self.has_component(component_name, repo, txn):
-                component = self.get_component(component_name, repo, txn)
-            else:
-                component = Component( name = component_name )
-            if not spec in component.sources:
-                component.sources.append(spec)
-            self.d.add_item(component_name, component, repo, txn) # update
-            self.add_child(component, repo, txn)
-        self.d.txn_proc(proc, txn)
-
-    def remove_spec(self, component_name, spec, repo = None, txn = None):
-        def proc(txn, repo):
-            if not self.has_component(component_name, repo, txn):
-                raise Error(_('Information for component %s not available') % component_name)
-            if not repo:
-                repo = self.d.which_repo(component_name, txn=txn) # get default repo then
-            component = self.get_component(component_name, repo, txn)
-            if spec in component.sources:
-                component.sources.remove(spec)
-            self.d.add_item(component_name, component, repo, txn) # update
-
-        ctx.txn_proc(lambda x: proc(txn, repo), txn)
-
-    def clear(self, txn = None):
-        self.d.clear(txn)
-
-    def remove_component(self, name, repo = None, txn = None):
-        name = str(name)
-        self.d.remove_item(name, repo, txn)
-
-    def remove_repo(self, repo, txn = None):
-        self.d.remove_repo(repo, txn=txn)

pisi/conflict.py

@@ -19,7 +19,7 @@ _ = __trans.ugettext
 import pisi.context as ctx
 from pisi.version import Version
 import pisi.pxml.autoxml as autoxml
-import pisi.itembyrepodb
+import pisi.db.itembyrepodb
 
 class Conflict:
 
@@ -75,7 +75,7 @@ given conflicting spec"""
     if not ctx.installdb.is_installed(pkg_name):
         return False
     else:
-        pkg = ctx.packagedb.get_package(pkg_name, pisi.itembyrepodb.installed)
+        pkg = ctx.packagedb.get_package(pkg_name, pisi.db.itembyrepodb.installed)
         (version, release) = (pkg.version, pkg.release)
         return confinfo.conflicts(pkg_name, version, release)
 

pisi/db/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2005 - 2007, TUBITAK/UEKAE
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# Please read the COPYING file.
+#

pisi/db/componentdb.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2005 - 2007, TUBITAK/UEKAE
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# Please read the COPYING file.
+#
+
+import types
+
+import gettext
+__trans = gettext.translation('pisi', fallback=True)
+_ = __trans.ugettext
+
+import pisi
+import pisi.context as ctx
+import pisi.db.itembyrepodb as itembyrepodb
+import pisi.component
+
+
+class ComponentDB(object):
+
+    def __init__(self):
+        self.d = itembyrepodb.ItemByRepoDB('component')
+
+    def close(self):
+        self.d.close()
+
+    def destroy(self):
+        self.d.destroy()
+
+    def has_component(self, name, repo = itembyrepodb.repos, txn = None):
+        name = str(name)
+        return self.d.has_key(name, repo, txn)
+
+    def get_component(self, name, repo=None, txn = None):
+        try:
+            return self.d.get_item(name, repo, txn=txn)
+        except itembyrepodb.NotfoundError, e:
+            raise Error(_('Component %s not found') % name)
+
+    def get_component_repo(self, name, repo=None, txn = None):
+        try:
+            return self.d.get_item_repo(name, repo, txn=txn)
+        except itembyrepodb.NotfoundError, e:
+            raise Error(_('Component %s not found') % name)
+
+    def get_union_comp(self, name, txn = None, repo = itembyrepodb.repos ):
+        """get a union of all repository components packages, not just the first repo in order.
+        get only basic repo info from the first repo"""
+        def proc(txn):
+            s = self.d.d.get(name, txn=txn)
+            pkgs = set()
+            srcs = set()
+            for repostr in self.d.order(repo = repo):
+                if s.has_key(repostr):
+                    pkgs |= set(s[repostr].packages)
+                    srcs |= set(s[repostr].sources)
+            comp = self.get_component(name)
+            comp.packages = list(pkgs)
+            comp.sources = list(srcs)
+            return comp
+        return self.d.txn_proc(proc, txn)
+
+    def list_components(self, repo=None):
+        return self.d.list(repo)
+
+    # walk: walks through the underlying  components' packages
+    def get_union_packages(self, component_name, walk=False, repo=pisi.db.itembyrepodb.repos, txn = None):
+        """returns union of all repository component's packages, not just the first repo's 
+        component's in order"""
+        
+        component = self.get_union_comp(component_name, txn, repo)
+        if not walk:
+            return component.packages
+
+        packages = []
+        packages.extend(component.packages)
+        for dep in component.dependencies:
+            packages.extend(self.get_union_packages(dep, walk, repo, txn))
+
+        return packages
+
+    # walk: walks through the underlying  components' packages
+    def get_packages(self, component_name, walk=False, repo=None, txn = None):
+        """returns the given component's and underlying recursive components' packages"""
+        
+        component = self.get_component(component_name, repo, txn)
+        if not walk:
+            return component.packages
+
+        packages = []
+        packages.extend(component.packages)
+        for dep in component.dependencies:
+            packages.extend(self.get_packages(dep, walk, repo, txn))
+
+        return packages
+
+    def add_child(self, component, repo, txn = None):
+        """update component tree"""
+        parent_name = ".".join(component.name.split(".")[:-1])
+        if not parent_name: # root component
+            return
+
+        if self.has_component(parent_name, repo, txn):
+            parent = self.get_component(parent_name, repo, txn)
+        else:
+            parent = pisi.component.Component(name = parent_name)
+
+        if component.name not in parent.dependencies:
+            parent.dependencies.append(component.name)
+            self.d.add_item(parent_name, parent, repo, txn)
+
+    def update_component(self, component, repo, txn = None):
+        def proc(txn):
+            if self.has_component(component.name, repo, txn):
+                # preserve list of sources, packages and dependencies
+                current = self.d.get_item(component.name, repo, txn)
+                component.packages = current.packages
+                component.sources = current.sources
+                component.dependencies = current.dependencies
+            self.d.add_item(component.name, component, repo, txn)
+            self.add_child(component, repo, txn)
+        self.d.txn_proc(proc, txn)
+
+    def add_package(self, component_name, package, repo, txn = None):
+        def proc(txn):
+            assert component_name
+            if self.has_component(component_name, repo, txn):
+                component = self.get_component(component_name, repo, txn)
+            else:
+                component = pisi.component.Component( name = component_name )
+            if not package in component.packages:
+                component.packages.append(package)
+            self.d.add_item(component_name, component, repo, txn) # update
+            self.add_child(component, repo, txn)
+        self.d.txn_proc(proc, txn)
+
+    def remove_package(self, component_name, package, repo = None, txn = None):
+        def proc(txn, repo):
+            if not self.has_component(component_name, repo, txn):
+                raise Error(_('Information for component %s not available') % component_name)
+            if not repo:
+                repo = self.d.which_repo(component_name, txn=txn) # get default repo then
+            component = self.get_component(component_name, repo, txn)
+            if package in component.packages:
+                component.packages.remove(package)
+            self.d.add_item(component_name, component, repo, txn) # update
+
+        ctx.txn_proc(lambda x: proc(txn, repo), txn)
+
+    def add_spec(self, component_name, spec, repo, txn = None):
+        def proc(txn):
+            assert component_name
+            if self.has_component(component_name, repo, txn):
+                component = self.get_component(component_name, repo, txn)
+            else:
+                component = pisi.component.Component( name = component_name )
+            if not spec in component.sources:
+                component.sources.append(spec)
+            self.d.add_item(component_name, component, repo, txn) # update
+            self.add_child(component, repo, txn)
+        self.d.txn_proc(proc, txn)
+
+    def remove_spec(self, component_name, spec, repo = None, txn = None):
+        def proc(txn, repo):
+            if not self.has_component(component_name, repo, txn):
+                raise Error(_('Information for component %s not available') % component_name)
+            if not repo:
+                repo = self.d.which_repo(component_name, txn=txn) # get default repo then
+            component = self.get_component(component_name, repo, txn)
+            if spec in component.sources:
+                component.sources.remove(spec)
+            self.d.add_item(component_name, component, repo, txn) # update
+
+        ctx.txn_proc(lambda x: proc(txn, repo), txn)
+
+    def clear(self, txn = None):
+        self.d.clear(txn)
+
+    def remove_component(self, name, repo = None, txn = None):
+        name = str(name)
+        self.d.remove_item(name, repo, txn)
+
+    def remove_repo(self, repo, txn = None):
+        self.d.remove_repo(repo, txn=txn)

pisi/db/dbshelve.py

@@ -38,8 +38,6 @@ import cPickle
 import bsddb3.db as db
 import bsddb3.dbobj as dbobj
 import string
-import sys
-import traceback
 import pisi
 
 class CodingError(pisi.Error):

pisi/db/filesdb.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2005 - 2007, TUBITAK/UEKAE
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# Please read the COPYING file.
+#
+
+import pisi.db.lockeddbshelve as shelve
+import pisi.files
+
+class FilesDB(shelve.LockedDBShelf):
+    '''
+        FilesDB provides db access operations for Files objects (files.Files)  
+    '''
+    def __init__(self):
+        shelve.LockedDBShelf.__init__(self, 'files')
+
+    def add_files(self, pkg_name, files, txn = None):
+        def proc(txn):
+            for x in files.list:
+                path = x.path
+                del x.path # don't store redundant attribute in db
+                self.put(path, (pkg_name, x), txn)
+                x.path = path # store it back in
+        self.txn_proc(proc, txn)
+
+    def remove_files(self, files, txn = None):
+        def proc(txn):
+            for x in files.list:
+                if self.has_key(x.path):
+                    self.delete(x.path, txn)
+        self.txn_proc(proc, txn)
+
+    def has_file(self, path, txn = None):
+        return self.has_key(str(path), txn)
+
+    def get_file(self, path, txn = None):
+        path = str(path)
+        def proc(txn):
+            if not self.has_key(path, txn):
+                return None
+            else:
+                (name, fileinfo) = self.get(path, txn)
+                fileinfo.path = path
+                return (name, fileinfo)
+        return self.txn_proc(proc, txn)
+
+    def match_files(self, glob):
+        # NB: avoid using, this reads the entire db
+        import fnmatch
+        glob = str(glob)
+        infos = []
+        for key in self.keys():
+            if fnmatch.fnmatch(key, glob):
+
+                # FIXME: Why should we assign path attribute manually
+                # in fileinfo? This is also done in get_file(), seems
+                # like a dirty workaround... - baris
+                name = self[key][0]
+                fileinfo = self[key][1]
+                fileinfo.path = key
+                infos.append((name, fileinfo))
+        return infos

pisi/db/installdb.py

@@ -24,10 +24,10 @@ _ = __trans.ugettext
 # PiSi
 import pisi
 import pisi.context as ctx
-import pisi.lockeddbshelve as shelve
+import pisi.db.lockeddbshelve as dbshelve
 from pisi.files import Files
 import pisi.util as util
-from pisi.util import join_path as join
+from pisi.util import join_path
 
 
 class InstallDBError(pisi.Error):
@@ -72,9 +72,9 @@ class InstallInfo:
 class InstallDB:
 
     def __init__(self):
-        self.d = shelve.LockedDBShelf('install')
+        self.d = dbshelve.LockedDBShelf('install')
-        self.dp = shelve.LockedDBShelf('configpending')
+        self.dp = dbshelve.LockedDBShelf('configpending')
-        self.files_dir = join(ctx.config.db_dir(), 'files')
+        self.files_dir = join_path(ctx.config.db_dir(), 'files')
 
     def close(self):
         self.d.close()
@@ -82,7 +82,7 @@ class InstallDB:
 
     def files_name(self, pkg, version, release):
         pkg_dir = self.pkg_dir(pkg, version, release)
-        return join(pkg_dir, ctx.const.files_xml)
+        return join_path(pkg_dir, ctx.const.files_xml)
 
     def files(self, pkg):
         pkg = str(pkg)
@@ -92,7 +92,7 @@ class InstallDB:
         return files
 
     def pkg_dir(self, pkg, version, release):
-        return join(ctx.config.lib_dir(), 'package',
+        return join_path(ctx.config.lib_dir(), 'package',
                     pkg + '-' + version + '-' + release)
 
     def is_recorded(self, pkg, txn = None):

pisi/db/itembyrepodb.py

@@ -21,7 +21,7 @@ _ = __trans.ugettext
 import pisi
 import pisi.util as util
 import pisi.context as ctx
-import pisi.lockeddbshelve as shelve
+import pisi.db.lockeddbshelve as shelve
 
 installed, thirdparty, repos, all = range(1, 5)
 
@@ -90,7 +90,6 @@ class ItemByRepoDB(object):
         if repo == None:
             repo = repos
         assert repo in [all, repos]
-        import pisi.repodb
         order = [ 'repo-' + x for x in ctx.repodb.list() ]
         if repo == all:
             order += ['trdparty', 'inst']

pisi/db/lockeddbshelve.py

@@ -16,7 +16,7 @@ import types
 
 import bsddb3.db as db
 import bsddb3.dbobj as dbobj
-import pisi.dbshelve as shelve
+import pisi.db.dbshelve as dbshelve
 
 import gettext
 __trans = gettext.translation('pisi', fallback=True)
@@ -24,7 +24,7 @@ _ = __trans.ugettext
 
 import pisi
 import pisi.context as ctx
-from pisi.util import join_path
+import pisi.util as util
 
 from pisi.version import Version
 
@@ -35,7 +35,7 @@ class Error(pisi.Error):
 # if write is given it knows it has write access
 # if force is given it updates the specified db version
 def check_dbversion(versionfile, ver, write=False, update=False):
-    verfn = join_path(pisi.context.config.db_dir(), versionfile)
+    verfn = util.join_path(ctx.config.db_dir(), versionfile)
     firsttime = False
     if os.path.exists(verfn):
         verfile = file(verfn, 'r')
@@ -64,7 +64,7 @@ def check_dbversion(versionfile, ver, write=False, update=False):
         raise Error(_('Database version %s not present.') % versionfile)
 
 def lock_dbenv():
-    ctx.dbenv_lock = file(join_path(pisi.context.config.db_dir(), 'dbenv.lock'), 'w')
+    ctx.dbenv_lock = file(util.join_path(pisi.context.config.db_dir(), 'dbenv.lock'), 'w')
     try:
         fcntl.flock(ctx.dbenv_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
     except IOError:
@@ -96,15 +96,15 @@ def init_dbenv(write=False, writeversion=False):
     else:
         ctx.dbenv = None # read-only access to database
 
-class LockedDBShelf(shelve.DBShelf):
+class LockedDBShelf(dbshelve.DBShelf):
     """A simple wrapper to implement locking for bsddb's dbshelf"""
 
     def __init__(self, dbname, mode=0644,
                  filetype=db.DB_BTREE, dbenv = None):
         if dbenv == None:
             dbenv = ctx.dbenv
-        shelve.DBShelf.__init__(self, dbenv)
+        dbshelve.DBShelf.__init__(self, dbenv)
-        filename = join_path(pisi.context.config.db_dir(), dbname + '.bdb')
+        filename = util.join_path(pisi.context.config.db_dir(), dbname + '.bdb')
         if dbenv and os.access(os.path.dirname(filename), os.W_OK):
             flags = 'w'
         elif os.access(filename, os.R_OK):

pisi/db/packagedb.py

@@ -28,9 +28,8 @@ _ = __trans.ugettext
 import pisi
 import pisi.util as util
 import pisi.context as ctx
-import pisi.lockeddbshelve as shelve
+import pisi.db.itembyrepodb as itembyrepodb
-from pisi.itembyrepodb import ItemByRepoDB
+
-import pisi.itembyrepodb as itembyrepodb
 
 class Error(pisi.Error):
     pass
@@ -45,26 +44,40 @@ class PackageDB(object):
     using shelf objects"""
 
     def __init__(self):
-        self.d = ItemByRepoDB('package')
+        self.d = itembyrepodb.ItemByRepoDB('package')
-        self.dr = ItemByRepoDB('revdep')
+        self.dbHistory = itembyrepodb.ItemByRepoDB('package_history')
+        self.dr = itembyrepodb.ItemByRepoDB('revdep')
 
     def close(self):
         self.d.close()
+        self.dbHistory.close()
         self.dr.close()
 
     def destroy(self):
         self.d.destroy()
+        self.dbHistory.destroy()
         self.dr.destroy()
         
+    def clear(self, txn = None):
+        self.d.clear()
+        self.dbHistory.clear()
+        self.dr.clear()
+
     def has_package(self, name, repo=None, txn = None):
         return self.d.has_key(name, repo, txn=txn)
 
     def get_package(self, name, repo=None, txn = None):
         try:
             return self.d.get_item(name, repo, txn=txn)
-        except pisi.itembyrepodb.NotfoundError, e:
+        except pisi.db.itembyrepodb.NotfoundError, e:
             raise Error(_('Package %s not found') % name)
 
+    def get_history(self, name, repo=None, txn = None):
+        try:
+            return self.dbHistory.get_item(name, repo, txn=txn)
+        except pisi.db.itembyrepodb.NotfoundError, e:
+            raise Error(_('History for Package %s not found') % name)
+
     def get_package_repo(self, name, repo=None, txn = None):
         return self.d.get_item_repo(name, repo, txn=txn)
 
@@ -91,6 +104,8 @@ class PackageDB(object):
         name = str(package_info.name)
 
         def proc(txn):
+            self.dbHistory.add_item(name, package_info.history, repo, txn)
+            package_info.history = None
             self.d.add_item(name, package_info, repo, txn)
             for dep in package_info.runtimeDependencies():
                 dep_name = str(dep.package)
@@ -103,18 +118,14 @@ class PackageDB(object):
                     self.dr.add_item(dep_name, [ (name, dep) ], repo, txn)
             # add component
             ctx.componentdb.add_package(package_info.partOf, package_info.name, repo, txn)
-
         ctx.txn_proc(proc, txn)
 
-    def clear(self, txn = None):
-        self.d.clear()
-        self.dr.clear()
-
     def remove_package(self, name, repo = None, txn = None):
         name = str(name)
         def proc(txn):
             package_info = self.d.get_item(name, repo, txn=txn)
             self.d.remove_item(name, repo, txn=txn)
+            self.dbHistory.remove_item(name, repo, txn=txn)
             for dep in package_info.runtimeDependencies():
                 dep_name = str(dep.package)
                 if self.dr.has_key(dep_name, repo, txn):

pisi/db/repodb.py

@@ -10,18 +10,14 @@
 # Please read the COPYING file.
 #
 
-import os, fcntl
-
 import gettext
 __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
 import pisi
-import pisi.lockeddbshelve as shelve
+import pisi.db.lockeddbshelve as shelve
 import pisi.context as ctx
-import pisi.packagedb as packagedb
+import pisi.db.packagedb as packagedb
-import pisi.util as util
-from pisi.uri import URI
 
 class Error(pisi.Error):
     pass

pisi/db/sourcedb.py

@@ -20,18 +20,13 @@ store a set of repositories in which the source appears.
 the actual guy to take is determined from the repo order.
 """
 
-import os
-import fcntl
-
 import gettext
 __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
-import pisi.util as util
 import pisi.context as ctx
-import pisi.lockeddbshelve as shelve
+import pisi.db.repodb
-import pisi.repodb
+import pisi.db.itembyrepodb as itembyrepodb
-from pisi.itembyrepodb import ItemByRepoDB
 
 class NotfoundError(pisi.Error):
     pass
@@ -39,8 +34,8 @@ class NotfoundError(pisi.Error):
 class SourceDB(object):
 
     def __init__(self):
-        self.d = ItemByRepoDB('source')
+        self.d = itembyrepodb.ItemByRepoDB('source')
-        self.dpkgtosrc = ItemByRepoDB('pkgtosrc')
+        self.dpkgtosrc = itembyrepodb.ItemByRepoDB('pkgtosrc')
 
     def close(self):
         self.d.close()
@@ -55,13 +50,13 @@ class SourceDB(object):
     def get_spec(self, name, repo=None, txn = None):
         try:
             return self.d.get_item(name, repo, txn)
-        except pisi.itembyrepodb.NotfoundError, e:
+        except pisi.db.itembyrepodb.NotfoundError, e:
             raise NotfoundError(_("Source package %s not found") % name)
 
     def get_spec_repo(self, name, repo=None, txn = None):
         try:
             return self.d.get_item_repo(name, repo, txn)
-        except pisi.itembyrepodb.NotfoundError, e:
+        except pisi.db.itembyrepodb.NotfoundError, e:
             raise NotfoundError(_("Source package %s not found") % name)
 
     def pkgtosrc(self, name, txn = None):

pisi/dependency.py

@@ -17,10 +17,10 @@ __trans = gettext.translation('pisi', fallback=True)
 _ = __trans.ugettext
 
 import pisi.context as ctx
-#import pisi.packagedb as packagedb
+#import pisi.db.packagedb as packagedb
 from pisi.version import Version
 import pisi.pxml.autoxml as autoxml
-import pisi.itembyrepodb
+import pisi.db.itembyrepodb
 
 class Dependency:
 
@@ -86,7 +86,7 @@ dependency spec"""
     if not ctx.installdb.is_installed(pkg_name):
         return False
     else:
-        pkg = ctx.packagedb.get_package(pkg_name, pisi.itembyrepodb.installed)
+        pkg = ctx.packagedb.get_package(pkg_name, pisi.db.itembyrepodb.installed)
         (version, release) = (pkg.version, pkg.release)
         return depinfo.satisfies(pkg_name, version, release)
 

pisi/files.py

@@ -15,8 +15,6 @@ during the build process of a package and used in installation.'''
 
 
 import pisi.pxml.autoxml as autoxml
-import pisi.lockeddbshelve as shelve
-
 
 class FileInfo:
     """File holds the information for a File node/tag in files.xml"""
@@ -48,56 +46,3 @@ class Files(autoxml.XmlFile):
 
     def append(self, fileinfo):
         self.list.append(fileinfo)
-
-
-class FilesDB(shelve.LockedDBShelf):
-
-    def __init__(self):
-        shelve.LockedDBShelf.__init__(self, 'files')
-
-    def add_files(self, pkg_name, files, txn = None):
-        def proc(txn):
-            for x in files.list:
-                path = x.path
-                del x.path # don't store redundant attribute in db
-                self.put(path, (pkg_name, x), txn)
-                x.path = path # store it back in
-        self.txn_proc(proc, txn)
-
-    def remove_files(self, files, txn = None):
-        def proc(txn):
-            for x in files.list:
-                if self.has_key(x.path):
-                    self.delete(x.path, txn)
-        self.txn_proc(proc, txn)
-
-    def has_file(self, path, txn = None):
-        return self.has_key(str(path), txn)
-
-    def get_file(self, path, txn = None):
-        path = str(path)
-        def proc(txn):
-            if not self.has_key(path, txn):
-                return None
-            else:
-                (name, fileinfo) = self.get(path, txn)
-                fileinfo.path = path
-                return (name, fileinfo)
-        return self.txn_proc(proc, txn)
-
-    def match_files(self, glob):
-        # NB: avoid using, this reads the entire db
-        import fnmatch
-        glob = str(glob)
-        infos = []
-        for key in self.keys():
-            if fnmatch.fnmatch(key, glob):
-
-                # FIXME: Why should we assign path attribute manually
-                # in fileinfo? This is also done in get_file(), seems
-                # like a dirty workaround... - baris
-                name = self[key][0]
-                fileinfo = self[key][1]
-                fileinfo.path = key
-                infos.append((name, fileinfo))
-        return infos

pisi/index.py

@@ -23,8 +23,8 @@ import pisi
 import pisi.context as ctx
 import pisi.specfile as specfile
 import pisi.metadata as metadata
-import pisi.packagedb as packagedb
+import pisi.db.packagedb as packagedb
-import pisi.sourcedb as sourcedb
+import pisi.db.sourcedb as sourcedb
 import pisi.util as util
 from pisi.package import Package
 from pisi.pxml.xmlfile import XmlFile

pisi/operations.py

@@ -24,9 +24,9 @@ import pisi.util as util
 import pisi.dependency as dependency
 import pisi.conflict
 import pisi.pgraph as pgraph
-import pisi.packagedb as packagedb
+import pisi.db.packagedb as packagedb
-import pisi.repodb
+import pisi.db.repodb
-import pisi.installdb
+import pisi.db.installdb
 from pisi.index import Index
 import pisi.cli
 import pisi.atomicoperations as atomicoperations
@@ -442,6 +442,7 @@ def upgrade_pkg_names(A = []):
             continue
 
         if security_only:
+            pkg.history = ctx.packageDB.get_history(x)
             updates = [i for i in pkg.history if Version(i.release) > Version(release)]
             if not pisi.util.any(lambda i:i.type == 'security', updates):
                 continue
@@ -637,7 +638,7 @@ def plan_remove(A):
     # try to construct a pisi graph of packages to
     # install / reinstall
 
-    G_f = pgraph.PGraph(ctx.packagedb, pisi.itembyrepodb.installed)               # construct G_f
+    G_f = pgraph.PGraph(ctx.packagedb, pisi.db.itembyrepodb.installed)               # construct G_f
 
     # find the (install closure) graph of G_f by package
     # set A using packagedb
@@ -647,12 +648,12 @@ def plan_remove(A):
     while len(B) > 0:
         Bp = set()
         for x in B:
-            pkg = ctx.packagedb.get_package(x, pisi.itembyrepodb.installed)
+            pkg = ctx.packagedb.get_package(x, pisi.db.itembyrepodb.installed)
-            rev_deps = ctx.packagedb.get_rev_deps(x, pisi.itembyrepodb.installed)
+            rev_deps = ctx.packagedb.get_rev_deps(x, pisi.db.itembyrepodb.installed)
             for (rev_dep, depinfo) in rev_deps:
                 # we don't deal with uninstalled rev deps
                 # and unsatisfied dependencies (this is important, too)
-                if ctx.packagedb.has_package(rev_dep, pisi.itembyrepodb.installed) and \
+                if ctx.packagedb.has_package(rev_dep, pisi.db.itembyrepodb.installed) and \
                    dependency.installed_satisfies_dep(depinfo):
                     if not rev_dep in G_f.vertices():
                         Bp.add(rev_dep)

pisi/pgraph.py

@@ -20,7 +20,7 @@ from graph import *
 
 class PGraph(Digraph):
 
-    def __init__(self, packagedb, repo = pisi.itembyrepodb.repos):
+    def __init__(self, packagedb, repo = pisi.db.itembyrepodb.repos):
         super(PGraph, self).__init__()
         self.packagedb = packagedb
         self.repo = repo

pisi/pisitest.py

@@ -0,0 +1,282 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2006, TUBITAK/UEKAE
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# Please read the COPYING file.
+
+import os
+import sys
+import exceptions
+
+import pisi
+import pisi.testdb
+import time
+
+
+def handle_exception(exception, value, tb):
+    print exception, value, tb
+    if exception == exceptions.KeyboardInterrupt:
+        pisi.api.finalize()
+        print("\n")
+        sys.exit()
+
+def testPickle(iter):
+    import cPickle
+    pisi.api.init(write=True)
+    
+    packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all)
+    packages = {}
+    totalPack =  len(packageNames)
+    
+    for name in packageNames:
+        packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all)
+    
+    start = time.clock()
+    for i in range(iter):
+        size = 0;
+        for pak in packages:
+            pickled = cPickle.dumps(packages[pak],1)
+            size += len(pickled)
+            #pisi.context.packagedb.add_package2(pak, pisi.db.itembyrepodb.installed)
+            #print pak.source.name, pak.installedSize
+    end = time.clock()
+    delta = (end - start)
+    print("Total Packages: %s" % totalPack)
+    print("Total Size: %s, Average Size: %s" % (size, size / totalPack))
+    print("Iterations: %s Total time: %ss. Average time :%ss." % (iter, delta , delta/iter) )
+    pisi.api.finalize()
+
+def testUnPickle(iter):
+    import cPickle
+    pisi.api.init(write=True)
+    
+    packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all)
+    packages = {}
+    pickles = {}
+    totalPack =  len(packageNames)
+    
+    for name in packageNames:
+        packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all)
+    
+    size = 0;
+    for pak in packages:
+        pickled = cPickle.dumps(packages[pak],1)
+        pickles[pak] = pickled
+            
+    start = time.clock()
+    for i in range(iter):
+        for name in pickles:
+            package = cPickle.loads(pickles[name])
+    end = time.clock()
+    
+    delta = (end - start)
+    print("Total Packages: %s" % totalPack)
+    print("Iterations: %s Total time: %ss. Average time :%ss." % (iter, delta , delta/iter) )
+    pisi.api.finalize()
+
+def testDatabaseSingleTransaction():
+    import cPickle
+    import pisi.context as ctx
+    import bsddb3.db as bsd
+    from pisi.db.itembyrepodb import ItemByRepoDB
+
+    pisi.api.init(write=True)
+    
+    packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all)
+    packages = {}
+    totalPack =  len(packageNames)
+    
+    try:
+        os.unlink("/var/db/pisi/package_perf.bdb")
+        os.unlink("/var/db/pisi/package_perf.bdb.lock")
+    except:
+        pass
+        
+    db = ItemByRepoDB('package_perf')
+    
+    for name in packageNames:
+        packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all)
+    
+    start = time.time()
+    txn = ctx.dbenv.txn_begin()
+    for pak in packages:
+        db.add_item(pak, packages[pak], pisi.db.itembyrepodb.installed, txn)
+    txn.commit()
+            
+    end = time.time()
+    
+    delta = (end - start)
+    print("Total Packages: %s" % totalPack)
+    print("Total time: %.2f" % (delta) )
+    db.close();
+    pisi.api.finalize()
+    
+def testDatabaseFullTransaction():
+    import cPickle
+    import pisi.context as ctx
+    import bsddb3.db as bsd
+    from pisi.db.itembyrepodb import ItemByRepoDB
+
+    pisi.api.init(write=True)
+    
+    packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all)
+    packages = {}
+    totalPack =  len(packageNames)
+    
+    try:
+        os.unlink("/var/db/pisi/package_perf.bdb")
+        os.unlink("/var/db/pisi/package_perf.bdb.lock")
+    except:
+        pass
+        
+    db = ItemByRepoDB('package_perf')
+    
+    for name in packageNames:
+        packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all)
+    
+    start = time.time()
+    for pak in packages:
+        txn = ctx.dbenv.txn_begin()
+        db.add_item(pak, packages[pak], pisi.db.itembyrepodb.installed, txn)
+        txn.commit()
+            
+    #ctx.txn_proc(addAllPacks, None)        
+    end = time.time()
+    
+    delta = (end - start)
+    print("Total Packages: %s" % totalPack)
+    print("Total time: %.2f" % (delta) )
+    db.close();
+    pisi.api.finalize()
+
+
+def testDatabaseRead(iter):
+    import cPickle
+    import pisi.context as ctx
+    import bsddb3.db as bsd
+    from pisi.db.itembyrepodb import ItemByRepoDB
+
+    pisi.api.init(write=True)
+    
+    packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all)
+    packages = {}
+    totalPack =  len(packageNames)
+
+    for name in packageNames:
+        packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all)
+    
+    try:
+        os.unlink("/var/db/pisi/package_perf.bdb")
+        os.unlink("/var/db/pisi/package_perf.bdb.lock")
+    except:
+        pass
+        
+    db = ItemByRepoDB('package_perf')
+    
+    count = 0;
+    txn = ctx.dbenv.txn_begin()
+    for pak in packages:
+        db.add_item(pak, packages[pak], pisi.db.itembyrepodb.installed, txn)
+        count += 1 
+    txn.commit()
+    print ( "%s packages written" % count)    
+    
+    start = time.time()
+
+    for i in range(iter):
+        for name in packageNames:
+            package = db.get_item(name, pisi.db.itembyrepodb.all, txn)
+                    
+    end = time.time()
+    
+    delta = (end - start)
+    print("Total Packages: %s" % totalPack)
+    print("%s iters, Total time: %.2fs., average Time: %.2fs." % (iter, delta, delta/iter) )
+    db.close();
+    pisi.api.finalize()
+
+def cleanDB():
+    try:
+        os.unlink("/var/db/pisi/package_perf.bdb")
+        os.unlink("/var/db/pisi/package_perf.bdb.lock")
+        os.unlink("/var/db/pisi/package_perf_info.bdb")
+        os.unlink("/var/db/pisi/package_perf_info.lock")
+        os.unlink("/var/db/pisi/package_perf_history.bdb")
+        os.unlink("/var/db/pisi/package_perf_history.bdb.lock")
+    except:
+        pass
+    
+
+def testDatabaseSingleTransaction2(iter):
+    import cPickle
+    import pisi.context as ctx
+    import bsddb3.db as bsd
+    from pisi.db.itembyrepodb import ItemByRepoDB
+
+    cleanDB()
+    pisi.api.init(write=True)
+    
+    packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all)
+    packages = {}
+    totalPack =  len(packageNames)
+
+        
+    dbPack = ItemByRepoDB('package_perf')
+    #dbInfo = ItemByRepoDB('package_perf_info')
+    dbHistory = ItemByRepoDB('package_perf_history')
+    
+    for name in packageNames:
+        packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all)
+    
+    print "Write test.."
+    start = time.time()
+    txn = ctx.dbenv.txn_begin()
+    for pak in packages:
+        packageDAO = pisi.testdb.PackageDAO(packages[pak])
+        #packageInfoDAO = pisi.testdb.PackageInfoDAO(packages[pak])
+        dbPack.add_item(pak, packageDAO, pisi.db.itembyrepodb.installed, txn)
+        #dbInfo.add_item(pak, packageInfoDAO, pisi.db.itembyrepodb.installed, txn)
+        dbHistory.add_item(pak, packages[pak].history, pisi.db.itembyrepodb.installed, txn)
+    txn.commit()
+    end = time.time()
+    
+    delta = (end - start)
+    print("Total time: %.2f" % (delta) )
+
+    txn = ctx.dbenv.txn_begin()
+    print "read test. Read only package bodies"
+    start = time.time()
+    for i in range(iter):
+        for name in packageNames:
+            packageDAO = dbPack.get_item(name, pisi.db.itembyrepodb.all, txn)
+    end = time.time()
+    txn.commit()
+    delta = (end - start)
+    print("%s iters, Total time: %.2fs., average Time: %.2fs." % (iter, delta, delta/iter) )
+
+    txn = ctx.dbenv.txn_begin()
+    print "read test. Read only histories"
+    start = time.time()
+    for i in range(iter):
+        for name in packageNames:
+            hist = dbHistory.get_item(name, pisi.db.itembyrepodb.all, txn)
+    end = time.time()
+    txn.commit()
+    delta = (end - start)
+    print("%s iters, Total time: %.2fs., average Time: %.2fs." % (iter, delta, delta/iter) )
+    
+    dbPack.close()
+    #dbInfo.close()
+    dbHistory.close()
+    pisi.api.finalize()
+
+if __name__ == "__main__":
+    sys.excepthook = handle_exception
+    #testDatabaseRead(10)
+    testDatabaseSingleTransaction2(20)

pisi/testdb.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2005 - 2007, TUBITAK/UEKAE
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# Please read the COPYING file.
+#
+
+import os
+import fcntl
+import types
+
+import gettext
+__trans = gettext.translation('pisi', fallback=True)
+_ = __trans.ugettext
+
+import pisi
+import pisi.util as util
+import pisi.context as ctx
+import pisi.db.itembyrepodb as itembyrepodb
+import pisi.metadata
+
+class TestDB:
+    def __init__(self):
+        self.db = itembyrepodb.ItemByRepoDB("packages2")
+    
+    def addPackage(self):
+        a = "Hello World"
+        self.db.add_item("hello", a, itembyrepodb.installed)
+    
+    def getPackage(self, name):
+        return self.db.get_item(name, itembyrepodb.installed)
+    
+    def addAllPackages(self):
+        pisi.api.init(write=True)
+        packages = pisi.context.installdb.list_installed()
+        for name in packages:
+            pak = pisi.context.packagedb.get_package(name, itembyrepodb.installed)
+            #pisi.context.packagedb.add_package(pak, itembyrepodb.installed)
+            #print pak.source.name, pak.installedSize
+        pisi.api.finalize()
+        
+    if __name__ == "__main__":
+        t = testdb.TestDB()
+        t.addAllPackages()
+
+class PackageInfoDAO:
+    def __init__(self, package):
+        """
+        package : pisi.metadata.Package
+        """
+        self.summary = package.summary
+        self.description = package.description
+
+ 
+class PackageDAO:
+    def __init__(self, package):
+        """
+        package : pisi.metadata.Package
+        """
+        #
+        #from specfile.Package
+        #
+        self.name = package.name
+        #Summary and description will be in another place
+        self.summary = package.summary
+        self.description = package.description
+        self.isA = package.isA
+        self.partOf = package.partOf
+        self.license = package.license
+        self.icon = package.icon
+        #from dependencies.Dependency
+        self.packageDependencies = package.packageDependencies
+        #String
+        self.componentDependencies = package.componentDependencies
+        self.files = package.files
+        self.conflicts = package.conflicts
+        self.providesComar = package.providesComar
+        self.additionalFiles = package.additionalFiles
+        #List of specfile.UPDATE. History will be in another place
+        #self.history = package.history
+        #
+        #from metadata.Package (This is plain stupid.. classes with same name, extending one another.)
+        #
+        self.build = package.build
+        self.distribution = package.distribution
+        self.distributionRelease = package.distributionRelease
+        self.architecture = package.architecture
+        self.installedSize = package.installedSize
+        self.packageSize = package.packageSize
+        self.packageHash = package.packageHash
+        self.packageURI = package.packageURI
+        #Delta (List of metadata.Delta objects)
+        #self.deltaPackages = package.deltaPackages
+        self.packageFormat = package.packageFormat
+        #Expanded from metadata.Source
+        self.sourceName = package.source.name
+        self.sourceHomepage = package.source.homepage
+        #Expanded from specfile.Packager
+        self.sourcePackagerName = package.source.packager.name
+        self.sourcePackagerEmail = package.source.packager.email

tests/filestests.py

@@ -13,7 +13,7 @@ import os
 
 import pisi.context as ctx
 import pisi.api
-import pisi.installdb
+import pisi.db.installdb
 import pisi.files as files
 from pisi import util
 

tests/installdbtests.py

@@ -13,7 +13,7 @@ import os
 
 import pisi.context as ctx
 import pisi.api
-import pisi.installdb
+import pisi.db.installdb
 from pisi import util
 
 import testcase

tests/packagedbtests.py

@@ -13,9 +13,10 @@ import os
 
 import pisi.context as ctx
 import pisi.api
-from pisi.packagedb import PackageDB
 from pisi import util
 from pisi.specfile import SpecFile
+import pisi.db.repodb
+import pisi.db.packagedb
 
 import testcase
 class PackageDBTestCase(testcase.TestCase):
@@ -28,7 +29,7 @@ class PackageDBTestCase(testcase.TestCase):
         
     def testAdd(self):
         if not ctx.repodb.has_repo('test'):
-            ctx.repodb.add_repo('test', pisi.repodb.Repo(pisi.uri.URI('fakerepo.xml')) )
+            ctx.repodb.add_repo('test', pisi.db.repodb.Repo(pisi.uri.URI('fakerepo.xml')) )
         ctx.packagedb.add_package(self.spec.packages[1], 'test')
         self.assert_(ctx.packagedb.has_package('popt-libs'))
         # close the database and remove lock

tests/packagetests.py

@@ -13,7 +13,8 @@ import os
 
 from pisi import util
 import pisi.context as ctx
-from pisi import package
+import pisi.db.packagedb
+import pisi.package
 
 import testcase
 class PackageTestCase(testcase.TestCase):

tests/repodbtests.py

@@ -13,7 +13,7 @@ import os
 
 import pisi.context as ctx
 import pisi.api
-import pisi.installdb
+import pisi.db.installdb
 from pisi import util
 
 import testcase

tests/searchtests.py

@@ -1,40 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2005 - 2007, TUBITAK/UEKAE
-#
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation; either version 2 of the License, or (at your option)
-# any later version.
-#
-# Please read the COPYING file.
-#
-
-import unittest
-import os
-
-import pisi.search
-import pisi.lockeddbshelve
-
-import testcase
-
-class SearchTestCase(testcase.TestCase):
-
-    def setUp(self):
-        testcase.TestCase.setUp(self, database = True)
-
-    def testSearch(self):
-        doc1 = "A set object is an unordered collection of immutable values."
-        doc2 = "Being an unordered collection, sets do not record element position or order of insertion."
-        doc3 = "There are currently two builtin set types, set and frozenset"
-        pisi.search.init(['test'], ['en'])
-        pisi.search.add_doc('test', 'en', 1, doc1, repo = pisi.itembyrepodb.installed)
-        pisi.search.add_doc('test', 'en', 2, doc2, repo = pisi.itembyrepodb.installed)
-        pisi.search.add_doc('test', 'en', 3, doc3, repo = pisi.itembyrepodb.installed)
-        q1 = pisi.search.query('test', 'en', ['set'], repo = pisi.itembyrepodb.all)
-        self.assertEqual(q1, set([1,3]))
-        q2 = pisi.search.query('test', 'en', ['an', 'collection'], repo = pisi.itembyrepodb.all)
-        self.assertEqual(q2, set([1,2]))
-        pisi.search.finalize()
-
-suite = unittest.makeSuite(SearchTestCase)

tests/sourcedbtests.py

@@ -13,7 +13,7 @@ import os
 
 import pisi.context as ctx
 import pisi.api
-import pisi.sourcedb
+import pisi.db.sourcedb as sourcedb
 from pisi import util
 from pisi.specfile import SpecFile
 
@@ -23,11 +23,11 @@ class SourceDBTestCase(testcase.TestCase):
     def setUp(self):
         testcase.TestCase.setUp(self)
 
-        self.sourcedb = pisi.sourcedb.init()
+        self.sourcedb = sourcedb.init()
         self.spec = SpecFile()
         self.spec.read("tests/popt/pspec.xml")
         if not ctx.repodb.has_repo('test'):
-            ctx.repodb.add_repo('test', pisi.repodb.Repo(pisi.uri.URI('fakerepo.xml')) )
+            ctx.repodb.add_repo('test', pisi.db.repodb.Repo(pisi.uri.URI('fakerepo.xml')) )
 
     def testAddRemove(self):
         self.sourcedb.add_spec(self.spec, 'test')