From d8dc2dd61ba48faf4f2e21862dab48682a01df73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mehmet=20D=2E=20Ak=C4=B1n?= Date: Thu, 8 Mar 2007 07:37:09 +0000 Subject: [PATCH] =?UTF-8?q?-=20pisi=20db=20s=C4=B1n=C4=B1flar=C4=B1=20db?= =?UTF-8?q?=20paketine=20al=C4=B1nd=C4=B1,=20baz=C4=B1=20import=20ve=20isi?= =?UTF-8?q?mlendirmeler=20d=C3=BCzeltildi.=20-=20Component=20ve=20Files=20?= =?UTF-8?q?db=20eri=C5=9Fim=20s=C4=B1n=C4=B1flar=C4=B1=20veri=20s=C4=B1n?= =?UTF-8?q?=C4=B1flar=C4=B1ndan=20ayr=C4=B1ld=C4=B1=20-=20testler=20k?= =?UTF-8?q?=C4=B1smen=20d=C3=BCzenlendi.=20Uyar=C4=B1lar:=20-=20rebuild-db?= =?UTF-8?q?=20gerekli=20-=20Baz=C4=B1=20testler=20k=C4=B1r=C4=B1lm=C4=B1?= =?UTF-8?q?=C5=9F,=20d=C3=BCzeltilmesi=20gerekiyor=20-=20Package=20manager?= =?UTF-8?q?=20GUI'si=20=C3=A7al=C4=B1=C5=9Fm=C4=B1yor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RefactorPlan | 3 + pisi/api.py | 46 +++--- pisi/atomicoperations.py | 14 +- pisi/cli/commands.py | 12 +- pisi/component.py | 173 -------------------- pisi/conflict.py | 4 +- pisi/db/__init__.py | 11 ++ pisi/db/componentdb.py | 190 +++++++++++++++++++++ pisi/{ => db}/dbshelve.py | 2 - pisi/db/filesdb.py | 68 ++++++++ pisi/{ => db}/installdb.py | 14 +- pisi/{ => db}/itembyrepodb.py | 3 +- pisi/{ => db}/lockeddbshelve.py | 14 +- pisi/{ => db}/packagedb.py | 33 ++-- pisi/{ => db}/repodb.py | 8 +- pisi/{ => db}/sourcedb.py | 17 +- pisi/dependency.py | 6 +- pisi/files.py | 57 +------ pisi/index.py | 4 +- pisi/operations.py | 15 +- pisi/pgraph.py | 2 +- pisi/pisitest.py | 282 ++++++++++++++++++++++++++++++++ pisi/testdb.py | 105 ++++++++++++ tests/filestests.py | 2 +- tests/installdbtests.py | 2 +- tests/packagedbtests.py | 5 +- tests/packagetests.py | 3 +- tests/repodbtests.py | 2 +- tests/searchtests.py | 40 ----- tests/sourcedbtests.py | 6 +- 30 files changed, 770 insertions(+), 373 deletions(-) create mode 100644 pisi/db/__init__.py create mode 100644 pisi/db/componentdb.py rename pisi/{ => db}/dbshelve.py (99%) create mode 100644 pisi/db/filesdb.py rename pisi/{ => db}/installdb.py (94%) rename pisi/{ => db}/itembyrepodb.py (99%) rename pisi/{ => db}/lockeddbshelve.py (94%) rename pisi/{ => db}/packagedb.py (86%) rename pisi/{ => db}/repodb.py (95%) rename pisi/{ => db}/sourcedb.py (89%) create mode 100644 pisi/pisitest.py create mode 100644 pisi/testdb.py delete mode 100644 tests/searchtests.py diff --git a/RefactorPlan b/RefactorPlan index b0fb3ede..2a544777 100644 --- a/RefactorPlan +++ b/RefactorPlan @@ -78,6 +78,9 @@ writing our own simple serialization routines for some objects like Conflicts or * context singleton is a mess. Maybe we should consider another approach to access common db objects +* ComponentDB should not hold lists of Packages that belongs to it. A new index for components to package DB +should be enough. + ==> class attributes * In some classes there are some attributes assigned but never used. (see remove_unused_attributes.patch) diff --git a/pisi/api.py b/pisi/api.py index 3f6ad8a2..79f44aa2 100644 --- a/pisi/api.py +++ b/pisi/api.py @@ -21,16 +21,20 @@ __trans = gettext.translation('pisi', fallback=True) _ = __trans.ugettext import pisi +import pisi.db import pisi.context as ctx from pisi.uri import URI import pisi.util as util import pisi.dependency as dependency import pisi.pgraph as pgraph import pisi.operations as operations -import pisi.packagedb as packagedb -import pisi.repodb -import pisi.installdb -import pisi.sourcedb +import pisi.db.packagedb as packagedb +import pisi.db.repodb +import pisi.db.installdb +import pisi.db.filesdb +import pisi.db.componentdb +import pisi.db.sourcedb +import pisi.db.lockeddbshelve as lockeddbshelve import pisi.component as component from pisi.index import Index import pisi.cli @@ -43,7 +47,7 @@ from pisi.atomicoperations import resurrect_package, build from pisi.metadata import MetaData from pisi.files import Files from pisi.file import File -import pisi.lockeddbshelve as shelve +import pisi.db.lockeddbshelve as shelve from pisi.version import Version class Error(pisi.Error): @@ -106,12 +110,12 @@ def init(database = True, write = True, ctx.database = database if database: shelve.init_dbenv(write=write) - ctx.repodb = pisi.repodb.init() - ctx.installdb = pisi.installdb.init() - ctx.filesdb = pisi.files.FilesDB() - ctx.componentdb = pisi.component.ComponentDB() + ctx.repodb = pisi.db.repodb.init() + ctx.installdb = pisi.db.installdb.init() + ctx.filesdb = pisi.db.filesdb.FilesDB() + ctx.componentdb = pisi.db.componentdb.ComponentDB() ctx.packagedb = packagedb.init_db() - ctx.sourcedb = pisi.sourcedb.init() + ctx.sourcedb = pisi.db.sourcedb.init() else: ctx.repodb = None ctx.installdb = None @@ -130,8 +134,8 @@ def finalize(): ctx.loghandler.flush() ctx.log.removeHandler(ctx.loghandler) - pisi.repodb.finalize() - pisi.installdb.finalize() + pisi.db.repodb.finalize() + pisi.db.installdb.finalize() if ctx.filesdb != None: ctx.filesdb.close() ctx.filesdb = None @@ -142,7 +146,7 @@ def finalize(): packagedb.finalize_db() ctx.packagedb = None if ctx.sourcedb: - pisi.sourcedb.finalize() + pisi.db.sourcedb.finalize() ctx.sourcedb = None if ctx.dbenv: ctx.dbenv.close() @@ -168,7 +172,7 @@ def list_upgradable(): return filter(pisi.operations.is_upgradable, ctx.installdb.list_installed()) -def package_graph(A, repo = pisi.itembyrepodb.installed, ignore_installed = False): +def package_graph(A, repo = pisi.db.itembyrepodb.installed, ignore_installed = False): """Construct a package relations graph. Graph will contain all dependencies of packages A, if ignore_installed @@ -241,14 +245,14 @@ def generate_conflicts(A): def generate_pending_order(A): # returns pending package list in reverse topological order of dependency - G_f = pgraph.PGraph(ctx.packagedb, pisi.itembyrepodb.installed) # construct G_f + G_f = pgraph.PGraph(ctx.packagedb, pisi.db.itembyrepodb.installed) # construct G_f for x in A.keys(): G_f.add_package(x) B = A while len(B) > 0: Bp = set() for x in B.keys(): - pkg = ctx.packagedb.get_package(x, pisi.itembyrepodb.installed) + pkg = ctx.packagedb.get_package(x, pisi.db.itembyrepodb.installed) for dep in pkg.runtimeDependencies(): if dep.package in G_f.vertices(): G_f.add_dep(x, dep) @@ -317,9 +321,9 @@ def info_file(package_fn): def info_name(package_name, installed=False): """Fetch package information for the given package.""" if installed: - package = ctx.packagedb.get_package(package_name, pisi.itembyrepodb.installed) + package = ctx.packagedb.get_package(package_name, pisi.db.itembyrepodb.installed) else: - package, repo = ctx.packagedb.get_package_repo(package_name, pisi.itembyrepodb.repos) + package, repo = ctx.packagedb.get_package_repo(package_name, pisi.db.itembyrepodb.repos) repostr = repo from pisi.metadata import MetaData @@ -338,7 +342,7 @@ def info_name(package_name, installed=False): files = None return metadata, files -def search_package_terms(terms, repo = pisi.itembyrepodb.all): +def search_package_terms(terms, repo = pisi.db.itembyrepodb.all): def search(package, term): term = unicode(term).lower() @@ -393,7 +397,7 @@ def add_repo(name, indexuri, at = None): if ctx.repodb.has_repo(name): raise Error(_('Repo %s already present.') % name) else: - repo = pisi.repodb.Repo(URI(indexuri)) + repo = pisi.db.repodb.Repo(URI(indexuri)) ctx.repodb.add_repo(name, repo, at = at) ctx.ui.info(_('Repo %s added to system.') % name) @@ -529,7 +533,7 @@ def rebuild_db(files=False): raise except Exception, e: #FIXME: what exception could we catch here, replace with that. files = True # exception means the files db version was wrong - pisi.lockeddbshelve.init_dbenv(write=True, writeversion=True) + lockeddbshelve.init_dbenv(write=True, writeversion=True) destroy(files) # bye bye # save parameters and shutdown pisi diff --git a/pisi/atomicoperations.py b/pisi/atomicoperations.py index 102e555f..f4bd0f24 100644 --- a/pisi/atomicoperations.py +++ b/pisi/atomicoperations.py @@ -22,7 +22,7 @@ import shutil import pisi import pisi.context as ctx -import pisi.packagedb as packagedb +import pisi.db.packagedb as packagedb import pisi.dependency as dependency import pisi.conflict import pisi.util as util @@ -158,7 +158,7 @@ class Install(AtomicOperation): # check if package is in database # If it is not, put it into 3rd party packagedb if not ctx.packagedb.has_package(self.pkginfo.name): - ctx.packagedb.add_package(self.pkginfo, pisi.itembyrepodb.thirdparty) + ctx.packagedb.add_package(self.pkginfo, pisi.db.itembyrepodb.thirdparty) # check file conflicts file_conflicts = [] @@ -188,7 +188,7 @@ class Install(AtomicOperation): if ctx.installdb.is_installed(pkg.name): # is this a reinstallation? #FIXME: consider REPOSITORY instead of DISTRIBUTION -- exa - #ipackage = ctx.packagedb.get_package(pkg.name, pisi.itembyrepodb.installed) + #ipackage = ctx.packagedb.get_package(pkg.name, pisi.db.itembyrepodb.installed) ipkg = ctx.installdb.get_info(pkg.name) repomismatch = ipkg.distribution != pkg.distribution @@ -398,7 +398,7 @@ class Install(AtomicOperation): ctx.filesdb.add_files(self.metadata.package.name, self.files, txn=txn) # installed packages - ctx.packagedb.add_package(self.pkginfo, pisi.itembyrepodb.installed, txn=txn) + ctx.packagedb.add_package(self.pkginfo, pisi.db.itembyrepodb.installed, txn=txn) def install_single(pkg, upgrade = False): @@ -427,7 +427,7 @@ class Remove(AtomicOperation): def __init__(self, package_name): super(Remove, self).__init__() self.package_name = package_name - self.package = ctx.packagedb.get_package(self.package_name, pisi.itembyrepodb.installed) + self.package = ctx.packagedb.get_package(self.package_name, pisi.db.itembyrepodb.installed) try: self.files = ctx.installdb.files(self.package_name) except pisi.Error, e: @@ -525,7 +525,7 @@ class Remove(AtomicOperation): def remove_db(self, txn): ctx.installdb.remove(self.package_name, txn) ctx.filesdb.remove_files(self.files, txn) - pisi.packagedb.remove_tracking_package(self.package_name, txn) + pisi.db.packagedb.remove_tracking_package(self.package_name, txn) def remove_single(package_name): @@ -552,7 +552,7 @@ def virtual_install(metadata, files, txn): ctx.filesdb.add_files(metadata.package.name, files, txn=txn) # installed packages - ctx.packagedb.add_package(metadata.package, pisi.itembyrepodb.installed, txn=txn) + ctx.packagedb.add_package(metadata.package, pisi.db.itembyrepodb.installed, txn=txn) def resurrect_package(package_fn, write_files, txn = None): """Resurrect the package from xml files""" diff --git a/pisi/cli/commands.py b/pisi/cli/commands.py index 8cf228ad..20f0850c 100644 --- a/pisi/cli/commands.py +++ b/pisi/cli/commands.py @@ -376,7 +376,7 @@ the package in graphviz format to 'pgraph.dot'. repo = ctx.get_option('repository') ctx.ui.info(_('Plotting packages in repository %s') % repo) else: - repo = pisi.itembyrepodb.repos + repo = pisi.db.itembyrepodb.repos if self.args: a = self.args else: @@ -389,7 +389,7 @@ the package in graphviz format to 'pgraph.dot'. # if A is empty, then graph all packages ctx.ui.info(_('Plotting a graph of relations among all installed packages')) a = ctx.installdb.list_installed() - repo = pisi.itembyrepodb.installed + repo = pisi.db.itembyrepodb.installed g = pisi.api.package_graph(a, repo = repo, ignore_installed = ctx.get_option('ignore_installed')) g.write_graphviz(file(ctx.get_option('output'), 'w')) @@ -944,7 +944,7 @@ Usage: info ... ctx.ui.info(_('[inst] '), noln=True) else: ctx.ui.info(_('Installed package:')) - self.print_pkginfo(metadata, files,pisi.itembyrepodb.installed) + self.print_pkginfo(metadata, files,pisi.db.itembyrepodb.installed) if ctx.packagedb.has_package(arg): metadata, files = pisi.api.info_name(arg, False) @@ -952,7 +952,7 @@ Usage: info ... ctx.ui.info(_('[repo] '), noln=True) else: ctx.ui.info(_('Package found in repository:')) - self.print_pkginfo(metadata, files, pisi.itembyrepodb.repos) + self.print_pkginfo(metadata, files, pisi.db.itembyrepodb.repos) if not ctx.packagedb.has_package(arg): ctx.ui.info(_("%s is not found in repositories") % arg) @@ -1124,7 +1124,7 @@ Usage: list-installed ctx.ui.info(_('Package Name |St| Version| Rel.| Build| Distro| Date')) print '========================================================================' for pkg in installed: - package = ctx.packagedb.get_package(pkg, pisi.itembyrepodb.installed) + package = ctx.packagedb.get_package(pkg, pisi.db.itembyrepodb.installed) inst_info = ctx.installdb.get_info(pkg) if self.options.long: ctx.ui.info(unicode(package)) @@ -1502,7 +1502,7 @@ Lists the packages that will be upgraded. ctx.ui.info(_('Package Name |St| Version| Rel.| Build| Distro| Date')) print '========================================================================' for pkg in upgradable_pkgs: - package = ctx.packagedb.get_package(pkg, pisi.itembyrepodb.installed) + package = ctx.packagedb.get_package(pkg, pisi.db.itembyrepodb.installed) inst_info = ctx.installdb.get_info(pkg) if self.options.long: ctx.ui.info(package) diff --git a/pisi/component.py b/pisi/component.py index 43223995..c3aa7b61 100644 --- a/pisi/component.py +++ b/pisi/component.py @@ -15,12 +15,8 @@ __trans = gettext.translation('pisi', fallback=True) _ = __trans.ugettext import pisi -import pisi.context as ctx import pisi.pxml.xmlfile as xmlfile import pisi.pxml.autoxml as autoxml -import pisi.lockeddbshelve as shelve -from pisi.itembyrepodb import ItemByRepoDB -import pisi.itembyrepodb as itembyrepodb class Error(pisi.Error): pass @@ -80,173 +76,4 @@ class Component(xmlfile.XmlFile): # the parts of this component. # to be filled by the component database, thus it is optional. t_Packages = [ [autoxml.String], autoxml.optional, "Parts/Package"] - t_Sources = [ [autoxml.String], autoxml.optional, "Parts/Source"] - -class ComponentDB(object): - """a database of components""" - - def __init__(self): - self.d = ItemByRepoDB('component') - - def close(self): - self.d.close() - - def destroy(self): - self.d.destroy() - - def has_component(self, name, repo = pisi.itembyrepodb.repos, txn = None): - name = str(name) - return self.d.has_key(name, repo, txn) - - def get_component(self, name, repo=None, txn = None): - try: - return self.d.get_item(name, repo, txn=txn) - except pisi.itembyrepodb.NotfoundError, e: - raise Error(_('Component %s not found') % name) - - def get_component_repo(self, name, repo=None, txn = None): - try: - return self.d.get_item_repo(name, repo, txn=txn) - except pisi.itembyrepodb.NotfoundError, e: - raise Error(_('Component %s not found') % name) - - def get_union_comp(self, name, txn = None, repo = pisi.itembyrepodb.repos ): - """get a union of all repository components packages, not just the first repo in order. - get only basic repo info from the first repo""" - def proc(txn): - s = self.d.d.get(name, txn=txn) - pkgs = set() - srcs = set() - for repostr in self.d.order(repo = repo): - if s.has_key(repostr): - pkgs |= set(s[repostr].packages) - srcs |= set(s[repostr].sources) - comp = self.get_component(name) - comp.packages = list(pkgs) - comp.sources = list(srcs) - return comp - return self.d.txn_proc(proc, txn) - - def list_components(self, repo=None): - return self.d.list(repo) - - # walk: walks through the underlying components' packages - def get_union_packages(self, component_name, walk=False, repo=pisi.itembyrepodb.repos, txn = None): - """returns union of all repository component's packages, not just the first repo's - component's in order""" - - component = self.get_union_comp(component_name, txn, repo) - if not walk: - return component.packages - - packages = [] - packages.extend(component.packages) - for dep in component.dependencies: - packages.extend(self.get_union_packages(dep, walk, repo, txn)) - - return packages - - # walk: walks through the underlying components' packages - def get_packages(self, component_name, walk=False, repo=None, txn = None): - """returns the given component's and underlying recursive components' packages""" - - component = self.get_component(component_name, repo, txn) - if not walk: - return component.packages - - packages = [] - packages.extend(component.packages) - for dep in component.dependencies: - packages.extend(self.get_packages(dep, walk, repo, txn)) - - return packages - - def add_child(self, component, repo, txn = None): - """update component tree""" - parent_name = ".".join(component.name.split(".")[:-1]) - if not parent_name: # root component - return - - if self.has_component(parent_name, repo, txn): - parent = self.get_component(parent_name, repo, txn) - else: - parent = Component(name = parent_name) - - if component.name not in parent.dependencies: - parent.dependencies.append(component.name) - self.d.add_item(parent_name, parent, repo, txn) - - def update_component(self, component, repo, txn = None): - def proc(txn): - if self.has_component(component.name, repo, txn): - # preserve list of sources, packages and dependencies - current = self.d.get_item(component.name, repo, txn) - component.packages = current.packages - component.sources = current.sources - component.dependencies = current.dependencies - self.d.add_item(component.name, component, repo, txn) - self.add_child(component, repo, txn) - self.d.txn_proc(proc, txn) - - def add_package(self, component_name, package, repo, txn = None): - def proc(txn): - assert component_name - if self.has_component(component_name, repo, txn): - component = self.get_component(component_name, repo, txn) - else: - component = Component( name = component_name ) - if not package in component.packages: - component.packages.append(package) - self.d.add_item(component_name, component, repo, txn) # update - self.add_child(component, repo, txn) - self.d.txn_proc(proc, txn) - - def remove_package(self, component_name, package, repo = None, txn = None): - def proc(txn, repo): - if not self.has_component(component_name, repo, txn): - raise Error(_('Information for component %s not available') % component_name) - if not repo: - repo = self.d.which_repo(component_name, txn=txn) # get default repo then - component = self.get_component(component_name, repo, txn) - if package in component.packages: - component.packages.remove(package) - self.d.add_item(component_name, component, repo, txn) # update - - ctx.txn_proc(lambda x: proc(txn, repo), txn) - - def add_spec(self, component_name, spec, repo, txn = None): - def proc(txn): - assert component_name - if self.has_component(component_name, repo, txn): - component = self.get_component(component_name, repo, txn) - else: - component = Component( name = component_name ) - if not spec in component.sources: - component.sources.append(spec) - self.d.add_item(component_name, component, repo, txn) # update - self.add_child(component, repo, txn) - self.d.txn_proc(proc, txn) - - def remove_spec(self, component_name, spec, repo = None, txn = None): - def proc(txn, repo): - if not self.has_component(component_name, repo, txn): - raise Error(_('Information for component %s not available') % component_name) - if not repo: - repo = self.d.which_repo(component_name, txn=txn) # get default repo then - component = self.get_component(component_name, repo, txn) - if spec in component.sources: - component.sources.remove(spec) - self.d.add_item(component_name, component, repo, txn) # update - - ctx.txn_proc(lambda x: proc(txn, repo), txn) - - def clear(self, txn = None): - self.d.clear(txn) - - def remove_component(self, name, repo = None, txn = None): - name = str(name) - self.d.remove_item(name, repo, txn) - - def remove_repo(self, repo, txn = None): - self.d.remove_repo(repo, txn=txn) diff --git a/pisi/conflict.py b/pisi/conflict.py index acf7b45f..8a1b5d60 100644 --- a/pisi/conflict.py +++ b/pisi/conflict.py @@ -19,7 +19,7 @@ _ = __trans.ugettext import pisi.context as ctx from pisi.version import Version import pisi.pxml.autoxml as autoxml -import pisi.itembyrepodb +import pisi.db.itembyrepodb class Conflict: @@ -75,7 +75,7 @@ given conflicting spec""" if not ctx.installdb.is_installed(pkg_name): return False else: - pkg = ctx.packagedb.get_package(pkg_name, pisi.itembyrepodb.installed) + pkg = ctx.packagedb.get_package(pkg_name, pisi.db.itembyrepodb.installed) (version, release) = (pkg.version, pkg.release) return confinfo.conflicts(pkg_name, version, release) diff --git a/pisi/db/__init__.py b/pisi/db/__init__.py new file mode 100644 index 00000000..31121064 --- /dev/null +++ b/pisi/db/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# diff --git a/pisi/db/componentdb.py b/pisi/db/componentdb.py new file mode 100644 index 00000000..c7b71623 --- /dev/null +++ b/pisi/db/componentdb.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +import types + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.context as ctx +import pisi.db.itembyrepodb as itembyrepodb +import pisi.component + + +class ComponentDB(object): + + def __init__(self): + self.d = itembyrepodb.ItemByRepoDB('component') + + def close(self): + self.d.close() + + def destroy(self): + self.d.destroy() + + def has_component(self, name, repo = itembyrepodb.repos, txn = None): + name = str(name) + return self.d.has_key(name, repo, txn) + + def get_component(self, name, repo=None, txn = None): + try: + return self.d.get_item(name, repo, txn=txn) + except itembyrepodb.NotfoundError, e: + raise Error(_('Component %s not found') % name) + + def get_component_repo(self, name, repo=None, txn = None): + try: + return self.d.get_item_repo(name, repo, txn=txn) + except itembyrepodb.NotfoundError, e: + raise Error(_('Component %s not found') % name) + + def get_union_comp(self, name, txn = None, repo = itembyrepodb.repos ): + """get a union of all repository components packages, not just the first repo in order. + get only basic repo info from the first repo""" + def proc(txn): + s = self.d.d.get(name, txn=txn) + pkgs = set() + srcs = set() + for repostr in self.d.order(repo = repo): + if s.has_key(repostr): + pkgs |= set(s[repostr].packages) + srcs |= set(s[repostr].sources) + comp = self.get_component(name) + comp.packages = list(pkgs) + comp.sources = list(srcs) + return comp + return self.d.txn_proc(proc, txn) + + def list_components(self, repo=None): + return self.d.list(repo) + + # walk: walks through the underlying components' packages + def get_union_packages(self, component_name, walk=False, repo=pisi.db.itembyrepodb.repos, txn = None): + """returns union of all repository component's packages, not just the first repo's + component's in order""" + + component = self.get_union_comp(component_name, txn, repo) + if not walk: + return component.packages + + packages = [] + packages.extend(component.packages) + for dep in component.dependencies: + packages.extend(self.get_union_packages(dep, walk, repo, txn)) + + return packages + + # walk: walks through the underlying components' packages + def get_packages(self, component_name, walk=False, repo=None, txn = None): + """returns the given component's and underlying recursive components' packages""" + + component = self.get_component(component_name, repo, txn) + if not walk: + return component.packages + + packages = [] + packages.extend(component.packages) + for dep in component.dependencies: + packages.extend(self.get_packages(dep, walk, repo, txn)) + + return packages + + def add_child(self, component, repo, txn = None): + """update component tree""" + parent_name = ".".join(component.name.split(".")[:-1]) + if not parent_name: # root component + return + + if self.has_component(parent_name, repo, txn): + parent = self.get_component(parent_name, repo, txn) + else: + parent = pisi.component.Component(name = parent_name) + + if component.name not in parent.dependencies: + parent.dependencies.append(component.name) + self.d.add_item(parent_name, parent, repo, txn) + + def update_component(self, component, repo, txn = None): + def proc(txn): + if self.has_component(component.name, repo, txn): + # preserve list of sources, packages and dependencies + current = self.d.get_item(component.name, repo, txn) + component.packages = current.packages + component.sources = current.sources + component.dependencies = current.dependencies + self.d.add_item(component.name, component, repo, txn) + self.add_child(component, repo, txn) + self.d.txn_proc(proc, txn) + + def add_package(self, component_name, package, repo, txn = None): + def proc(txn): + assert component_name + if self.has_component(component_name, repo, txn): + component = self.get_component(component_name, repo, txn) + else: + component = pisi.component.Component( name = component_name ) + if not package in component.packages: + component.packages.append(package) + self.d.add_item(component_name, component, repo, txn) # update + self.add_child(component, repo, txn) + self.d.txn_proc(proc, txn) + + def remove_package(self, component_name, package, repo = None, txn = None): + def proc(txn, repo): + if not self.has_component(component_name, repo, txn): + raise Error(_('Information for component %s not available') % component_name) + if not repo: + repo = self.d.which_repo(component_name, txn=txn) # get default repo then + component = self.get_component(component_name, repo, txn) + if package in component.packages: + component.packages.remove(package) + self.d.add_item(component_name, component, repo, txn) # update + + ctx.txn_proc(lambda x: proc(txn, repo), txn) + + def add_spec(self, component_name, spec, repo, txn = None): + def proc(txn): + assert component_name + if self.has_component(component_name, repo, txn): + component = self.get_component(component_name, repo, txn) + else: + component = pisi.component.Component( name = component_name ) + if not spec in component.sources: + component.sources.append(spec) + self.d.add_item(component_name, component, repo, txn) # update + self.add_child(component, repo, txn) + self.d.txn_proc(proc, txn) + + def remove_spec(self, component_name, spec, repo = None, txn = None): + def proc(txn, repo): + if not self.has_component(component_name, repo, txn): + raise Error(_('Information for component %s not available') % component_name) + if not repo: + repo = self.d.which_repo(component_name, txn=txn) # get default repo then + component = self.get_component(component_name, repo, txn) + if spec in component.sources: + component.sources.remove(spec) + self.d.add_item(component_name, component, repo, txn) # update + + ctx.txn_proc(lambda x: proc(txn, repo), txn) + + def clear(self, txn = None): + self.d.clear(txn) + + def remove_component(self, name, repo = None, txn = None): + name = str(name) + self.d.remove_item(name, repo, txn) + + def remove_repo(self, repo, txn = None): + self.d.remove_repo(repo, txn=txn) diff --git a/pisi/dbshelve.py b/pisi/db/dbshelve.py similarity index 99% rename from pisi/dbshelve.py rename to pisi/db/dbshelve.py index 03f41e33..6ddf3da4 100644 --- a/pisi/dbshelve.py +++ b/pisi/db/dbshelve.py @@ -38,8 +38,6 @@ import cPickle import bsddb3.db as db import bsddb3.dbobj as dbobj import string -import sys -import traceback import pisi class CodingError(pisi.Error): diff --git a/pisi/db/filesdb.py b/pisi/db/filesdb.py new file mode 100644 index 00000000..4480797c --- /dev/null +++ b/pisi/db/filesdb.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +import pisi.db.lockeddbshelve as shelve +import pisi.files + +class FilesDB(shelve.LockedDBShelf): + ''' + FilesDB provides db access operations for Files objects (files.Files) + ''' + def __init__(self): + shelve.LockedDBShelf.__init__(self, 'files') + + def add_files(self, pkg_name, files, txn = None): + def proc(txn): + for x in files.list: + path = x.path + del x.path # don't store redundant attribute in db + self.put(path, (pkg_name, x), txn) + x.path = path # store it back in + self.txn_proc(proc, txn) + + def remove_files(self, files, txn = None): + def proc(txn): + for x in files.list: + if self.has_key(x.path): + self.delete(x.path, txn) + self.txn_proc(proc, txn) + + def has_file(self, path, txn = None): + return self.has_key(str(path), txn) + + def get_file(self, path, txn = None): + path = str(path) + def proc(txn): + if not self.has_key(path, txn): + return None + else: + (name, fileinfo) = self.get(path, txn) + fileinfo.path = path + return (name, fileinfo) + return self.txn_proc(proc, txn) + + def match_files(self, glob): + # NB: avoid using, this reads the entire db + import fnmatch + glob = str(glob) + infos = [] + for key in self.keys(): + if fnmatch.fnmatch(key, glob): + + # FIXME: Why should we assign path attribute manually + # in fileinfo? This is also done in get_file(), seems + # like a dirty workaround... - baris + name = self[key][0] + fileinfo = self[key][1] + fileinfo.path = key + infos.append((name, fileinfo)) + return infos diff --git a/pisi/installdb.py b/pisi/db/installdb.py similarity index 94% rename from pisi/installdb.py rename to pisi/db/installdb.py index d3a97087..08468a68 100644 --- a/pisi/installdb.py +++ b/pisi/db/installdb.py @@ -24,10 +24,10 @@ _ = __trans.ugettext # PiSi import pisi import pisi.context as ctx -import pisi.lockeddbshelve as shelve +import pisi.db.lockeddbshelve as dbshelve from pisi.files import Files import pisi.util as util -from pisi.util import join_path as join +from pisi.util import join_path class InstallDBError(pisi.Error): @@ -72,9 +72,9 @@ class InstallInfo: class InstallDB: def __init__(self): - self.d = shelve.LockedDBShelf('install') - self.dp = shelve.LockedDBShelf('configpending') - self.files_dir = join(ctx.config.db_dir(), 'files') + self.d = dbshelve.LockedDBShelf('install') + self.dp = dbshelve.LockedDBShelf('configpending') + self.files_dir = join_path(ctx.config.db_dir(), 'files') def close(self): self.d.close() @@ -82,7 +82,7 @@ class InstallDB: def files_name(self, pkg, version, release): pkg_dir = self.pkg_dir(pkg, version, release) - return join(pkg_dir, ctx.const.files_xml) + return join_path(pkg_dir, ctx.const.files_xml) def files(self, pkg): pkg = str(pkg) @@ -92,7 +92,7 @@ class InstallDB: return files def pkg_dir(self, pkg, version, release): - return join(ctx.config.lib_dir(), 'package', + return join_path(ctx.config.lib_dir(), 'package', pkg + '-' + version + '-' + release) def is_recorded(self, pkg, txn = None): diff --git a/pisi/itembyrepodb.py b/pisi/db/itembyrepodb.py similarity index 99% rename from pisi/itembyrepodb.py rename to pisi/db/itembyrepodb.py index f010a15b..84d12195 100644 --- a/pisi/itembyrepodb.py +++ b/pisi/db/itembyrepodb.py @@ -21,7 +21,7 @@ _ = __trans.ugettext import pisi import pisi.util as util import pisi.context as ctx -import pisi.lockeddbshelve as shelve +import pisi.db.lockeddbshelve as shelve installed, thirdparty, repos, all = range(1, 5) @@ -90,7 +90,6 @@ class ItemByRepoDB(object): if repo == None: repo = repos assert repo in [all, repos] - import pisi.repodb order = [ 'repo-' + x for x in ctx.repodb.list() ] if repo == all: order += ['trdparty', 'inst'] diff --git a/pisi/lockeddbshelve.py b/pisi/db/lockeddbshelve.py similarity index 94% rename from pisi/lockeddbshelve.py rename to pisi/db/lockeddbshelve.py index 64da9c33..2b3c0e9e 100644 --- a/pisi/lockeddbshelve.py +++ b/pisi/db/lockeddbshelve.py @@ -16,7 +16,7 @@ import types import bsddb3.db as db import bsddb3.dbobj as dbobj -import pisi.dbshelve as shelve +import pisi.db.dbshelve as dbshelve import gettext __trans = gettext.translation('pisi', fallback=True) @@ -24,7 +24,7 @@ _ = __trans.ugettext import pisi import pisi.context as ctx -from pisi.util import join_path +import pisi.util as util from pisi.version import Version @@ -35,7 +35,7 @@ class Error(pisi.Error): # if write is given it knows it has write access # if force is given it updates the specified db version def check_dbversion(versionfile, ver, write=False, update=False): - verfn = join_path(pisi.context.config.db_dir(), versionfile) + verfn = util.join_path(ctx.config.db_dir(), versionfile) firsttime = False if os.path.exists(verfn): verfile = file(verfn, 'r') @@ -64,7 +64,7 @@ def check_dbversion(versionfile, ver, write=False, update=False): raise Error(_('Database version %s not present.') % versionfile) def lock_dbenv(): - ctx.dbenv_lock = file(join_path(pisi.context.config.db_dir(), 'dbenv.lock'), 'w') + ctx.dbenv_lock = file(util.join_path(pisi.context.config.db_dir(), 'dbenv.lock'), 'w') try: fcntl.flock(ctx.dbenv_lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: @@ -96,15 +96,15 @@ def init_dbenv(write=False, writeversion=False): else: ctx.dbenv = None # read-only access to database -class LockedDBShelf(shelve.DBShelf): +class LockedDBShelf(dbshelve.DBShelf): """A simple wrapper to implement locking for bsddb's dbshelf""" def __init__(self, dbname, mode=0644, filetype=db.DB_BTREE, dbenv = None): if dbenv == None: dbenv = ctx.dbenv - shelve.DBShelf.__init__(self, dbenv) - filename = join_path(pisi.context.config.db_dir(), dbname + '.bdb') + dbshelve.DBShelf.__init__(self, dbenv) + filename = util.join_path(pisi.context.config.db_dir(), dbname + '.bdb') if dbenv and os.access(os.path.dirname(filename), os.W_OK): flags = 'w' elif os.access(filename, os.R_OK): diff --git a/pisi/packagedb.py b/pisi/db/packagedb.py similarity index 86% rename from pisi/packagedb.py rename to pisi/db/packagedb.py index abc569fd..a90ee0e1 100644 --- a/pisi/packagedb.py +++ b/pisi/db/packagedb.py @@ -28,9 +28,8 @@ _ = __trans.ugettext import pisi import pisi.util as util import pisi.context as ctx -import pisi.lockeddbshelve as shelve -from pisi.itembyrepodb import ItemByRepoDB -import pisi.itembyrepodb as itembyrepodb +import pisi.db.itembyrepodb as itembyrepodb + class Error(pisi.Error): pass @@ -45,16 +44,24 @@ class PackageDB(object): using shelf objects""" def __init__(self): - self.d = ItemByRepoDB('package') - self.dr = ItemByRepoDB('revdep') + self.d = itembyrepodb.ItemByRepoDB('package') + self.dbHistory = itembyrepodb.ItemByRepoDB('package_history') + self.dr = itembyrepodb.ItemByRepoDB('revdep') def close(self): self.d.close() + self.dbHistory.close() self.dr.close() def destroy(self): self.d.destroy() + self.dbHistory.destroy() self.dr.destroy() + + def clear(self, txn = None): + self.d.clear() + self.dbHistory.clear() + self.dr.clear() def has_package(self, name, repo=None, txn = None): return self.d.has_key(name, repo, txn=txn) @@ -62,9 +69,15 @@ class PackageDB(object): def get_package(self, name, repo=None, txn = None): try: return self.d.get_item(name, repo, txn=txn) - except pisi.itembyrepodb.NotfoundError, e: + except pisi.db.itembyrepodb.NotfoundError, e: raise Error(_('Package %s not found') % name) + def get_history(self, name, repo=None, txn = None): + try: + return self.dbHistory.get_item(name, repo, txn=txn) + except pisi.db.itembyrepodb.NotfoundError, e: + raise Error(_('History for Package %s not found') % name) + def get_package_repo(self, name, repo=None, txn = None): return self.d.get_item_repo(name, repo, txn=txn) @@ -91,6 +104,8 @@ class PackageDB(object): name = str(package_info.name) def proc(txn): + self.dbHistory.add_item(name, package_info.history, repo, txn) + package_info.history = None self.d.add_item(name, package_info, repo, txn) for dep in package_info.runtimeDependencies(): dep_name = str(dep.package) @@ -103,18 +118,14 @@ class PackageDB(object): self.dr.add_item(dep_name, [ (name, dep) ], repo, txn) # add component ctx.componentdb.add_package(package_info.partOf, package_info.name, repo, txn) - ctx.txn_proc(proc, txn) - def clear(self, txn = None): - self.d.clear() - self.dr.clear() - def remove_package(self, name, repo = None, txn = None): name = str(name) def proc(txn): package_info = self.d.get_item(name, repo, txn=txn) self.d.remove_item(name, repo, txn=txn) + self.dbHistory.remove_item(name, repo, txn=txn) for dep in package_info.runtimeDependencies(): dep_name = str(dep.package) if self.dr.has_key(dep_name, repo, txn): diff --git a/pisi/repodb.py b/pisi/db/repodb.py similarity index 95% rename from pisi/repodb.py rename to pisi/db/repodb.py index 3b787f4c..dbea2ab9 100644 --- a/pisi/repodb.py +++ b/pisi/db/repodb.py @@ -10,18 +10,14 @@ # Please read the COPYING file. # -import os, fcntl - import gettext __trans = gettext.translation('pisi', fallback=True) _ = __trans.ugettext import pisi -import pisi.lockeddbshelve as shelve +import pisi.db.lockeddbshelve as shelve import pisi.context as ctx -import pisi.packagedb as packagedb -import pisi.util as util -from pisi.uri import URI +import pisi.db.packagedb as packagedb class Error(pisi.Error): pass diff --git a/pisi/sourcedb.py b/pisi/db/sourcedb.py similarity index 89% rename from pisi/sourcedb.py rename to pisi/db/sourcedb.py index a3f5f4f3..e692772b 100644 --- a/pisi/sourcedb.py +++ b/pisi/db/sourcedb.py @@ -20,18 +20,13 @@ store a set of repositories in which the source appears. the actual guy to take is determined from the repo order. """ -import os -import fcntl - import gettext __trans = gettext.translation('pisi', fallback=True) _ = __trans.ugettext -import pisi.util as util import pisi.context as ctx -import pisi.lockeddbshelve as shelve -import pisi.repodb -from pisi.itembyrepodb import ItemByRepoDB +import pisi.db.repodb +import pisi.db.itembyrepodb as itembyrepodb class NotfoundError(pisi.Error): pass @@ -39,8 +34,8 @@ class NotfoundError(pisi.Error): class SourceDB(object): def __init__(self): - self.d = ItemByRepoDB('source') - self.dpkgtosrc = ItemByRepoDB('pkgtosrc') + self.d = itembyrepodb.ItemByRepoDB('source') + self.dpkgtosrc = itembyrepodb.ItemByRepoDB('pkgtosrc') def close(self): self.d.close() @@ -55,13 +50,13 @@ class SourceDB(object): def get_spec(self, name, repo=None, txn = None): try: return self.d.get_item(name, repo, txn) - except pisi.itembyrepodb.NotfoundError, e: + except pisi.db.itembyrepodb.NotfoundError, e: raise NotfoundError(_("Source package %s not found") % name) def get_spec_repo(self, name, repo=None, txn = None): try: return self.d.get_item_repo(name, repo, txn) - except pisi.itembyrepodb.NotfoundError, e: + except pisi.db.itembyrepodb.NotfoundError, e: raise NotfoundError(_("Source package %s not found") % name) def pkgtosrc(self, name, txn = None): diff --git a/pisi/dependency.py b/pisi/dependency.py index 99c79db5..88b31080 100644 --- a/pisi/dependency.py +++ b/pisi/dependency.py @@ -17,10 +17,10 @@ __trans = gettext.translation('pisi', fallback=True) _ = __trans.ugettext import pisi.context as ctx -#import pisi.packagedb as packagedb +#import pisi.db.packagedb as packagedb from pisi.version import Version import pisi.pxml.autoxml as autoxml -import pisi.itembyrepodb +import pisi.db.itembyrepodb class Dependency: @@ -86,7 +86,7 @@ dependency spec""" if not ctx.installdb.is_installed(pkg_name): return False else: - pkg = ctx.packagedb.get_package(pkg_name, pisi.itembyrepodb.installed) + pkg = ctx.packagedb.get_package(pkg_name, pisi.db.itembyrepodb.installed) (version, release) = (pkg.version, pkg.release) return depinfo.satisfies(pkg_name, version, release) diff --git a/pisi/files.py b/pisi/files.py index ecdc6c5d..eb42080a 100644 --- a/pisi/files.py +++ b/pisi/files.py @@ -15,8 +15,6 @@ during the build process of a package and used in installation.''' import pisi.pxml.autoxml as autoxml -import pisi.lockeddbshelve as shelve - class FileInfo: """File holds the information for a File node/tag in files.xml""" @@ -47,57 +45,4 @@ class Files(autoxml.XmlFile): t_List = [ [FileInfo], autoxml.optional, "File"] def append(self, fileinfo): - self.list.append(fileinfo) - - -class FilesDB(shelve.LockedDBShelf): - - def __init__(self): - shelve.LockedDBShelf.__init__(self, 'files') - - def add_files(self, pkg_name, files, txn = None): - def proc(txn): - for x in files.list: - path = x.path - del x.path # don't store redundant attribute in db - self.put(path, (pkg_name, x), txn) - x.path = path # store it back in - self.txn_proc(proc, txn) - - def remove_files(self, files, txn = None): - def proc(txn): - for x in files.list: - if self.has_key(x.path): - self.delete(x.path, txn) - self.txn_proc(proc, txn) - - def has_file(self, path, txn = None): - return self.has_key(str(path), txn) - - def get_file(self, path, txn = None): - path = str(path) - def proc(txn): - if not self.has_key(path, txn): - return None - else: - (name, fileinfo) = self.get(path, txn) - fileinfo.path = path - return (name, fileinfo) - return self.txn_proc(proc, txn) - - def match_files(self, glob): - # NB: avoid using, this reads the entire db - import fnmatch - glob = str(glob) - infos = [] - for key in self.keys(): - if fnmatch.fnmatch(key, glob): - - # FIXME: Why should we assign path attribute manually - # in fileinfo? This is also done in get_file(), seems - # like a dirty workaround... - baris - name = self[key][0] - fileinfo = self[key][1] - fileinfo.path = key - infos.append((name, fileinfo)) - return infos + self.list.append(fileinfo) \ No newline at end of file diff --git a/pisi/index.py b/pisi/index.py index 8043be7f..eae7b919 100644 --- a/pisi/index.py +++ b/pisi/index.py @@ -23,8 +23,8 @@ import pisi import pisi.context as ctx import pisi.specfile as specfile import pisi.metadata as metadata -import pisi.packagedb as packagedb -import pisi.sourcedb as sourcedb +import pisi.db.packagedb as packagedb +import pisi.db.sourcedb as sourcedb import pisi.util as util from pisi.package import Package from pisi.pxml.xmlfile import XmlFile diff --git a/pisi/operations.py b/pisi/operations.py index a48b95c3..05ea3d3f 100644 --- a/pisi/operations.py +++ b/pisi/operations.py @@ -24,9 +24,9 @@ import pisi.util as util import pisi.dependency as dependency import pisi.conflict import pisi.pgraph as pgraph -import pisi.packagedb as packagedb -import pisi.repodb -import pisi.installdb +import pisi.db.packagedb as packagedb +import pisi.db.repodb +import pisi.db.installdb from pisi.index import Index import pisi.cli import pisi.atomicoperations as atomicoperations @@ -442,6 +442,7 @@ def upgrade_pkg_names(A = []): continue if security_only: + pkg.history = ctx.packageDB.get_history(x) updates = [i for i in pkg.history if Version(i.release) > Version(release)] if not pisi.util.any(lambda i:i.type == 'security', updates): continue @@ -637,7 +638,7 @@ def plan_remove(A): # try to construct a pisi graph of packages to # install / reinstall - G_f = pgraph.PGraph(ctx.packagedb, pisi.itembyrepodb.installed) # construct G_f + G_f = pgraph.PGraph(ctx.packagedb, pisi.db.itembyrepodb.installed) # construct G_f # find the (install closure) graph of G_f by package # set A using packagedb @@ -647,12 +648,12 @@ def plan_remove(A): while len(B) > 0: Bp = set() for x in B: - pkg = ctx.packagedb.get_package(x, pisi.itembyrepodb.installed) - rev_deps = ctx.packagedb.get_rev_deps(x, pisi.itembyrepodb.installed) + pkg = ctx.packagedb.get_package(x, pisi.db.itembyrepodb.installed) + rev_deps = ctx.packagedb.get_rev_deps(x, pisi.db.itembyrepodb.installed) for (rev_dep, depinfo) in rev_deps: # we don't deal with uninstalled rev deps # and unsatisfied dependencies (this is important, too) - if ctx.packagedb.has_package(rev_dep, pisi.itembyrepodb.installed) and \ + if ctx.packagedb.has_package(rev_dep, pisi.db.itembyrepodb.installed) and \ dependency.installed_satisfies_dep(depinfo): if not rev_dep in G_f.vertices(): Bp.add(rev_dep) diff --git a/pisi/pgraph.py b/pisi/pgraph.py index ca1c8413..af4003fe 100644 --- a/pisi/pgraph.py +++ b/pisi/pgraph.py @@ -20,7 +20,7 @@ from graph import * class PGraph(Digraph): - def __init__(self, packagedb, repo = pisi.itembyrepodb.repos): + def __init__(self, packagedb, repo = pisi.db.itembyrepodb.repos): super(PGraph, self).__init__() self.packagedb = packagedb self.repo = repo diff --git a/pisi/pisitest.py b/pisi/pisitest.py new file mode 100644 index 00000000..9a35ca6a --- /dev/null +++ b/pisi/pisitest.py @@ -0,0 +1,282 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2006, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. + +import os +import sys +import exceptions + +import pisi +import pisi.testdb +import time + + +def handle_exception(exception, value, tb): + print exception, value, tb + if exception == exceptions.KeyboardInterrupt: + pisi.api.finalize() + print("\n") + sys.exit() + +def testPickle(iter): + import cPickle + pisi.api.init(write=True) + + packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all) + packages = {} + totalPack = len(packageNames) + + for name in packageNames: + packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all) + + start = time.clock() + for i in range(iter): + size = 0; + for pak in packages: + pickled = cPickle.dumps(packages[pak],1) + size += len(pickled) + #pisi.context.packagedb.add_package2(pak, pisi.db.itembyrepodb.installed) + #print pak.source.name, pak.installedSize + end = time.clock() + delta = (end - start) + print("Total Packages: %s" % totalPack) + print("Total Size: %s, Average Size: %s" % (size, size / totalPack)) + print("Iterations: %s Total time: %ss. Average time :%ss." % (iter, delta , delta/iter) ) + pisi.api.finalize() + +def testUnPickle(iter): + import cPickle + pisi.api.init(write=True) + + packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all) + packages = {} + pickles = {} + totalPack = len(packageNames) + + for name in packageNames: + packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all) + + size = 0; + for pak in packages: + pickled = cPickle.dumps(packages[pak],1) + pickles[pak] = pickled + + start = time.clock() + for i in range(iter): + for name in pickles: + package = cPickle.loads(pickles[name]) + end = time.clock() + + delta = (end - start) + print("Total Packages: %s" % totalPack) + print("Iterations: %s Total time: %ss. Average time :%ss." % (iter, delta , delta/iter) ) + pisi.api.finalize() + +def testDatabaseSingleTransaction(): + import cPickle + import pisi.context as ctx + import bsddb3.db as bsd + from pisi.db.itembyrepodb import ItemByRepoDB + + pisi.api.init(write=True) + + packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all) + packages = {} + totalPack = len(packageNames) + + try: + os.unlink("/var/db/pisi/package_perf.bdb") + os.unlink("/var/db/pisi/package_perf.bdb.lock") + except: + pass + + db = ItemByRepoDB('package_perf') + + for name in packageNames: + packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all) + + start = time.time() + txn = ctx.dbenv.txn_begin() + for pak in packages: + db.add_item(pak, packages[pak], pisi.db.itembyrepodb.installed, txn) + txn.commit() + + end = time.time() + + delta = (end - start) + print("Total Packages: %s" % totalPack) + print("Total time: %.2f" % (delta) ) + db.close(); + pisi.api.finalize() + +def testDatabaseFullTransaction(): + import cPickle + import pisi.context as ctx + import bsddb3.db as bsd + from pisi.db.itembyrepodb import ItemByRepoDB + + pisi.api.init(write=True) + + packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all) + packages = {} + totalPack = len(packageNames) + + try: + os.unlink("/var/db/pisi/package_perf.bdb") + os.unlink("/var/db/pisi/package_perf.bdb.lock") + except: + pass + + db = ItemByRepoDB('package_perf') + + for name in packageNames: + packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all) + + start = time.time() + for pak in packages: + txn = ctx.dbenv.txn_begin() + db.add_item(pak, packages[pak], pisi.db.itembyrepodb.installed, txn) + txn.commit() + + #ctx.txn_proc(addAllPacks, None) + end = time.time() + + delta = (end - start) + print("Total Packages: %s" % totalPack) + print("Total time: %.2f" % (delta) ) + db.close(); + pisi.api.finalize() + + +def testDatabaseRead(iter): + import cPickle + import pisi.context as ctx + import bsddb3.db as bsd + from pisi.db.itembyrepodb import ItemByRepoDB + + pisi.api.init(write=True) + + packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all) + packages = {} + totalPack = len(packageNames) + + for name in packageNames: + packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all) + + try: + os.unlink("/var/db/pisi/package_perf.bdb") + os.unlink("/var/db/pisi/package_perf.bdb.lock") + except: + pass + + db = ItemByRepoDB('package_perf') + + count = 0; + txn = ctx.dbenv.txn_begin() + for pak in packages: + db.add_item(pak, packages[pak], pisi.db.itembyrepodb.installed, txn) + count += 1 + txn.commit() + print ( "%s packages written" % count) + + start = time.time() + + for i in range(iter): + for name in packageNames: + package = db.get_item(name, pisi.db.itembyrepodb.all, txn) + + end = time.time() + + delta = (end - start) + print("Total Packages: %s" % totalPack) + print("%s iters, Total time: %.2fs., average Time: %.2fs." % (iter, delta, delta/iter) ) + db.close(); + pisi.api.finalize() + +def cleanDB(): + try: + os.unlink("/var/db/pisi/package_perf.bdb") + os.unlink("/var/db/pisi/package_perf.bdb.lock") + os.unlink("/var/db/pisi/package_perf_info.bdb") + os.unlink("/var/db/pisi/package_perf_info.lock") + os.unlink("/var/db/pisi/package_perf_history.bdb") + os.unlink("/var/db/pisi/package_perf_history.bdb.lock") + except: + pass + + +def testDatabaseSingleTransaction2(iter): + import cPickle + import pisi.context as ctx + import bsddb3.db as bsd + from pisi.db.itembyrepodb import ItemByRepoDB + + cleanDB() + pisi.api.init(write=True) + + packageNames = pisi.context.packagedb.list_packages(pisi.db.itembyrepodb.all) + packages = {} + totalPack = len(packageNames) + + + dbPack = ItemByRepoDB('package_perf') + #dbInfo = ItemByRepoDB('package_perf_info') + dbHistory = ItemByRepoDB('package_perf_history') + + for name in packageNames: + packages[name] = pisi.context.packagedb.get_package(name, pisi.db.itembyrepodb.all) + + print "Write test.." + start = time.time() + txn = ctx.dbenv.txn_begin() + for pak in packages: + packageDAO = pisi.testdb.PackageDAO(packages[pak]) + #packageInfoDAO = pisi.testdb.PackageInfoDAO(packages[pak]) + dbPack.add_item(pak, packageDAO, pisi.db.itembyrepodb.installed, txn) + #dbInfo.add_item(pak, packageInfoDAO, pisi.db.itembyrepodb.installed, txn) + dbHistory.add_item(pak, packages[pak].history, pisi.db.itembyrepodb.installed, txn) + txn.commit() + end = time.time() + + delta = (end - start) + print("Total time: %.2f" % (delta) ) + + txn = ctx.dbenv.txn_begin() + print "read test. Read only package bodies" + start = time.time() + for i in range(iter): + for name in packageNames: + packageDAO = dbPack.get_item(name, pisi.db.itembyrepodb.all, txn) + end = time.time() + txn.commit() + delta = (end - start) + print("%s iters, Total time: %.2fs., average Time: %.2fs." % (iter, delta, delta/iter) ) + + txn = ctx.dbenv.txn_begin() + print "read test. Read only histories" + start = time.time() + for i in range(iter): + for name in packageNames: + hist = dbHistory.get_item(name, pisi.db.itembyrepodb.all, txn) + end = time.time() + txn.commit() + delta = (end - start) + print("%s iters, Total time: %.2fs., average Time: %.2fs." % (iter, delta, delta/iter) ) + + dbPack.close() + #dbInfo.close() + dbHistory.close() + pisi.api.finalize() + +if __name__ == "__main__": + sys.excepthook = handle_exception + #testDatabaseRead(10) + testDatabaseSingleTransaction2(20) \ No newline at end of file diff --git a/pisi/testdb.py b/pisi/testdb.py new file mode 100644 index 00000000..5a7eb44c --- /dev/null +++ b/pisi/testdb.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2005 - 2007, TUBITAK/UEKAE +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# Please read the COPYING file. +# + +import os +import fcntl +import types + +import gettext +__trans = gettext.translation('pisi', fallback=True) +_ = __trans.ugettext + +import pisi +import pisi.util as util +import pisi.context as ctx +import pisi.db.itembyrepodb as itembyrepodb +import pisi.metadata + +class TestDB: + def __init__(self): + self.db = itembyrepodb.ItemByRepoDB("packages2") + + def addPackage(self): + a = "Hello World" + self.db.add_item("hello", a, itembyrepodb.installed) + + def getPackage(self, name): + return self.db.get_item(name, itembyrepodb.installed) + + def addAllPackages(self): + pisi.api.init(write=True) + packages = pisi.context.installdb.list_installed() + for name in packages: + pak = pisi.context.packagedb.get_package(name, itembyrepodb.installed) + #pisi.context.packagedb.add_package(pak, itembyrepodb.installed) + #print pak.source.name, pak.installedSize + pisi.api.finalize() + + if __name__ == "__main__": + t = testdb.TestDB() + t.addAllPackages() + +class PackageInfoDAO: + def __init__(self, package): + """ + package : pisi.metadata.Package + """ + self.summary = package.summary + self.description = package.description + + +class PackageDAO: + def __init__(self, package): + """ + package : pisi.metadata.Package + """ + # + #from specfile.Package + # + self.name = package.name + #Summary and description will be in another place + self.summary = package.summary + self.description = package.description + self.isA = package.isA + self.partOf = package.partOf + self.license = package.license + self.icon = package.icon + #from dependencies.Dependency + self.packageDependencies = package.packageDependencies + #String + self.componentDependencies = package.componentDependencies + self.files = package.files + self.conflicts = package.conflicts + self.providesComar = package.providesComar + self.additionalFiles = package.additionalFiles + #List of specfile.UPDATE. History will be in another place + #self.history = package.history + # + #from metadata.Package (This is plain stupid.. classes with same name, extending one another.) + # + self.build = package.build + self.distribution = package.distribution + self.distributionRelease = package.distributionRelease + self.architecture = package.architecture + self.installedSize = package.installedSize + self.packageSize = package.packageSize + self.packageHash = package.packageHash + self.packageURI = package.packageURI + #Delta (List of metadata.Delta objects) + #self.deltaPackages = package.deltaPackages + self.packageFormat = package.packageFormat + #Expanded from metadata.Source + self.sourceName = package.source.name + self.sourceHomepage = package.source.homepage + #Expanded from specfile.Packager + self.sourcePackagerName = package.source.packager.name + self.sourcePackagerEmail = package.source.packager.email diff --git a/tests/filestests.py b/tests/filestests.py index a784d330..34f32421 100644 --- a/tests/filestests.py +++ b/tests/filestests.py @@ -13,7 +13,7 @@ import os import pisi.context as ctx import pisi.api -import pisi.installdb +import pisi.db.installdb import pisi.files as files from pisi import util diff --git a/tests/installdbtests.py b/tests/installdbtests.py index 082ff1e4..a2e09a4f 100644 --- a/tests/installdbtests.py +++ b/tests/installdbtests.py @@ -13,7 +13,7 @@ import os import pisi.context as ctx import pisi.api -import pisi.installdb +import pisi.db.installdb from pisi import util import testcase diff --git a/tests/packagedbtests.py b/tests/packagedbtests.py index 4c315b7a..5cab01b3 100644 --- a/tests/packagedbtests.py +++ b/tests/packagedbtests.py @@ -13,9 +13,10 @@ import os import pisi.context as ctx import pisi.api -from pisi.packagedb import PackageDB from pisi import util from pisi.specfile import SpecFile +import pisi.db.repodb +import pisi.db.packagedb import testcase class PackageDBTestCase(testcase.TestCase): @@ -28,7 +29,7 @@ class PackageDBTestCase(testcase.TestCase): def testAdd(self): if not ctx.repodb.has_repo('test'): - ctx.repodb.add_repo('test', pisi.repodb.Repo(pisi.uri.URI('fakerepo.xml')) ) + ctx.repodb.add_repo('test', pisi.db.repodb.Repo(pisi.uri.URI('fakerepo.xml')) ) ctx.packagedb.add_package(self.spec.packages[1], 'test') self.assert_(ctx.packagedb.has_package('popt-libs')) # close the database and remove lock diff --git a/tests/packagetests.py b/tests/packagetests.py index 93d3ecf0..4e77ed06 100644 --- a/tests/packagetests.py +++ b/tests/packagetests.py @@ -13,7 +13,8 @@ import os from pisi import util import pisi.context as ctx -from pisi import package +import pisi.db.packagedb +import pisi.package import testcase class PackageTestCase(testcase.TestCase): diff --git a/tests/repodbtests.py b/tests/repodbtests.py index 463f7b3d..4bcdd687 100644 --- a/tests/repodbtests.py +++ b/tests/repodbtests.py @@ -13,7 +13,7 @@ import os import pisi.context as ctx import pisi.api -import pisi.installdb +import pisi.db.installdb from pisi import util import testcase diff --git a/tests/searchtests.py b/tests/searchtests.py deleted file mode 100644 index a44af47d..00000000 --- a/tests/searchtests.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2005 - 2007, TUBITAK/UEKAE -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. -# -# Please read the COPYING file. -# - -import unittest -import os - -import pisi.search -import pisi.lockeddbshelve - -import testcase - -class SearchTestCase(testcase.TestCase): - - def setUp(self): - testcase.TestCase.setUp(self, database = True) - - def testSearch(self): - doc1 = "A set object is an unordered collection of immutable values." - doc2 = "Being an unordered collection, sets do not record element position or order of insertion." - doc3 = "There are currently two builtin set types, set and frozenset" - pisi.search.init(['test'], ['en']) - pisi.search.add_doc('test', 'en', 1, doc1, repo = pisi.itembyrepodb.installed) - pisi.search.add_doc('test', 'en', 2, doc2, repo = pisi.itembyrepodb.installed) - pisi.search.add_doc('test', 'en', 3, doc3, repo = pisi.itembyrepodb.installed) - q1 = pisi.search.query('test', 'en', ['set'], repo = pisi.itembyrepodb.all) - self.assertEqual(q1, set([1,3])) - q2 = pisi.search.query('test', 'en', ['an', 'collection'], repo = pisi.itembyrepodb.all) - self.assertEqual(q2, set([1,2])) - pisi.search.finalize() - -suite = unittest.makeSuite(SearchTestCase) diff --git a/tests/sourcedbtests.py b/tests/sourcedbtests.py index 3f273ad0..b0e723d2 100644 --- a/tests/sourcedbtests.py +++ b/tests/sourcedbtests.py @@ -13,7 +13,7 @@ import os import pisi.context as ctx import pisi.api -import pisi.sourcedb +import pisi.db.sourcedb as sourcedb from pisi import util from pisi.specfile import SpecFile @@ -23,11 +23,11 @@ class SourceDBTestCase(testcase.TestCase): def setUp(self): testcase.TestCase.setUp(self) - self.sourcedb = pisi.sourcedb.init() + self.sourcedb = sourcedb.init() self.spec = SpecFile() self.spec.read("tests/popt/pspec.xml") if not ctx.repodb.has_repo('test'): - ctx.repodb.add_repo('test', pisi.repodb.Repo(pisi.uri.URI('fakerepo.xml')) ) + ctx.repodb.add_repo('test', pisi.db.repodb.Repo(pisi.uri.URI('fakerepo.xml')) ) def testAddRemove(self): self.sourcedb.add_spec(self.spec, 'test')