From 9038e0bd6eafb41c4def4a47c651e69f243cd33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eray=20=C3=96zkural?= Date: Wed, 24 May 2006 12:57:46 +0000 Subject: [PATCH] * fix: make inverted index stuff aware of multiple repositories. ouch. --- pisi/__init__.py | 4 +-- pisi/api.py | 12 ++++---- pisi/packagedb.py | 22 +++++++++------ pisi/search/__init__.py | 16 +++++------ pisi/search/invertedindex.py | 53 ++++++++++++++++++++++-------------- 5 files changed, 62 insertions(+), 45 deletions(-) diff --git a/pisi/__init__.py b/pisi/__init__.py index 8eeacf31..8810a176 100644 --- a/pisi/__init__.py +++ b/pisi/__init__.py @@ -12,9 +12,9 @@ # PiSi version -__version__ = "1.1_beta7" +__version__ = "1.1_beta8" -__dbversion__ = "1.1_beta7" +__dbversion__ = "1.1_beta8" __filesdbversion__ = "1.0.5" # yes, this is the real bottleneck __all__ = [ 'api', 'config', 'packagedb', 'installdb', 'search' ] diff --git a/pisi/api.py b/pisi/api.py index 4fb3cf67..0635a2b1 100644 --- a/pisi/api.py +++ b/pisi/api.py @@ -288,22 +288,22 @@ def search_package_names(query): r.add(pkgname) return r -def search_package_terms(terms, lang = None, search_names = True): +def search_package_terms(terms, lang = None, search_names = True, repo = pisi.itembyrepodb.all): if not lang: lang = pisi.pxml.autoxml.LocalText.get_lang() - r1 = pisi.search.query_terms('summary', lang, terms) - r2 = pisi.search.query_terms('description', lang, terms) + r1 = pisi.search.query_terms('summary', lang, terms, repo = repo) + r2 = pisi.search.query_terms('description', lang, terms, repo = repo) r = r1.union(r2) if search_names: for term in terms: r |= search_package_names(term) return r -def search_package(query, lang = None, search_names = True): +def search_package(query, lang = None, search_names = True, repo = pisi.itembyrepodb.all): if not lang: lang = pisi.pxml.autoxml.LocalText.get_lang() - r1 = pisi.search.query('summary', lang, query) - r2 = pisi.search.query('description', lang, query) + r1 = pisi.search.query('sumdesc', lang, query, repo = repo) + r2 = pisi.search.query('description', lang, query, repo = repo) r = r1.union(r2) if search_names: r |= search_package_names(query) diff --git a/pisi/packagedb.py b/pisi/packagedb.py index 8f1533ae..93a5466c 100644 --- a/pisi/packagedb.py +++ b/pisi/packagedb.py @@ -9,15 +9,15 @@ # # Please read the COPYING file. # - -# package database -# interface for update/query to local package repository - # Authors: Eray Ozkural # Baris Metin -# we basically store everything in PackageInfo class -# yes, we are cheap +"""package database +interface for update/query to local package repository + +we basically store everything in PackageInfo class +yes, we are cheap +""" import os import fcntl @@ -108,10 +108,10 @@ class PackageDB(object): # index summary and description for (lang, doc) in package_info.summary.iteritems(): if lang in ['en', 'tr']: - pisi.search.add_doc('summary', lang, package_info.name, doc, txn) + pisi.search.add_doc('summary', lang, package_info.name, doc, repo=repo, txn=txn) for (lang, doc) in package_info.description.iteritems(): if lang in ['en', 'tr']: - pisi.search.add_doc('description', lang, package_info.name, doc, txn) + pisi.search.add_doc('description', lang, package_info.name, doc, repo=repo, txn=txn) ctx.txn_proc(proc, txn) @@ -133,6 +133,12 @@ class PackageDB(object): if self.dr.has_key(name, repo, txn=txn): self.dr.remove_item(name, repo, txn=txn) ctx.componentdb.remove_package(package_info.partOf, package_info.name, repo, txn) + for (lang, doc) in package_info.summary.iteritems(): + if lang in ['en', 'tr']: + pisi.search.remove_doc('summary', lang, package_info.name, doc, repo=repo, txn=txn) + for (lang, doc) in package_info.description.iteritems(): + if lang in ['en', 'tr']: + pisi.search.remove_doc('description', lang, package_info.name, doc, repo=repo, txn=txn) self.d.txn_proc(proc, txn) def remove_repo(self, repo, txn = None): diff --git a/pisi/search/__init__.py b/pisi/search/__init__.py index 4a8cc6bb..03554c48 100644 --- a/pisi/search/__init__.py +++ b/pisi/search/__init__.py @@ -47,17 +47,17 @@ def finalize(): ctx.invidx[id][lang].close() ctx.invidx = {} -def add_doc(id, lang, docid, str, txn = None): +def add_doc(id, lang, docid, str, repo = None, txn = None): terms = p.preprocess(lang, str) - ctx.invidx[id][lang].add_doc(docid, terms, txn) + ctx.invidx[id][lang].add_doc(docid, terms, repo=repo, txn=txn) -def remove_doc(id, lang, docid, str, txn = None): - ctx.invidx[id][lang].remove_doc(docid, txn) +def remove_doc(id, lang, docid, str, repo = None, txn = None): + ctx.invidx[id][lang].remove_doc(docid, repo = repo, txn = txn) -def query_terms(id, lang, terms, txn = None): +def query_terms(id, lang, terms, repo = None, txn = None): terms = map(lambda x: p.lower(lang, x), terms) - return ctx.invidx[id][lang].query(terms, txn) + return ctx.invidx[id][lang].query(terms, repo = repo, txn = txn) -def query(id, lang, str, txn = None): +def query(id, lang, str, repo = None, txn = None): terms = p.preprocess(lang, str) - return query_terms(id, lang, terms, txn) + return query_terms(id, lang, terms, repo = repo, txn = txn) diff --git a/pisi/search/invertedindex.py b/pisi/search/invertedindex.py index 57d30173..5f7795a6 100644 --- a/pisi/search/invertedindex.py +++ b/pisi/search/invertedindex.py @@ -14,59 +14,70 @@ import types import pisi.lockeddbshelve as shelve +from pisi.itembyrepodb import ItemByRepoDB +import pisi.itembyrepodb as itembyrepodb class InvertedIndex(object): """a database of term -> set of documents""" def __init__(self, id, lang): - self.d = shelve.LockedDBShelf('ii-%s-%s' % (id, lang)) + self.d = ItemByRepoDB('ii-%s-%s' % (id, lang)) def close(self): self.d.close() - def has_term(self, term, txn = None): - return self.d.has_key(shelve.LockedDBShelf.encodekey(term), txn) + def has_term(self, term, repo = None, txn = None): + return self.d.has_key(shelve.LockedDBShelf.encodekey(term), repo=repo,txn=txn) - def get_term(self, term, txn = None): + def get_term(self, term, repo = None, txn = None): """get set of doc ids given term""" term = shelve.LockedDBShelf.encodekey(term) def proc(txn): - if not self.has_term(term, txn): + if not self.has_term(term, repo=repo, txn=txn): return set() - return self.d.get(term, txn) + return self.d.get_item(term, repo=repo, txn=txn) return self.d.txn_proc(proc, txn) - def query(self, terms, txn = None): + def get_union_term(self, name, txn = None, repo = itembyrepodb.repos ): + """get a union of all repository components packages, not just the first repo in order. + get only basic repo info from the first repo""" + name = shelve.LockedDBShelf.encodekey(name) def proc(txn): - docs = [ self.get_term(x, txn) for x in terms ] + terms= set() + if self.d.d.has_key(name): + s = self.d.d.get(name, txn=txn) + for repostr in self.d.order(repo = repo): + if s.has_key(repostr): + terms |= s[repostr] + return terms + return self.d.txn_proc(proc, txn) + + def query(self, terms, repo = None, txn = None): + def proc(txn): + docs = [ self.get_union_term(x, repo=repo, txn=txn) for x in terms ] if docs: return reduce(lambda x,y: x.intersection(y), docs) else: return set() return self.d.txn_proc(proc, txn) - def list_terms(self, txn= None): - list = [] - def f(txn): - for term in self.d.iterkeys(txn): - list.append(term) - return list - return self.d.txn_proc(f, txn) + def list_terms(self, repo = None, txn= None): + return self.d.list(f, repo=repo, txn=txn) - def add_doc(self, doc, terms, txn = None): + def add_doc(self, doc, terms, repo = None, txn = None): def f(txn): for term_i in terms: term_i = shelve.LockedDBShelf.encodekey(term_i) - term_i_docs = self.get_term(term_i, txn) + term_i_docs = self.get_term(term_i, repo=repo, txn=txn) term_i_docs.add(doc) - self.d.put(term_i, term_i_docs, txn) # update + self.d.add_item(term_i, term_i_docs, repo=repo, txn=txn) # update return self.d.txn_proc(f, txn) - def remove_doc(self, doc, terms): + def remove_doc(self, doc, terms,repo=None, txn=None): def f(txn): for term_i in terms: term_i = shelve.LockedDBShelf.encodekey(term_i) - term_i_docs = self.get_term(term_i) + term_i_docs = self.get_term(term_i,repo=repo, txn=txn) term_i_docs.remove(doc) - self.d.put(term_i, term_i_docs, txn) # update + self.d.add_item(term_i, term_i_docs, repo=repo, txn=txn) # update return self.d.txn_proc(f, txn)