LCOV - lcov.info - ept/debtags/maint/debtagsindexer.cc

LTP GCOV extension - code coverage report
Current view: | directory - ept/debtags/maint - debtagsindexer.cc
Test: | lcov.info
Date: | 2008-08-14 |  | Instrumented lines: | 119
Code covered: | 74.8 % |  | Executed lines: | 89

           1                 : #include <ept/debtags/maint/debtagsindexer.h>
           2                 : #include <ept/debtags/maint/path.h>
           3                 : #include <ept/debtags/maint/pkgid.h>
           4                 : #include <ept/debtags/maint/serializer.h>
           5                 : #include <ept/debtags/vocabulary.h>
           6                 :
           7                 : #include <tagcoll/coll/intdiskindex.h>
           8                 : #include <tagcoll/coll/simple.h>
           9                 : #include <tagcoll/TextFormat.h>
          10                 : #include <tagcoll/stream/filters.h>
          11                 :
          12                 : #include <wibble/exception.h>
          13                 :
          14                 : #include <cstring>
          15                 :
          16                 : using namespace std;
          17                 :
          18                 : namespace ept {
          19                 : namespace debtags {
          20                 :
          21                 : /// MMapIndexer that indexes the package names
          22                 : struct PkgIdGenerator : public tagcoll::diskindex::MMapIndexer
          23               2 : {
          24                 :         // Sorted set of all available package names
          25                 :         std::set<std::string> pkgs;
          26                 :
          27               3 :         int encodedSize() const
          28                 :         {
          29               3 :                 int size = pkgs.size() * sizeof(int);
          30           63438 :                 for (std::set<std::string>::const_iterator i = pkgs.begin();
          31                 :                                 i != pkgs.end(); ++i)
          32           63435 :                         size += i->size() + 1;
          33               3 :                 return tagcoll::diskindex::MMap::align(size);
          34                 :         }
          35                 :
          36               2 :         void encode(char* buf) const
          37                 :         {
          38               2 :                 int pos = pkgs.size() * sizeof(int);
          39               2 :                 int idx = 0;
          40           42292 :                 for (std::set<std::string>::const_iterator i = pkgs.begin();
          41                 :                                 i != pkgs.end(); ++i)
          42                 :                 {
          43           42290 :                         ((int*)buf)[idx++] = pos;
          44           42290 :                         memcpy(buf + pos, i->c_str(), i->size() + 1);
          45           42290 :                         pos += i->size() + 1;
          46                 :                 }
          47               2 :         }
          48                 : };
          49                 :
          50                 :
          51              10 : DebtagsIndexer::DebtagsIndexer(Vocabulary& voc)
          52                 :         : voc(voc),
          53                 :           mainSource(Path::debtagsSourceDir()),
          54              10 :           userSource(Path::debtagsUserSourceDir())
          55                 : {
          56              10 :         rescan();
          57              10 : }
          58                 :
          59              10 : void DebtagsIndexer::rescan()
          60                 : {
          61              10 :         ts_main_src = mainSource.timestamp();
          62              10 :         ts_user_src = userSource.timestamp();
          63              10 :         ts_main_tag = Path::timestamp(Path::tagdb());
          64              20 :         ts_main_idx = Path::timestamp(Path::tagdbIndex());
          65              20 :         ts_user_tag = Path::timestamp(Path::userTagdb());
          66              20 :         ts_user_idx = Path::timestamp(Path::userTagdbIndex());
          67              10 : }
          68                 :
          69              10 : bool DebtagsIndexer::needsRebuild() const
          70                 : {
          71                 :         // If there are no indexes of any kind, then we need rebuilding
          72              10 :         if (ts_user_tag == 0 && ts_user_idx == 0 && ts_main_tag == 0 && ts_main_idx == 0)
          73               2 :                 return true;
          74                 :
          75                 :         // If the user index is ok, then we are fine
          76               8 :         if (ts_user_tag >= sourceTimestamp() && ts_user_idx >= sourceTimestamp())
          77               8 :                 return false;
          78                 :
          79                 :         // If there are user sources, then we cannot use the system index
          80               0 :         if (ts_user_src > 0)
          81               0 :                 return true;
          82                 :
          83                 :         // If there are no user sources, then we can fallback on the system
          84                 :         // indexes in case the user indexes are not up to date
          85               0 :         if (ts_main_tag >= sourceTimestamp() && ts_main_idx >= sourceTimestamp())
          86               0 :                 return false;
          87                 :
          88               0 :         return true;
          89                 : }
          90                 :
          91              10 : bool DebtagsIndexer::userIndexIsRedundant() const
          92                 : {
          93                 :         // If there is no user index, then it is not redundant
          94              10 :         if (ts_user_tag == 0 && ts_user_idx == 0)
          95               1 :                 return false;
          96                 :
          97                 :         // If we have user sources, then the user index is never redundant
          98               9 :         if (ts_user_src > 0)
          99               9 :                 return false;
         100                 :
         101                 :         // If the system index is not up to date, then the user index is not
         102                 :         // redundant
         103               0 :         if (ts_main_tag < sourceTimestamp() || ts_main_idx < sourceTimestamp())
         104               0 :                 return false;
         105                 :
         106               0 :         return true;
         107                 : }
         108                 :
         109               2 : bool DebtagsIndexer::rebuild(const std::string& tagfname, const std::string& idxfname)
         110                 : {
         111                 :         using namespace tagcoll;
         112                 :
         113               2 :         diskindex::MasterMMapIndexer master(idxfname);
         114                 :
         115                 :         // Read and merge tag data
         116               2 :         coll::Simple<string, string> merged;
         117               2 :         mainSource.readTags(inserter(merged));
         118               2 :         userSource.readTags(inserter(merged));
         119                 :
         120               2 :         if (merged.empty())
         121                 :                 //throw wibble::exception::Consistency("Reading debtags sources from " + Path::debtagsSourceDir() + " and " + Path::debtagsUserSourceDir(), "Unable to find any tag data");
         122               1 :                 return false;
         123                 :
         124                 :         // Create the pkgid index
         125               1 :         PkgIdGenerator pkgidGen;
         126           21146 :         for (coll::Simple<string, string>::const_iterator i = merged.begin();
         127                 :                         i != merged.end(); ++i)
         128           21145 :                 pkgidGen.pkgs.insert(i->first);
         129                 :
         130                 :         // Temporary in-memory index to use for converting packages to ints while
         131                 :         // creating the debtags index
         132               1 :         char buf[pkgidGen.encodedSize()];
         133               1 :         pkgidGen.encode(buf);
         134               1 :         PkgId pkgid(buf, pkgidGen.encodedSize());
         135                 :
         136                 :         // Create the Debtags index
         137               1 :         coll::IntDiskIndexer tagindexer;
         138               1 :         merged.output(stringToInt(pkgid, voc, inserter(tagindexer)));
         139                 :
         140                 :         // MMap 0: pkgid
         141               1 :         master.append(pkgidGen);
         142                 :         // MMap 1: pkg->tag
         143               1 :         master.append(tagindexer.pkgIndexer());
         144                 :         // MMap 2: tag->pkg
         145               1 :         master.append(tagindexer.tagIndexer());
         146                 :
         147                 :         // Write the tag database in text format
         148               1 :         std::string tmpdb = tagfname + ".tmp";
         149               1 :         FILE* out = fopen(tmpdb.c_str(), "wt");
         150               1 :         if (!out) throw wibble::exception::File(tmpdb, "creating temporary copy of tag index");
         151               1 :         merged.output(textformat::StdioWriter(out));
         152               1 :         fclose(out);
         153                 :
         154                 :         // Perform "atomic" update of the tag database
         155                 :         // FIXME: cannot be atomic because race conditions happening between file
         156                 :         // renames
         157               1 :         if (rename(tmpdb.c_str(), tagfname.c_str()) == -1)
         158               0 :                 throw wibble::exception::System("Renaming " + tmpdb + " to " + tagfname);
         159                 :
         160               1 :         master.commit();
         161               1 :         return true;
         162                 : }
         163                 :
         164              10 : bool DebtagsIndexer::rebuildIfNeeded()
         165                 : {
         166              10 :         if (needsRebuild())
         167                 :         {
         168                 :                 // Decide if we rebuild the user index or the system index
         169                 :
         170               2 :                 if (ts_user_src == 0 && Path::access(Path::debtagsIndexDir(), W_OK) == 0)
         171                 :                 {
         172                 :                         // There are no user sources and we can write to the system index
         173                 :                         // directory: rebuild the system index
         174               1 :                         if (!rebuild(Path::tagdb(), Path::tagdbIndex()))
         175               1 :                                 return false;
         176               0 :                         ts_main_tag = Path::timestamp(Path::tagdb());
         177               0 :                         ts_main_idx = Path::timestamp(Path::tagdbIndex());
         178               0 :                         if (Path::tagdb() == Path::userTagdb())
         179               0 :                                 ts_user_tag = ts_main_tag;
         180               0 :                         if (Path::tagdbIndex() == Path::userTagdbIndex())
         181               0 :                                 ts_user_idx = ts_main_idx;
         182                 :                 } else {
         183               1 :                         wibble::sys::fs::mkFilePath(Path::userTagdb());
         184               2 :                         wibble::sys::fs::mkFilePath(Path::userTagdbIndex());
         185               2 :                         if (!rebuild(Path::userTagdb(), Path::userTagdbIndex()))
         186               0 :                                 return false;
         187               1 :                         ts_user_tag = Path::timestamp(Path::userTagdb());
         188               2 :                         ts_user_idx = Path::timestamp(Path::userTagdbIndex());
         189                 :                 }
         190               1 :                 return true;
         191                 :         }
         192               8 :         return false;
         193                 : }
         194                 :
         195              10 : bool DebtagsIndexer::deleteRedundantUserIndex()
         196                 : {
         197              10 :         if (userIndexIsRedundant())
         198                 :         {
         199                 :                 // Delete the user indexes if they exist
         200               0 :                 if (Path::tagdb() != Path::userTagdb())
         201                 :                 {
         202               0 :                         unlink(Path::userTagdb().c_str());
         203               0 :                         ts_user_tag = 0;
         204                 :                 }
         205               0 :                 if (Path::tagdbIndex() != Path::userTagdbIndex())
         206                 :                 {
         207               0 :                         unlink(Path::userTagdbIndex().c_str());
         208               0 :                         ts_user_idx = 0;
         209                 :                 }
         210               0 :                 return true;
         211                 :         }
         212              10 :         return false;
         213                 : }
         214                 :
         215              10 : bool DebtagsIndexer::getUpToDateTagdb(std::string& tagfname, std::string& idxfname)
         216                 : {
         217                 :         // If there are no indexes of any kind, then we have nothing to return
         218              10 :         if (ts_user_tag == 0 && ts_user_idx == 0 && ts_main_tag == 0 && ts_main_idx == 0)
         219               1 :                 return false;
         220                 :
         221                 :         // If the user index is up to date, use it
         222               9 :         if (ts_user_tag >= sourceTimestamp() &&
         223                 :                 ts_user_idx >= sourceTimestamp())
         224                 :         {
         225               9 :                 tagfname = Path::userTagdb();
         226              18 :                 idxfname = Path::userTagdbIndex();
         227               9 :                 return true;
         228                 :         }
         229                 :
         230                 :         // If the user index is not up to date and we have user sources, we cannot
         231                 :         // fall back to the system index
         232               0 :         if (ts_user_src != 0)
         233               0 :                 return false;
         234                 :
         235                 :         // Fallback to the system index
         236               0 :         if (ts_main_tag >= sourceTimestamp() &&
         237                 :                 ts_main_idx >= sourceTimestamp())
         238                 :         {
         239               0 :                 tagfname = Path::tagdb();
         240               0 :                 idxfname = Path::tagdbIndex();
         241               0 :                 return true;
         242                 :         }
         243                 :
         244               0 :         return false;
         245                 : }
         246                 :
         247                 :
         248                 :
         249              10 : bool DebtagsIndexer::obtainWorkingDebtags(Vocabulary& voc, std::string& tagfname, std::string& idxfname)
         250                 : {
         251              10 :         DebtagsIndexer t(voc);
         252                 :
         253              10 :         t.rebuildIfNeeded();
         254              10 :         t.deleteRedundantUserIndex();
         255              10 :         return t.getUpToDateTagdb(tagfname, idxfname);
         256                 : }
         257                 :
         258                 : }
         259                 : }
         260                 :
         261                 : #include <ept/debtags/maint/sourcedir.tcc>
         262                 : #include <tagcoll/coll/simple.tcc>
         263                 :
         264                 : // vim:set ts=4 sw=4:
         265                 : // -*- C++ -*-

Generated by: LTP GCOV extension version 1.6