Uncompressed WT with ArrayDoc

[SXSI/TextCollection.git] / TCImplementation.cpp

diff --git a/TCImplementation.cpp b/TCImplementation.cpp
index 32f1898..09f6d24 100644 (file)
--- a/TCImplementation.cpp
+++ b/TCImplementation.cpp
@@ -41,7 +41,7 @@ namespace SXSI
 {
 
 // Save file version info
-const uchar TCImplementation::versionFlag = 7;
+const uchar TCImplementation::versionFlag = 8;
 
 /**
  * Constructor inits an empty dynamic FM-index.
@@ -554,6 +554,11 @@ TextCollection::document_result TCImplementation::Contains(uchar const * pattern
     return result;
 }
 
+
+/**
+ ***
+* * FIXME Lessthan or equal
+ */
 TextCollection::document_result TCImplementation::LessThan(uchar const * pattern) const
 {
     TextPosition m = strlen((char *)pattern);
@@ -763,6 +768,9 @@ TCImplementation::TCImplementation(FILE *file, unsigned samplerate_)
     : n(0), samplerate(samplerate_), alphabetrank(0), sampled(0), suffixes(0), 
       suffixDocId(0), numberOfTexts(0), maxTextLength(0), Doc(0)
 {
+//    Tools::StartTimer();
+//    std::cout << std::endl << "Loading..."<< std::endl;
+
     uchar verFlag = 0;
     if (std::fread(&verFlag, 1, 1, file) != 1)
         throw std::runtime_error("TCImplementation::Load(): file read error (version flag).");
@@ -784,7 +792,9 @@ TCImplementation::TCImplementation(FILE *file, unsigned samplerate_)
     if (std::fread(&(this->bwtEndPos), sizeof(TextPosition), 1, file) != 1)
         throw std::runtime_error("TCImplementation::Load(): file read error (bwt end position).");
 
+//    std::cout << "Loading alphabet rank (" << Tools::GetTime() << " s)." << std::endl;
     alphabetrank = static_sequence::load(file);
+//    std::cout << "Loading samples (" << Tools::GetTime() << " s)." << std::endl;
     sampled = static_bitsequence::load(file);
     suffixes = new BlockArray(file);
     suffixDocId = new BlockArray(file);
@@ -794,9 +804,13 @@ TCImplementation::TCImplementation(FILE *file, unsigned samplerate_)
     if (std::fread(&(this->maxTextLength), sizeof(ulong), 1, file) != 1)
         throw std::runtime_error("TCImplementation::Load(): file read error (maxTextLength).");
 
-    Doc = static_sequence::load(file);
+//    std::cout << "Loading Doc (" << Tools::GetTime() << " s)." << std::endl;
+    Doc = new ArrayDoc(file); //static_sequence::load(file);
+//    std::cout << "Loading text storage (" << Tools::GetTime() << " s)." << std::endl;
     textStorage = TextStorage::Load(file);
 
+//    std::cout << "Loading done(" << Tools::GetTime() << " s)." << std::endl;
+
     // FIXME Construct data structures with new samplerate
     //maketables(); 
 }
@@ -1010,8 +1024,8 @@ void TCImplementation::makewavelet(uchar *bwt)
 #endif
 
     alphabet_mapper * am = new alphabet_mapper_none();
-    static_bitsequence_builder * bmb = new static_bitsequence_builder_rrr02(8); // FIXME samplerate?
-    wt_coder * wtc = new wt_coder_binary(bwt,n,am);
+    static_bitsequence_builder * bmb = new static_bitsequence_builder_brw32(8); //rrr02(8); // FIXME samplerate?
+    wt_coder * wtc = new wt_coder_huff(bwt,n,am);//binary(bwt,n,am); // FIXME Huffman shape
     alphabetrank = new static_sequence_wvtree(bwt,n,wtc,bmb,am);
     delete bmb;
     bwt = 0; // already deleted
@@ -1049,7 +1063,8 @@ void TCImplementation::maketables(ulong sampleLength, char tsType)
 
     // Mapping from end-markers to doc ID's:
     unsigned logNumberOfTexts = Tools::CeilLog2(numberOfTexts);
-    uint *endmarkerDocId = new uint[(numberOfTexts * logNumberOfTexts)/(8*sizeof(uint)) + 1];
+//    uint *endmarkerDocId = new uint[(numberOfTexts * logNumberOfTexts)/(8*sizeof(uint)) + 1];
+    BlockArray *endmarkerDocId = new BlockArray(numberOfTexts, logNumberOfTexts);
 
     BlockArray* positions = new BlockArray(sampleLength, Tools::CeilLog2(this->n));
     uint *sampledpositions = new uint[n/(sizeof(uint)*8)+1];
@@ -1088,7 +1103,8 @@ void TCImplementation::maketables(ulong sampleLength, char tsType)
             
             // Record the order of end-markers in BWT:
             ulong endmarkerRank = alphabetrank_i_tmp - 1;
-            set_field(endmarkerDocId, logNumberOfTexts, endmarkerRank, (textId + 1) % numberOfTexts);
+            //set_field(endmarkerDocId, logNumberOfTexts, endmarkerRank, (textId + 1) % numberOfTexts);
+            (*endmarkerDocId)[endmarkerRank] = (textId + 1) % numberOfTexts;
 
             // Store text length and text start position:
             if (textId < (DocId)numberOfTexts - 1)
@@ -1201,12 +1217,14 @@ void TCImplementation::maketables(ulong sampleLength, char tsType)
     HeapProfiler::ResetMaxHeapConsumption();
 #endif
 
-    alphabet_mapper * am = new alphabet_mapper_none();
+    /*alphabet_mapper * am = new alphabet_mapper_none();
     static_bitsequence_builder * bmb = new static_bitsequence_builder_rrr02(32); // FIXME samplerate?
     Doc = new static_sequence_wvtree_noptrs(endmarkerDocId, numberOfTexts, logNumberOfTexts, bmb, am, true);
-    delete bmb;
+    delete bmb;*/
     // delete [] endmarkerDocId; // already deleted in static_sequence_wvtree_noptrs!
 
+    Doc = new ArrayDoc(endmarkerDocId);
+
 #ifdef DEBUG_MEMUSAGE
     std::cerr << "max heap usage after Doc: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
 #endif