-/******************************************************************************
- * Copyright (C) 2006-2008 by Veli Mäkinen and Niko Välimäki *
- * *
- * *
- * This program is free software; you can redistribute it and/or modify *
- * it under the terms of the GNU Lesser General Public License as published *
- * by the Free Software Foundation; either version 2 of the License, or *
- * (at your option) any later version. *
- * *
- * This program is distributed in the hope that it will be useful, *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
- * GNU Lesser General Public License for more details. *
- * *
- * You should have received a copy of the GNU Lesser General Public License *
- * along with this program; if not, write to the *
- * Free Software Foundation, Inc., *
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. *
- *****************************************************************************/
-#include "CSA.h"
-#include "TextCollection.h"
-
-// Conflicts with std::min and std::max
-#undef min
-#undef max
-#include "dcover/bwt.hpp"
-
-//#include "HeapProfiler.h" // FIXME remove
-
-#include <iostream>
-#include <queue>
-#include <set>
-#include <vector>
-#include <utility>
-#include <stdexcept>
-#include <cassert>
-#include <cstring> // For strlen()
-using std::vector;
-using std::pair;
-using std::make_pair;
-using std::map;
-
-namespace SXSI
-{
-
-// Save file version info
-const uchar CSA::versionFlag = 2;
-
-////////////////////////////////////////////////////////////////////////////
-// Class CSA::THuffAlphabetRank
-// FIXME Unused code
-CSA::THuffAlphabetRank::THuffAlphabetRank(uchar *s, TextPosition n, TCodeEntry *codetable, unsigned level) {
- left = NULL;
- right = NULL;
- bitrank = NULL;
- ch = s[0];
- leaf = false;
- this->codetable = codetable;
-
- bool *B = new bool[n];
- TextPosition sum=0,i;
- /*
- for (i=0; i< n; i++) {
- printf("%c:", (char)((int)s[i]-128));
- for (r=0;r<codetable[(int)s[i]].bits;r++)
- if (codetable[(int)s[i]].code & (1u <<r))
- printf("1");
- else printf("0");
- printf("\n");
- }
- printf("\n");
- if (level > 100) return;*/
- for (i=0;i<n;i++)
- if (codetable[(int)s[i]].code & (1u << level)) {
- B[i] = true;
- sum++;
- }
- else B[i] = false;
- if (sum==0 || sum==n) {
- delete [] B;
- leaf = true;
- return;
- }
- uchar *sfirst, *ssecond;
- //if (n-sum > 0)
- sfirst = new uchar[n-sum];
- //if (sum > 0)
- ssecond = new uchar[sum];
- unsigned j=0,k=0;
- for (i=0;i<n;i++)
- if (B[i]) ssecond[k++] = s[i];
- else sfirst[j++] = s[i];
- ulong *Binbits = new ulong[n/W+1];
- for (i=0;i<n;i++)
- Tools::SetField(Binbits,1,i,B[i]);
- delete [] B;
- bitrank = new BitRank(Binbits,n,true);
- //if (j > 0) {
- left = new THuffAlphabetRank(sfirst,j,codetable,level+1);
- delete [] sfirst;
- //}
- //if (k>0) {
- right = new THuffAlphabetRank(ssecond,k,codetable,level+1);
- delete [] ssecond;
- //}
-}
-
-
-bool CSA::THuffAlphabetRank::Test(uchar *s, ulong n) {
- // testing that the code works correctly
- int C[256];
- unsigned i,j;
- bool correct=true;
- for (j=0;j<256;j++)
- C[j] = 0;
- for (i=0;i<n;i++) {
- C[(int)s[i]]++;
- if (C[(int)s[i]] != (int)rank((int)s[i],i)) {
- correct = false;
- printf("%d (%c): %d<>%d\n",i,(int)s[i]-128,C[(int)s[i]],(int)rank((int)s[i],i));
- }
- }
- return correct;
-}
-
-CSA::THuffAlphabetRank::~THuffAlphabetRank() {
- if (left!=NULL) delete left;
- if (right!=NULL) delete right;
- if (bitrank!=NULL)
- delete bitrank;
-}
-
-
-////////////////////////////////////////////////////////////////////////////
-// Class CSA
-
-/**
- * Constructor inits an empty dynamic FM-index.
- * Samplerate defaults to TEXTCOLLECTION_DEFAULT_SAMPLERATE.
- */
-CSA::CSA(unsigned samplerate)
- : n(0), samplerate(0), alphabetrank(0), codetable(0), sampled(0), suffixes(0),
- suffixDocId(0), numberOfTexts(0), numberOfAllTexts(0), maxTextLength(0), endmarkerDocId(0),
- emptyTextRank(0)
-{
- this->samplerate = samplerate;
-
-#ifdef CSA_TEST_BWT
- // FIXME TODO : DynFMI needs distribution of characters before hand
- // This will create fully balanced wavelet tree for all chars [0, 255].
- uchar temp[256];
- for (unsigned i = 0; i < 255; ++i)
- temp[i] = i+1;
- temp[255] = 0;
- dynFMI = new DynFMI(temp, 1, 255, false);
-
- /* Debug code: take char distribution from data.txt.
- uchar *temp = Tools::GetFileContents("data.txt", 0);
- dynFMI = new DynFMI(temp,1790000, strlen((char *)temp), false);
- delete [] temp;*/
-#endif
-}
-
-/**
- * Insert new text
- *
- * Given text must include end-marker.
- * Text identifiers are numbered starting from 0.
- */
-void CSA::InsertText(uchar const * text)
-{
- // Sanity check:
-#ifdef CSA_TEST_BWT
- assert(dynFMI != 0);
-#endif
-
- TextPosition m = std::strlen((char *)text) + 1;
- if (m > maxTextLength)
- maxTextLength = m; // Store length of the longest text seen so far.
-
- if (m > 1)
- {
- this->n += m;
- this->numberOfTexts ++;
- this->numberOfAllTexts ++;
-#ifdef CSA_TEST_BWT
- dynFMI->addText(text, m);
-#endif
-
- texts.append((char *) text);
- texts.append(1, '\0');
- }
- else
- {
- emptyTextId.insert(numberOfAllTexts); // FIXME Using too much space here
- this->numberOfAllTexts ++;
- }
-}
-
-void CSA::MakeStatic()
-{
- // Sanity check:
-#ifdef CSA_TEST_BWT
- if (dynFMI == 0)
- throw std::runtime_error("CSA::MakeStatic(): Data structure is already static (dynFMI == 0).");
-#endif
-
- if (texts.size() == 0) // Empty collection
- {
- ++n;
- ++numberOfTexts;
- texts.append(1, '\0');
- }
-
- // Bitvector of empty texts, FIXME Remove?
- {
- //std::cout << std::endl << "texts: " << numberOfTexts << ", all texts " << numberOfAllTexts << ", size : " << emptyTextId.size() <<std::endl;
- ulong *tempB = new ulong[numberOfAllTexts/W + 1];
- for (ulong i = 0; i < numberOfAllTexts/W + 1; ++i)
- tempB[i] = 0;
- for (std::set<unsigned>::const_iterator it = emptyTextId.begin(); it != emptyTextId.end(); ++it)
- Tools::SetField(tempB, 1, (*it), 1);
- emptyTextId.clear();
- emptyTextRank = new BSGAP(tempB, numberOfAllTexts, true);
- }
-
- texts.reserve(0); // Release extra capacity
-
-/* FILE *fp = fopen("texts.txt", "wb");
- std::cout << "Wrote " << fwrite(texts.c_str(), 1, n, fp) << " bytes into texts.txt." << std::endl;
- fclose(fp);*/
-
- uchar *bwt = new uchar[n];
- {
- // More succinct solution via StringIterator, see below.
-/* unsigned* maptexts = new unsigned[n+1];
- // Map text chars to [0..255+numberOfTexts]
- unsigned count = 0;
- for (ulong i = 0; i < n; ++i)
- if (texts[i] == 0)
- maptexts[i] = ++count; // endmarkers \in [1..numberOfTexts]
- else {
- uchar c = (uchar)texts[i];
- maptexts[i] = (unsigned)c + numberOfTexts;
- }
- maptexts[n] = '\0';
- assert(count == numberOfTexts);
-
- std::cout << "maptext: ";
- for (ulong i = 0; i <= n; ++i)
- std::cout << maptexts[i] << ", ";
- std::cout << std::endl;*/
-
- // Mark endmarker positions into bitvector
- ulong * endmarkers = new ulong[n/W+1];
- for (ulong i = 0; i < n/W+1; ++i)
- endmarkers[i] = 0;
- for (ulong i = 0; i < n; ++i)
- if (texts[i] == 0)
- Tools::SetField(endmarkers, 1, i, 1);
- BitRank* br = new BitRank(endmarkers, n, true);
- assert(numberOfTexts == br->rank(n-1));
-
- // Build iterators, FIXME clean up iterator construction.
- StringIterator itBegin((uchar const *)texts.c_str(), (uchar const *)texts.c_str(), n, numberOfTexts, br);
- StringIterator itEnd((uchar const *)texts.c_str() + n,(uchar const *)texts.c_str(), n, numberOfTexts, br);
-
- bwtEndPos = (ulong)compute_bwt(itBegin, itEnd, //&maptexts[0], &maptexts[n],
- &bwt[0], numberOfTexts);
-
- bwt[--bwtEndPos] = '\0';
- texts.erase();
- texts.reserve(0); // Release capacity
- delete br; // bitvector of endmarkers
- } // End of bw transform
-// std::cerr << "Time after BWT: " << Tools::GetTime() << std::endl;
-
-/* fp = fopen("bwt.txt", "wb");
- std::cout << "Wrote " << fwrite(bwt, 1, n, fp) << " bytes into bwt.txt." << std::endl;
- fclose(fp);*/
-
-
-#ifdef CSA_TEST_BWT
- {
- uchar *bwtTest = dynFMI->getBWT();
- printf("123456789012345678901234567890123456789\n");
- for (TextPosition i = 0; i < n && i < 100; i ++)
- if (bwt[i] < 50)
- printf("%d", (int)bwt[i]);
- else
- printf("%c", bwt[i]);
- printf("\n");
- for (TextPosition i = 0; i < n && i < 100; i ++)
- if (bwtTest[i] < 50)
- printf("%d", (int)bwtTest[i]);
- else
- printf("%c", bwtTest[i]);
- printf("\n");
-
- // Sanity check
- assert(numberOfTexts == dynFMI->getCollectionSize());
-
- delete dynFMI;
- dynFMI = 0;
- for (ulong i = 0; i < n; ++i)
- if (bwt[i] != bwtTest[i])
- {
- std::cout << "i = " << i << ", bwt = " << (unsigned)bwt[i] << ", " << (unsigned)bwtTest[i] << std::endl;
- assert(0);
- }
- delete [] bwtTest;
- }
-#endif // CSA_TEST_BWT
-
- makewavelet(bwt); // Deletes bwt!
- bwt = 0;
-
- // Make sampling tables
- maketables();
-}
-
-bool CSA::EmptyText(DocId k) const
-{
- assert(k < (DocId)numberOfTexts);
- if (emptyTextRank->IsBitSet(k))
- return true;
- return false;
-}
-
-uchar* CSA::GetText(DocId k) const
-{
- assert(k < (DocId)numberOfTexts);
-
- ulong textRank = 0;
- if (emptyTextRank->IsBitSet(k, &textRank))
- {
- uchar* result = new uchar[1];
- result[0] = 0;
- return result;
- }
- // Map to non-empty text
- k -= textRank; //emptyTextRank->rank(k);
-
- TextPosition i = k;
-
- string result;
- // Reserve average string length to avoid reallocs
- result.reserve(n/numberOfTexts);
-
- uchar c = alphabetrank->access(i);
- while (c != '\0')
- {
- result.push_back(c);
- i = C[c]+alphabetrank->rank(c,i)-1;
-
- c = alphabetrank->access(i); // "next" char.
- }
-
- // Convert to uchar (FIXME return string?)
- i = result.size();
- uchar* res = new uchar[i+1];
- res[i] = '\0';
- for (ulong j = 0; j < i; ++j)
- res[i-j-1] = result[j];
- return res;
-}
-/*
- * Not supported
-uchar* CSA::GetText(DocId k, TextPosition i, TextPosition j) const
-{
- assert(k < (DocId)numberOfTexts);
- assert(j < (*textLength)[k]);
- assert(i <= j);
-
- ulong textRank = 0;
- if (emptyTextRank->IsBitSet(k, &textRank))
- {
- uchar* result = new uchar[1];
- result[0] = 0;
- return result;
- }
-
- // Map to non-empty text
- k -= textRank; // emptyTextRank->rank(k);
-
- // Start position of k'th text
- ulong start = (*textStartPos)[k];
-
- return Substring(i + start, j-i+1);
- }*/
-
-/******************************************************************
- * Existential queries
- */
-bool CSA::IsPrefix(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return true;
-
- TextPosition sp = 0, ep = 0;
- Search(pattern, m, &sp, &ep);
-
- // Check for end-marker(s) in result interval
- if (CountEndmarkers(sp, ep))
- return true;
- return false;
-}
-
-bool CSA::IsPrefix(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return true;
-
- TextPosition sp = 0, ep = 0;
- Search(pattern, m, &sp, &ep);
-
- // Check for end-marker(s) in result interval
- if (CountEndmarkers(sp, ep, begin, end))
- return true;
- return false;
-}
-
-
-bool CSA::IsSuffix(uchar const *pattern) const
-{
- // Here counting is as fast as IsSuffix():
- if (CountSuffix(pattern) > 0)
- return true;
- return false;
-}
-
-bool CSA::IsSuffix(uchar const *pattern, DocId begin, DocId end) const
-{
- // Here counting is as fast as IsSuffix():
- if (CountSuffix(pattern, begin, end) > 0)
- return true;
- return false;
-}
-
-bool CSA::IsEqual(uchar const *pattern) const
-{
- TextPosition m = std::strlen((char *)pattern);
- if (m == 0)
- {
- if (numberOfAllTexts - numberOfTexts > 0u)
- return true; // Empty texts exists
- return false; // No empty texts exists
- }
-
- TextPosition sp = 0, ep = 0;
- // Match including end-marker
- Search(pattern, m+1, &sp, &ep);
-
- // Check for end-marker(s) in result interval
- if (CountEndmarkers(sp, ep))
- return true;
- return false;
-}
-
-bool CSA::IsEqual(uchar const *pattern, DocId begin, DocId end) const
-{
- TextPosition m = std::strlen((char *)pattern);
- if (m == 0)
- {
- if (numberOfAllTexts - numberOfTexts > 0u)
- return true; // Empty texts exists
- return false; // No empty texts exists
- }
-
- TextPosition sp = 0, ep = 0;
- // Match including end-marker
- Search(pattern, m+1, &sp, &ep, begin, end);
-
- // Check for end-marker(s) in result interval
- if (CountEndmarkers(sp, ep))
- return true;
- return false;
-}
-
-bool CSA::IsContains(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return true;
-
- TextPosition sp = 0, ep = 0;
- // Just check if pattern exists somewhere
- ulong count = Search(pattern, m, &sp, &ep);
-
- if (count > 0)
- return true;
- return false;
-}
-
-bool CSA::IsContains(uchar const * pattern, DocId begin, DocId end) const
-{
- // Here counting is as fast as existential querying
- if (CountContains(pattern, begin, end) > 0)
- return true; // FIXME No need to filter result set
- return false;
-}
-
-bool CSA::IsLessThan(uchar const * pattern) const
-{
- if (CountLessThan(pattern) > 0)
- return true;
- return false;
-}
-
-bool CSA::IsLessThan(uchar const * pattern, DocId begin, DocId end) const
-{
- if (CountLessThan(pattern, begin, end) > 0)
- return true;
- return false;
-}
-
-/******************************************************************
- * Counting queries
- */
-ulong CSA::Count(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return 0;
-
- TextPosition sp = 0, ep = 0;
- unsigned count = (unsigned) Search(pattern, m, &sp, &ep);
- return count;
-}
-
-unsigned CSA::CountPrefix(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return numberOfAllTexts;
-
- TextPosition sp = 0, ep = 0;
- Search(pattern, m, &sp, &ep);
-
- // Count end-markers in result interval
- return CountEndmarkers(sp, ep);
-}
-
-unsigned CSA::CountPrefix(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return numberOfAllTexts;
-
- TextPosition sp = 0, ep = 0;
- Search(pattern, m, &sp, &ep);
-
- // Count end-markers in result interval
- return CountEndmarkers(sp, ep, begin, end);
-}
-
-unsigned CSA::CountSuffix(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return numberOfAllTexts;
-
- TextPosition sp = 0, ep = 0;
- // Search with end-marker
- unsigned count = (unsigned) Search(pattern, m+1, &sp, &ep);
-
- return count;
-}
-
-unsigned CSA::CountSuffix(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return numberOfAllTexts;
-
- TextPosition sp = 0, ep = 0;
- // Search with end-marker
- unsigned count = (unsigned) Search(pattern, m+1, &sp, &ep, begin, end);
-
- return count;
-}
-
-unsigned CSA::CountEqual(uchar const *pattern) const
-{
- TextPosition m = strlen((char const *)pattern);
- if (m == 0)
- return numberOfAllTexts - numberOfTexts; // Empty texts.
-
- TextPosition sp = 0, ep = 0;
- // Match including end-marker
- Search(pattern, m+1, &sp, &ep);
-
- // Count end-markers in result interval
- return CountEndmarkers(sp, ep);
-}
-
-unsigned CSA::CountEqual(uchar const *pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char const *)pattern);
- if (m == 0)
- return numberOfAllTexts - numberOfTexts; // Empty texts.
-
- TextPosition sp = 0, ep = 0;
- // Match including end-marker
- Search(pattern, m+1, &sp, &ep, begin, end);
-
- // Count end-markers in result interval
- return CountEndmarkers(sp, ep); // Already within [begin, end]
-}
-
-unsigned CSA::CountContains(uchar const * pattern) const
-{
- TextPosition m = strlen((char const *)pattern);
- if (m == 0)
- return numberOfAllTexts; // Total number of texts.
-
- // Here counting is as slow as fetching the result set
- // because we have to filter out occ's that fall within same document.
- TextCollection::document_result result = Contains(pattern);
- return result.size();
-}
-
-unsigned CSA::CountContains(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char const *)pattern);
- if (m == 0)
- return numberOfAllTexts; // Total number of texts.
-
- // Here counting is as slow as fetching the result set
- // because we have to filter out occ's that fall within same document.
- TextCollection::document_result result = Contains(pattern, begin, end);
- return result.size();
-}
-
-// Less than or equal
-unsigned CSA::CountLessThan(uchar const * pattern) const
-{
- TextPosition m = strlen((char const *)pattern);
- if (m == 0)
- return numberOfAllTexts - numberOfTexts; // Empty texts.
-
- TextPosition sp = 0, ep = 0;
- SearchLessThan(pattern, m, &sp, &ep);
-
- // Count end-markers in result interval
- return CountEndmarkers(sp, ep);
-}
-
-unsigned CSA::CountLessThan(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char const *)pattern);
- if (m == 0)
- return numberOfAllTexts - numberOfTexts; // Empty texts.
-
- TextPosition sp = 0, ep = 0;
- SearchLessThan(pattern, m, &sp, &ep);
-
- // Count end-markers in result interval
- return CountEndmarkers(sp, ep, begin, end);
-}
-
-/**
- * Document reporting queries
- */
-TextCollection::document_result CSA::Prefix(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return TextCollection::document_result(); // FIXME Should return all 1...k
-
- TextPosition sp = 0, ep = 0;
- Search(pattern, m, &sp, &ep);
-
- TextCollection::document_result result;
-
- // Report end-markers in result interval
- unsigned resultSize = CountEndmarkers(sp, ep);
- if (resultSize == 0)
- return result;
-
- result.reserve(resultSize); // Try to avoid reallocation.
-
- // Iterate through end-markers in [sp,ep]:
- unsigned i = 0;
- if (sp > 0)
- i = alphabetrank->rank(0, sp - 1);
- while (resultSize)
- {
- // End-marker we found belongs to the "preceeding" doc in the collection
- DocId docId = ((*endmarkerDocId)[i] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(docId);
-
- -- resultSize;
- ++ i;
- }
-
- return result;
-}
-
-TextCollection::document_result CSA::Prefix(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return TextCollection::document_result(); // FIXME Should return all 1...k
-
- TextPosition sp = 0, ep = 0;
- Search(pattern, m, &sp, &ep);
-
- TextCollection::document_result result;
-
- // Report end-markers in result interval
- unsigned resultSize = CountEndmarkers(sp, ep);
- if (resultSize == 0)
- return result;
-
- result.reserve(resultSize); // Try to avoid reallocation.
-
- // Iterate through end-markers in [sp,ep] and [begin, end]:
- unsigned i = 0;
- if (sp > 0)
- i = alphabetrank->rank(0, sp - 1);
- while (resultSize)
- {
- // End-marker we found belongs to the "preceeding" doc in the collection
- DocId docId = ((*endmarkerDocId)[i] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- if (docId >= begin && docId <= end)
- result.push_back(docId);
-
- -- resultSize;
- ++ i;
- }
-
- return result;
-}
-
-TextCollection::document_result CSA::Suffix(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return TextCollection::document_result(); // FIXME Should return all 1...k
-
- TextPosition sp = 0, ep = 0;
- // Search with end-marker
- Search(pattern, m+1, &sp, &ep);
-
- TextCollection::document_result result;
- result.reserve(ep-sp+1); // Try to avoid reallocation.
-
- ulong sampled_rank_i = 0;
- // Check each occurrence
- for (; sp <= ep; ++sp)
- {
- TextPosition i = sp;
-
- uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
- {
- i = C[c]+alphabetrank->rank(c,i)-1;
- c = alphabetrank->access(i);
- }
- // Assert: c == '\0' OR sampled->IsBitSet(i)
-
- if (c == '\0')
- {
- // Rank among the end-markers in BWT
- unsigned endmarkerRank = alphabetrank->rank(0, i) - 1;
-
- // End-marker that we found belongs to the "preceeding" doc in collection:
- DocId docId = ((*endmarkerDocId)[endmarkerRank] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(docId);
- }
- else // Sampled position
- {
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(docId);
- }
- }
-
- return result;
-}
-
-TextCollection::document_result CSA::Suffix(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return TextCollection::document_result(); // FIXME Should return all 1...k
-
- TextPosition sp = 0, ep = 0;
- // Search with end-marker
- Search(pattern, m+1, &sp, &ep, begin, end);
-
- TextCollection::document_result result;
- result.reserve(ep-sp+1); // Try to avoid reallocation.
-
- ulong sampled_rank_i = 0;
- // Check each occurrence, already within [begin, end]
- for (; sp <= ep; ++sp)
- {
- TextPosition i = sp;
-
- uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
- {
- i = C[c]+alphabetrank->rank(c,i)-1;
- c = alphabetrank->access(i);
- }
- // Assert: c == '\0' OR sampled->IsBitSet(i)
-
- if (c == '\0')
- {
- // Rank among the end-markers in BWT
- unsigned endmarkerRank = alphabetrank->rank(0, i) - 1;
-
- // End-marker that we found belongs to the "preceeding" doc in collection:
- DocId docId = ((*endmarkerDocId)[endmarkerRank] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(docId);
- }
- else // Sampled position
- {
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(docId);
- }
- }
-
- return result;
-}
-
-
-TextCollection::document_result CSA::Equal(uchar const *pattern) const
-{
- TextPosition m = strlen((char const *)pattern);
- if (m == 0)
- return TextCollection::document_result(); // FIXME Should return all empty texts
-
- TextPosition sp = 0, ep = 0;
- // Match including end-marker
- Search(pattern, m+1, &sp, &ep);
-
- TextCollection::document_result result;
-
- // Report end-markers in result interval
- unsigned resultSize = CountEndmarkers(sp, ep);
- if (resultSize == 0)
- return result;
-
- result.reserve(resultSize); // Try to avoid reallocation.
-
- unsigned i = 0;
- if (sp > 0)
- i = alphabetrank->rank(0, sp - 1);
- while (resultSize)
- {
- // End-marker we found belongs to the "previous" doc in the collection
- DocId docId = ((*endmarkerDocId)[i] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(docId);
-
- -- resultSize;
- ++ i;
- }
-
- return result;
-}
-
-TextCollection::document_result CSA::Equal(uchar const *pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char const *)pattern);
- if (m == 0)
- return TextCollection::document_result(); // FIXME Should return all empty texts
-
- TextPosition sp = 0, ep = 0;
- // Match including end-marker
- Search(pattern, m+1, &sp, &ep, begin, end);
-
- TextCollection::document_result result;
-
- // Report end-markers in result interval
- unsigned resultSize = CountEndmarkers(sp, ep);
- if (resultSize == 0)
- return result;
-
- result.reserve(resultSize); // Try to avoid reallocation.
-
- unsigned i = 0;
- if (sp > 0)
- i = alphabetrank->rank(0, sp - 1);
- while (resultSize)
- {
- // End-marker we found belongs to the "previous" doc in the collection
- DocId docId = ((*endmarkerDocId)[i] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(docId); // already within [begin, end]
-
- -- resultSize;
- ++ i;
- }
-
- return result;
-}
-
-
-TextCollection::document_result CSA::Contains(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return TextCollection::document_result();
-
- TextPosition sp = 0, ep = 0;
- // Search all occurrences
- Search(pattern, m, &sp, &ep);
-
- // We want unique document indentifiers, using std::set to collect them
- std::set<DocId> resultSet;
-
- ulong sampled_rank_i = 0;
- // Check each occurrence
- for (; sp <= ep; ++sp)
- {
- TextPosition i = sp;
- uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
- {
- i = C[c]+alphabetrank->rank(c,i)-1; // LF-mapping
- c = alphabetrank->access(i);
- }
- if (c == '\0')
- {
- // Rank among the end-markers in BWT
- unsigned endmarkerRank = alphabetrank->rank(0, i) - 1;
-
- // End-marker that we found belongs to the "preceeding" doc in collection:
- DocId docId = ((*endmarkerDocId)[endmarkerRank] + 1) % numberOfTexts;
- resultSet.insert(docId);
- }
- else
- {
- DocId di = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
- assert((unsigned)di < numberOfTexts);
- resultSet.insert(di);
- }
- }
-
- // Convert std::set to std::vector
- TextCollection::document_result result(resultSet.begin(), resultSet.end());
- // Map to doc ID's
- for (document_result::iterator it = result.begin(); it != result.end(); ++it)
- *it = emptyTextRank->select0(*it+1);
- return result;
-}
-
-TextCollection::document_result CSA::Contains(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return TextCollection::document_result();
-
- TextPosition sp = 0, ep = 0;
- // Search all occurrences
- Search(pattern, m, &sp, &ep);
-
- // We want unique document indentifiers, using std::set to collect them
- std::set<DocId> resultSet;
-
- ulong sampled_rank_i = 0;
- // Check each occurrence
- for (; sp <= ep; ++sp)
- {
- TextPosition i = sp;
- uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
- {
- i = C[c]+alphabetrank->rank(c,i)-1; // LF-mapping
- c = alphabetrank->access(i);
- }
- if (c == '\0')
- {
- // Rank among the end-markers in BWT
- unsigned endmarkerRank = alphabetrank->rank(0, i) - 1;
-
- // End-marker that we found belongs to the "preceeding" doc in collection:
- DocId docId = ((*endmarkerDocId)[endmarkerRank] + 1) % numberOfTexts;
- if (docId >= begin && docId <= end)
- resultSet.insert(docId);
- }
- else
- {
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
- assert((unsigned)docId < numberOfTexts);
- if (docId >= begin && docId <= end)
- resultSet.insert(docId);
- }
- }
-
- // Convert std::set to std::vector
- TextCollection::document_result result(resultSet.begin(), resultSet.end());
- // Map to doc ID's
- for (document_result::iterator it = result.begin(); it != result.end(); ++it)
- *it = emptyTextRank->select0(*it+1);
- return result;
-}
-
-TextCollection::document_result CSA::LessThan(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return TextCollection::document_result(); // empty result set
-
- TextPosition sp = 0, ep = 0;
- SearchLessThan(pattern, m, &sp, &ep);
-
- TextCollection::document_result result;
-
- // Report end-markers in result interval
- unsigned resultSize = CountEndmarkers(sp, ep);
- if (resultSize == 0)
- return result;
-
- result.reserve(resultSize); // Try to avoid reallocation.
-
- // Iterate through end-markers in [sp,ep]:
- unsigned i = 0;
- if (sp > 0)
- i = alphabetrank->rank(0, sp - 1);
- while (resultSize)
- {
- // End-marker we found belongs to the "preceeding" doc in the collection
- DocId docId = ((*endmarkerDocId)[i] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(docId);
-
- -- resultSize;
- ++ i;
- }
-
- return result;
-}
-
-TextCollection::document_result CSA::LessThan(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return TextCollection::document_result(); // empty result set
-
- TextPosition sp = 0, ep = 0;
- SearchLessThan(pattern, m, &sp, &ep);
-
- TextCollection::document_result result;
-
- // Report end-markers in result interval
- unsigned resultSize = CountEndmarkers(sp, ep);
- if (resultSize == 0)
- return result;
-
- result.reserve(resultSize); // Try to avoid reallocation.
-
- // Iterate through end-markers in [sp,ep] and [begin, end]:
- unsigned i = 0;
- if (sp > 0)
- i = alphabetrank->rank(0, sp - 1);
- while (resultSize)
- {
- // End-marker we found belongs to the "preceeding" doc in the collection
- DocId docId = ((*endmarkerDocId)[i] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- if (docId >= begin && docId <= end)
- result.push_back(docId);
-
- -- resultSize;
- ++ i;
- }
-
- return result;
-}
-
-/**
- * Full result set queries
- */
-TextCollection::full_result CSA::FullContains(uchar const * pattern) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return full_result(); // FIXME Throw exception?
-
- TextPosition sp = 0, ep = 0;
- // Search all occurrences
- Search(pattern, m, &sp, &ep);
-
- full_result result;
- result.reserve(ep-sp+1); // Try to avoid reallocation.
-
- ulong sampled_rank_i = 0;
- // Report each occurrence
- for (; sp <= ep; ++sp)
- {
- TextPosition i = sp;
- TextPosition dist = 0;
- uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
- {
- i = C[c]+alphabetrank->rank(c,i)-1;
- c = alphabetrank->access(i);
- ++ dist;
- }
- if (c == '\0')
- {
- // Rank among the end-markers in BWT
- unsigned endmarkerRank = alphabetrank->rank(0, i) - 1;
-
- // End-marker that we found belongs to the "preceeding" doc in collection:
- DocId docId = ((*endmarkerDocId)[endmarkerRank] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(make_pair(docId, dist));
- }
- else
- {
- TextPosition textPos = (*suffixes)[sampled_rank_i-1]+dist; //sampled->rank(i)-1] + dist;
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
-// textPos = textPos - (*textStartPos)[docId]; // Offset inside the text
-
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- result.push_back(make_pair(docId, textPos));
- }
- }
-
- return result;
-}
-
-TextCollection::full_result CSA::FullContains(uchar const * pattern, DocId begin, DocId end) const
-{
- TextPosition m = strlen((char *)pattern);
- if (m == 0)
- return full_result(); // FIXME Throw exception?
-
- TextPosition sp = 0, ep = 0;
- // Search all occurrences
- Search(pattern, m, &sp, &ep);
-
- full_result result;
- result.reserve(ep-sp+1); // Try to avoid reallocation.
-
- ulong sampled_rank_i = 0;
- // Report each occurrence
- for (; sp <= ep; ++sp)
- {
- TextPosition i = sp;
- TextPosition dist = 0;
- uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
- {
- i = C[c]+alphabetrank->rank(c,i)-1;
- c = alphabetrank->access(i);
- ++ dist;
- }
- if (c == '\0')
- {
- // Rank among the end-markers in BWT
- unsigned endmarkerRank = alphabetrank->rank(0, i) - 1;
-
- // End-marker that we found belongs to the "preceeding" doc in collection:
- DocId docId = ((*endmarkerDocId)[endmarkerRank] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- if (docId >= begin && docId <= end)
- result.push_back(make_pair(docId, dist));
- }
- else
- {
- TextPosition textPos = (*suffixes)[sampled_rank_i-1]+dist; //sampled->rank(i)-1] + dist;
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
-// textPos = textPos - (*textStartPos)[docId]; // Offset inside the text
-
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- if (docId >= begin && docId <= end)
- result.push_back(make_pair(docId, textPos));
- }
- }
-
- return result;
-}
-
-
-/**
- * Save index to a file handle
- *
- * Throws a std::runtime_error exception on i/o error.
- * First byte that is saved represents the version number of the save file.
- * In version 2 files, the data fields are:
- * uchar versionFlag;
- TextPosition n;
- unsigned samplerate;
- unsigned C[256];
- TextPosition bwtEndPos;
- static_sequence * alphabetrank;
- BSGAP *sampled;
- BlockArray *suffixes;
- BlockArray *suffixDocId;
- unsigned numberOfTexts;
- unsigned numberOfAllTexts;
- ulong maxTextLength;
- BlockArray *endmarkerDocId;
- BSGAP *emptyTextRank;
- */
-void CSA::Save(FILE *file) const
-{
- // Saving version info:
- if (std::fwrite(&versionFlag, 1, 1, file) != 1)
- throw std::runtime_error("CSA::Save(): file write error (version flag).");
-
- if (std::fwrite(&(this->n), sizeof(TextPosition), 1, file) != 1)
- throw std::runtime_error("CSA::Save(): file write error (n).");
- if (std::fwrite(&(this->samplerate), sizeof(unsigned), 1, file) != 1)
- throw std::runtime_error("CSA::Save(): file write error (samplerate).");
-
- for(ulong i = 0; i < 256; ++i)
- if (std::fwrite(this->C + i, sizeof(unsigned), 1, file) != 1)
- throw std::runtime_error("CSA::Save(): file write error (C table).");
-
- if (std::fwrite(&(this->bwtEndPos), sizeof(TextPosition), 1, file) != 1)
- throw std::runtime_error("CSA::Save(): file write error (bwt end position).");
-
- alphabetrank->save(file);
- sampled->Save(file);
- suffixes->Save(file);
- suffixDocId->Save(file);
-
- if (std::fwrite(&(this->numberOfTexts), sizeof(unsigned), 1, file) != 1)
- throw std::runtime_error("CSA::Save(): file write error (numberOfTexts).");
- if (std::fwrite(&(this->numberOfAllTexts), sizeof(unsigned), 1, file) != 1)
- throw std::runtime_error("CSA::Save(): file write error (numberOfAllTexts).");
- if (std::fwrite(&(this->maxTextLength), sizeof(ulong), 1, file) != 1)
- throw std::runtime_error("CSA::Save(): file write error (maxTextLength).");
-
- endmarkerDocId->Save(file);
- emptyTextRank->Save(file);
- fflush(file);
-}
-
-
-/**
- * Load index from a file handle
- *
- * Throws a std::runtime_error exception on i/o error.
- * For more info, see CSA::Save().
- */
-void CSA::Load(FILE *file, unsigned samplerate)
-{
- // Delete everything
-#ifdef CSA_TEST_BWT
- delete dynFMI; dynFMI = 0;
-#endif
- delete alphabetrank; alphabetrank = 0;
- delete sampled; sampled = 0;
- delete suffixes; suffixes = 0;
- delete suffixDocId; suffixDocId = 0;
- delete [] codetable; codetable = 0;
-
- delete endmarkerDocId; endmarkerDocId = 0;
- delete emptyTextRank; emptyTextRank = 0;
-
- this->maxTextLength = 0;
- this->numberOfTexts = 0;
- this->numberOfAllTexts = 0;
- this->samplerate = samplerate;
- this->n = 0;
-
- uchar verFlag = 0;
- if (std::fread(&verFlag, 1, 1, file) != 1)
- throw std::runtime_error("CSA::Load(): file read error (version flag).");
- if (verFlag != CSA::versionFlag)
- throw std::runtime_error("CSA::Load(): invalid save file version.");
-
- if (std::fread(&(this->n), sizeof(TextPosition), 1, file) != 1)
- throw std::runtime_error("CSA::Load(): file read error (n).");
- if (std::fread(&samplerate, sizeof(unsigned), 1, file) != 1)
- throw std::runtime_error("CSA::Load(): file read error (samplerate).");
-// FIXME samplerate can not be changed during load.
-// if (this->samplerate == 0)
-// this->samplerate = samplerate;
-
- for(ulong i = 0; i < 256; ++i)
- if (std::fread(this->C + i, sizeof(unsigned), 1, file) != 1)
- throw std::runtime_error("CSA::Load(): file read error (C table).");
-
- if (std::fread(&(this->bwtEndPos), sizeof(TextPosition), 1, file) != 1)
- throw std::runtime_error("CSA::Load(): file read error (bwt end position).");
-
- alphabetrank = static_sequence::load(file);
- sampled = new BSGAP(file);
- suffixes = new BlockArray(file);
- suffixDocId = new BlockArray(file);
-
- if (std::fread(&(this->numberOfTexts), sizeof(unsigned), 1, file) != 1)
- throw std::runtime_error("CSA::Load(): file read error (numberOfTexts).");
- if (std::fread(&(this->numberOfAllTexts), sizeof(unsigned), 1, file) != 1)
- throw std::runtime_error("CSA::Load(): file read error (numberOfAllTexts).");
- if (std::fread(&(this->maxTextLength), sizeof(ulong), 1, file) != 1)
- throw std::runtime_error("CSA::Load(): file read error (maxTextLength).");
-
- endmarkerDocId = new BlockArray(file);
- emptyTextRank = new BSGAP(file);
-
- // FIXME Construct data structures with new samplerate
- //maketables();
-}
-
-
-
-/**
- * Rest of the functions follow...
- */
-
-
-// FIXME Use 2D-search structure
-unsigned CSA::CountEndmarkers(TextPosition sp, TextPosition ep, DocId begin, DocId end) const
-{
- if (sp > ep)
- return 0;
-
- ulong ranksp = 0;
- if (sp != 0)
- ranksp = alphabetrank->rank(0, sp - 1);
-
- unsigned resultSize = alphabetrank->rank(0, ep) - ranksp;
- if (resultSize == 0)
- return 0;
-
- // Count end-markers in result interval and within [begin, end]
- unsigned count = 0;
- unsigned i = ranksp;
- while (resultSize)
- {
- // End-marker we found belongs to the "previous" doc in the collection
- DocId docId = ((*endmarkerDocId)[i] + 1) % numberOfTexts;
- // Map to doc ID:
- docId = emptyTextRank->select0(docId+1);
- if (docId >= begin && docId <= end)
- ++ count;
-
- -- resultSize;
- ++ i;
- }
- return count;
-}
-
-ulong CSA::Search(uchar const * pattern, TextPosition m, TextPosition *spResult, TextPosition *epResult) const
-{
- // use the FM-search replacing function Occ(c,1,i) with alphabetrank->rank(c,i)
- int c = (int)pattern[m-1];
- TextPosition i=m-1;
- TextPosition sp = C[c];
- TextPosition ep = C[c+1]-1;
- while (sp<=ep && i>=1)
- {
-// printf("i = %lu, c = %c, sp = %lu, ep = %lu\n", i, pattern[i], sp, ep);
- c = (int)pattern[--i];
- sp = C[c]+alphabetrank->rank(c,sp-1);
- ep = C[c]+alphabetrank->rank(c,ep)-1;
- }
- *spResult = sp;
- *epResult = ep;
- if (sp<=ep)
- return ep - sp + 1;
- else
- return 0;
-}
-
-ulong CSA::Search(uchar const * pattern, TextPosition m, TextPosition *spResult, TextPosition *epResult, DocId begin, DocId end) const
-{
- // use the FM-search replacing function Occ(c,1,i) with alphabetrank->rank(c,i)
- int c = (int)pattern[m-1];
- assert(c == 0); // Start from endmarkers
- TextPosition i=m-1;
- TextPosition sp = begin;
- TextPosition ep = end;
- while (sp<=ep && i>=1)
- {
-// printf("i = %lu, c = %c, sp = %lu, ep = %lu\n", i, pattern[i], sp, ep);
- c = (int)pattern[--i];
- sp = C[c]+alphabetrank->rank(c,sp-1);
- ep = C[c]+alphabetrank->rank(c,ep)-1;
- }
- *spResult = sp;
- *epResult = ep;
- if (sp<=ep)
- return ep - sp + 1;
- else
- return 0;
-}
-
-
-ulong CSA::SearchLessThan(uchar const * pattern, TextPosition m, TextPosition *spResult, TextPosition *epResult) const
-{
- // use the FM-search replacing function Occ(c,1,i) with alphabetrank->rank(c,i)
- uint c = (int)pattern[m-1];
- TextPosition i=m-1;
- TextPosition sp = 1;
- TextPosition ep = C[c+1]-1;
- while (sp<=ep && i>=1)
- {
-// printf("i = %lu, c = %c, sp = %lu, ep = %lu\n", i, pattern[i], sp, ep);
- c = (int)pattern[--i];
- uint result = alphabetrank->rank(c,ep);
- if (result == ~0u)
- ep = 0;
- else
- ep = C[c]+result-1;
- }
- *spResult = sp;
- *epResult = ep;
- if (sp<=ep)
- return ep - sp + 1;
- else
- return 0;
-}
-
-
-CSA::~CSA() {
-#ifdef CSA_TEST_BWT
- delete dynFMI;
-#endif
- delete alphabetrank;
- delete sampled;
- delete suffixes;
- delete suffixDocId;
- delete [] codetable; // FIXME remove
-
- delete endmarkerDocId;
- delete emptyTextRank;
-}
-
-void CSA::makewavelet(uchar *bwt)
-{
- ulong i, min = 0,
- max;
- for (i=0;i<256;i++)
- C[i]=0;
- for (i=0;i<n;++i)
- C[(int)bwt[i]]++;
- for (i=0;i<256;i++)
- if (C[i]>0) {min = i; break;}
- for (i=255;i>=min;--i)
- if (C[i]>0) {max = i; break;}
-
- ulong prev=C[0], temp;
- C[0]=0;
- for (i=1;i<256;i++) {
- temp = C[i];
- C[i]=C[i-1]+prev;
- prev = temp;
- }
-// this->codetable = node::makecodetable(bwt,n);
-// alphabetrank = new THuffAlphabetRank(bwt,n, this->codetable,0);
-// delete [] bwt;
- //alphabetrank = new RLWaveletTree(bwt, n); // Deletes bwt!
-// std::cerr << "heap usage: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
-// std::cerr << "max heap usage before WT: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
-
-// HeapProfiler::ResetMaxHeapConsumption(); // FIXME remove
-
- alphabet_mapper * am = new alphabet_mapper_none();
- static_bitsequence_builder * bmb = new static_bitsequence_builder_rrr02(8); // FIXME samplerate?
-// static_bitsequence_builder * bmb = new static_bitsequence_builder_brw32(16); // FIXME samplerate?
- wt_coder * wtc = new wt_coder_binary(bwt,n,am);
- alphabetrank = new static_sequence_wvtree(bwt,n,wtc,bmb,am);
- delete bmb;
- bwt = 0; // already deleted
-
-// std::cerr << "heap usage: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
-// std::cerr << "max heap usage after WT: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
-}
-
-void CSA::maketables()
-{
- // Calculate BWT end-marker position (of last inserted text)
- // Note: bwtEndPos already set!
- // and the length of the first text (counter l):
-#ifdef CSA_TEST_BWT
- {
- ulong i = 0;
- ulong l = 1;
- uint alphabetrank_i_tmp = 0;
- uchar c = alphabetrank->access(i, alphabetrank_i_tmp);
- while (c != '\0')
- {
- i = C[c]+alphabetrank_i_tmp-1;
- ++l;
- c = alphabetrank->access(i, alphabetrank_i_tmp);
- }
-
- if (i != bwtEndPos) // compare result
- {
- std::cout << "i = " << i << ", bwtendpos = " << bwtEndPos << std::endl;
- exit(0);
- }
- }
-#endif
-
- // Build up arrays for text length and starting positions
- // FIXME Temp, remove
- //BlockArray* textLength = new BlockArray(numberOfTexts, Tools::CeilLog2(maxTextLength));
- BlockArray* textStartPos = new BlockArray(numberOfTexts, Tools::CeilLog2(this->n));
- //(*textLength)[0] = l;
- (*textStartPos)[0] = 0;
-
- // Construct samples
- ulong sampleLength = (n%samplerate==0) ? n/samplerate : n/samplerate+1;
- unsigned ceilLog2n = Tools::CeilLog2(n);
- BlockArray* positions = new BlockArray(sampleLength, ceilLog2n);
- BlockArray* tmpSuffix = new BlockArray(sampleLength, ceilLog2n);
-
- // Mapping from end-markers to doc ID's:
- endmarkerDocId = new BlockArray(numberOfTexts, Tools::CeilLog2(numberOfTexts));
-
- ulong *sampledpositions = new ulong[n/W+1];
- for (ulong i=0;i<n/W+1;i++)
- sampledpositions[i]=0lu;
-
- ulong x,p=bwtEndPos;
- ulong sampleCount = 0;
- // Keeping track of text position of prev. end-marker seen
- ulong posOfSuccEndmarker = n;
- DocId textId = numberOfTexts;
- ulong ulongmax = 0;
- ulongmax--;
- uint alphabetrank_i_tmp =0;
-
- //positions:
- for (ulong i=n-1;i<ulongmax;i--) { // TODO bad solution with ulongmax?
- // i substitutes SA->GetPos(i)
- x=(i==n-1)?0:i+1;
-
- if (x % samplerate == 0 && posOfSuccEndmarker - x > samplerate) {
- Tools::SetField(sampledpositions,1,p,1);
- (*positions)[sampleCount] = p;
- (*tmpSuffix)[sampleCount] = x; // FIXME remove
- sampleCount ++;
- }
-
- uchar c = alphabetrank->access(p, alphabetrank_i_tmp);
- if (c == '\0')
- {
- --textId;
-
- // Record the order of end-markers in BWT:
- ulong endmarkerRank = alphabetrank_i_tmp - 1;
- (*endmarkerDocId)[endmarkerRank] = textId;
-
- // Store text length and text start position:
- if (textId < (DocId)numberOfTexts - 1)
- {
- //(*textLength)[textId + 1] = posOfSuccEndmarker - x;
- (*textStartPos)[textId + 1] = x; // x is the position of end-marker.
- posOfSuccEndmarker = x;
- }
-
- // LF-mapping from '\0' does not work with this (pseudo) BWT (see details from Wolfgang's thesis).
- p = textId; // Correct LF-mapping to the last char of the previous text.
- }
- else
- p = C[c]+alphabetrank_i_tmp-1;
- }
- assert(textId == 0);
-
- sampled = new BSGAP(sampledpositions,n,true);
- sampleLength = sampled->rank(n-1);
- assert(sampleCount == sampleLength);
-
- // Suffixes store an offset from the text start position
- suffixes = new BlockArray(sampleLength, Tools::CeilLog2(maxTextLength));
- suffixDocId = new BlockArray(sampleLength, Tools::CeilLog2(numberOfTexts));
-
- for(ulong i=0; i<sampleLength; i++) {
- assert((*positions)[i] < n);
- ulong j = sampled->rank((*positions)[i]);
- if (j==0) j=sampleLength;
- TextPosition textPos = (*tmpSuffix)[i];
- (*suffixDocId)[j-1] = DocIdAtTextPos(textStartPos, textPos);
-
- assert((unsigned)DocIdAtTextPos(textStartPos, textPos) < numberOfTexts);
- assert((*suffixDocId)[j-1] < numberOfTexts);
- // calculate offset from text start:
- (*suffixes)[j-1] = textPos - (*textStartPos)[(*suffixDocId)[j-1]];
- }
- // FIXME Temp, remove
- delete tmpSuffix;
- delete positions;
-// delete textLength;
- delete textStartPos;
-}
-
-
-/**
- * Finds document identifier for given text position
- *
- * Starting text position of the document is stored into second parameter.
- * Binary searching on text starting positions.
- */
-TextCollection::DocId CSA::DocIdAtTextPos(BlockArray* textStartPos, TextPosition i) const
-{
- assert(i < n);
-
- DocId a = 0;
- DocId b = numberOfTexts - 1;
- while (a < b)
- {
- DocId c = a + (b - a)/2;
- if ((*textStartPos)[c] > i)
- b = c - 1;
- else if ((*textStartPos)[c+1] > i)
- return c;
- else
- a = c + 1;
- }
-
- assert(a < (DocId)numberOfTexts);
- assert(i >= (*textStartPos)[a]);
- assert(i < (a == (DocId)numberOfTexts - 1 ? n : (*textStartPos)[a+1]));
- return a;
-}
-
-CSA::TCodeEntry * CSA::node::makecodetable(uchar *text, TextPosition n)
-{
- TCodeEntry *result = new TCodeEntry[ 256 ];
-
- count_chars( text, n, result );
- std::priority_queue< node, std::vector< node >, std::greater<node> > q;
-//
-// First I push all the leaf nodes into the queue
-//
- for ( unsigned int i = 0 ; i < 256 ; i++ )
- if ( result[ i ].count )
- q.push(node( i, result[ i ].count ) );
-//
-// This loop removes the two smallest nodes from the
-// queue. It creates a new internal node that has
-// those two nodes as children. The new internal node
-// is then inserted into the priority queue. When there
-// is only one node in the priority queue, the tree
-// is complete.
-//
-
- while ( q.size() > 1 ) {
- node *child0 = new node( q.top() );
- q.pop();
- node *child1 = new node( q.top() );
- q.pop();
- q.push( node( child0, child1 ) );
- }
-//
-// Now I compute and return the codetable
-//
- q.top().maketable(0u,0u, result);
- q.pop();
- return result;
-}
-
-
-void CSA::node::maketable(unsigned code, unsigned bits, TCodeEntry *codetable) const
-{
- if ( child0 )
- {
- child0->maketable( SetBit(code,bits,0), bits+1, codetable );
- child1->maketable( SetBit(code,bits,1), bits+1, codetable );
- delete child0;
- delete child1;
- }
- else
- {
- codetable[value].code = code;
- codetable[value].bits = bits;
- }
-}
-
-void CSA::node::count_chars(uchar *text, TextPosition n, TCodeEntry *counts )
-{
- ulong i;
- for (i = 0 ; i < 256 ; i++ )
- counts[ i ].count = 0;
- for (i=0; i<n; i++)
- counts[(int)text[i]].count++;
-}
-
-unsigned CSA::node::SetBit(unsigned x, unsigned pos, unsigned bit) {
- return x | (bit << pos);
-}
-
-} // namespace SXSI
-