git-svn-id: svn+ssh://idea.nguyen.vg/svn/sxsi/trunk/TextCollection@333
3cdefd35-fc62-479d-8e8d-
bae585ffb9ca
*****************************************************************************/
#include "TCImplementation.h"
*****************************************************************************/
#include "TCImplementation.h"
-//#include "HeapProfiler.h" // FIXME remove
+#ifdef DEBUG_MEMUSAGE
+#include "HeapProfiler.h" // FIXME remove
+#endif
#include <iostream>
#include <map>
#include <iostream>
#include <map>
TextCollection::document_result result;
result.reserve(ep-sp+1); // Try to avoid reallocation.
TextCollection::document_result result;
result.reserve(ep-sp+1); // Try to avoid reallocation.
- ulong sampled_rank_i = 0;
// Check each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
uchar c = alphabetrank->access(i);
// Check each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
+ while (c != '\0' && !sampled->access(i))
{
i = C[c]+alphabetrank->rank(c,i)-1;
c = alphabetrank->access(i);
{
i = C[c]+alphabetrank->rank(c,i)-1;
c = alphabetrank->access(i);
}
else // Sampled position
{
}
else // Sampled position
{
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
+ DocId docId = (*suffixDocId)[sampled->rank1(i)-1];
result.push_back(docId);
}
}
result.push_back(docId);
}
}
TextCollection::document_result result;
result.reserve(ep-sp+1); // Try to avoid reallocation.
TextCollection::document_result result;
result.reserve(ep-sp+1); // Try to avoid reallocation.
- ulong sampled_rank_i = 0;
// Check each occurrence, already within [begin, end]
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
uchar c = alphabetrank->access(i);
// Check each occurrence, already within [begin, end]
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
+ while (c != '\0' && !sampled->access(i))
{
i = C[c]+alphabetrank->rank(c,i)-1;
c = alphabetrank->access(i);
{
i = C[c]+alphabetrank->rank(c,i)-1;
c = alphabetrank->access(i);
}
else // Sampled position
{
}
else // Sampled position
{
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
+ DocId docId = (*suffixDocId)[sampled->rank1(i)-1];
result.push_back(docId);
}
}
result.push_back(docId);
}
}
// We want unique document indentifiers, using std::set to collect them
std::set<DocId> resultSet;
// We want unique document indentifiers, using std::set to collect them
std::set<DocId> resultSet;
- ulong sampled_rank_i = 0;
// Check each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
uchar c = alphabetrank->access(i);
// Check each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
+ while (c != '\0' && !sampled->access(i))
{
i = C[c]+alphabetrank->rank(c,i)-1; // LF-mapping
c = alphabetrank->access(i);
{
i = C[c]+alphabetrank->rank(c,i)-1; // LF-mapping
c = alphabetrank->access(i);
- DocId di = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
+ DocId di = (*suffixDocId)[sampled->rank1(i)-1];
assert((unsigned)di < numberOfTexts);
resultSet.insert(di);
}
assert((unsigned)di < numberOfTexts);
resultSet.insert(di);
}
// We want unique document indentifiers, using std::set to collect them
std::set<DocId> resultSet;
// We want unique document indentifiers, using std::set to collect them
std::set<DocId> resultSet;
- ulong sampled_rank_i = 0;
// Check each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
uchar c = alphabetrank->access(i);
// Check each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
+ while (c != '\0' && !sampled->access(i))
{
i = C[c]+alphabetrank->rank(c,i)-1; // LF-mapping
c = alphabetrank->access(i);
{
i = C[c]+alphabetrank->rank(c,i)-1; // LF-mapping
c = alphabetrank->access(i);
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
+ DocId docId = (*suffixDocId)[sampled->rank1(i)-1];
assert((unsigned)docId < numberOfTexts);
if (docId >= begin && docId <= end)
resultSet.insert(docId);
assert((unsigned)docId < numberOfTexts);
if (docId >= begin && docId <= end)
resultSet.insert(docId);
full_result result;
result.reserve(ep-sp+1); // Try to avoid reallocation.
full_result result;
result.reserve(ep-sp+1); // Try to avoid reallocation.
- ulong sampled_rank_i = 0;
// Report each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
TextPosition dist = 0;
uchar c = alphabetrank->access(i);
// Report each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
TextPosition dist = 0;
uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
+ while (c != '\0' && !sampled->access(i))
{
i = C[c]+alphabetrank->rank(c,i)-1;
c = alphabetrank->access(i);
{
i = C[c]+alphabetrank->rank(c,i)-1;
c = alphabetrank->access(i);
- TextPosition textPos = (*suffixes)[sampled_rank_i-1]+dist; //sampled->rank(i)-1] + dist;
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
-// textPos = textPos - (*textStartPos)[docId]; // Offset inside the text
+ TextPosition textPos = (*suffixes)[sampled->rank1(i)-1] + dist;
+ DocId docId = (*suffixDocId)[sampled->rank1(i)-1];
result.push_back(make_pair(docId, textPos));
}
result.push_back(make_pair(docId, textPos));
}
full_result result;
result.reserve(ep-sp+1); // Try to avoid reallocation.
full_result result;
result.reserve(ep-sp+1); // Try to avoid reallocation.
- ulong sampled_rank_i = 0;
// Report each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
TextPosition dist = 0;
uchar c = alphabetrank->access(i);
// Report each occurrence
for (; sp <= ep; ++sp)
{
TextPosition i = sp;
TextPosition dist = 0;
uchar c = alphabetrank->access(i);
- while (c != '\0' && !sampled->IsBitSet(i, &sampled_rank_i))
+ while (c != '\0' && !sampled->access(i))
{
i = C[c]+alphabetrank->rank(c,i)-1;
c = alphabetrank->access(i);
{
i = C[c]+alphabetrank->rank(c,i)-1;
c = alphabetrank->access(i);
- TextPosition textPos = (*suffixes)[sampled_rank_i-1]+dist; //sampled->rank(i)-1] + dist;
- DocId docId = (*suffixDocId)[sampled_rank_i-1]; //sampled->rank(i)-1];
-// textPos = textPos - (*textStartPos)[docId]; // Offset inside the text
+ TextPosition textPos = (*suffixes)[sampled->rank1(i)-1] + dist;
+ DocId docId = (*suffixDocId)[sampled->rank1(i)-1];
if (docId >= begin && docId <= end)
result.push_back(make_pair(docId, textPos));
if (docId >= begin && docId <= end)
result.push_back(make_pair(docId, textPos));
throw std::runtime_error("TCImplementation::Save(): file write error (bwt end position).");
alphabetrank->save(file);
throw std::runtime_error("TCImplementation::Save(): file write error (bwt end position).");
alphabetrank->save(file);
suffixes->Save(file);
suffixDocId->Save(file);
suffixes->Save(file);
suffixDocId->Save(file);
throw std::runtime_error("TCImplementation::Load(): file read error (bwt end position).");
alphabetrank = static_sequence::load(file);
throw std::runtime_error("TCImplementation::Load(): file read error (bwt end position).");
alphabetrank = static_sequence::load(file);
- sampled = new BSGAP(file);
+ sampled = static_bitsequence::load(file);
suffixes = new BlockArray(file);
suffixDocId = new BlockArray(file);
suffixes = new BlockArray(file);
suffixDocId = new BlockArray(file);
//alphabetrank = new RLWaveletTree(bwt, n); // Deletes bwt!
// std::cerr << "heap usage: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
//alphabetrank = new RLWaveletTree(bwt, n); // Deletes bwt!
// std::cerr << "heap usage: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
-// std::cerr << "max heap usage before WT: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
-// HeapProfiler::ResetMaxHeapConsumption(); // FIXME remove
+#ifdef DEBUG_MEMUSAGE
+ std::cerr << "max heap usage before WT: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
+ HeapProfiler::ResetMaxHeapConsumption(); // FIXME remove
+#endif
alphabet_mapper * am = new alphabet_mapper_none();
static_bitsequence_builder * bmb = new static_bitsequence_builder_rrr02(8); // FIXME samplerate?
alphabet_mapper * am = new alphabet_mapper_none();
static_bitsequence_builder * bmb = new static_bitsequence_builder_rrr02(8); // FIXME samplerate?
delete bmb;
bwt = 0; // already deleted
delete bmb;
bwt = 0; // already deleted
-// std::cerr << "heap usage after WT: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
-// std::cerr << "max heap usage after WT: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
+#ifdef DEBUG_MEMUSAGE
+ std::cerr << "heap usage after WT: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
+ std::cerr << "max heap usage after WT: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
+#endif
}
void TCImplementation::maketables()
}
void TCImplementation::maketables()
// Mapping from end-markers to doc ID's:
BlockArray *endmarkerDocId = new BlockArray(numberOfTexts, Tools::CeilLog2(numberOfTexts));
// Mapping from end-markers to doc ID's:
BlockArray *endmarkerDocId = new BlockArray(numberOfTexts, Tools::CeilLog2(numberOfTexts));
- ulong *sampledpositions = new ulong[n/W+1];
- for (ulong i=0;i<n/W+1;i++)
- sampledpositions[i]=0lu;
+ uint *sampledpositions = new uint[n/(sizeof(uint)*8)+1];
+ for (ulong i = 0; i < n / (sizeof(uint)*8) + 1; i++)
+ sampledpositions[i] = 0;
ulong x,p=bwtEndPos;
ulong sampleCount = 0;
ulong x,p=bwtEndPos;
ulong sampleCount = 0;
x=(i==n-1)?0:i+1;
if (x % samplerate == 0 && posOfSuccEndmarker - x > samplerate) {
x=(i==n-1)?0:i+1;
if (x % samplerate == 0 && posOfSuccEndmarker - x > samplerate) {
- Tools::SetField(sampledpositions,1,p,1);
+ set_field(sampledpositions,1,p,1);
(*positions)[sampleCount] = p;
(*tmpSuffix)[sampleCount] = x; // FIXME remove
sampleCount ++;
(*positions)[sampleCount] = p;
(*tmpSuffix)[sampleCount] = x; // FIXME remove
sampleCount ++;
- sampled = new BSGAP(sampledpositions,n,true);
- sampleLength = sampled->rank(n-1);
+ sampled = new static_bitsequence_rrr02(sampledpositions, n, 16);
+ delete [] sampledpositions;
+ sampleLength = sampled->rank1(n-1);
assert(sampleCount == sampleLength);
// Suffixes store an offset from the text start position
assert(sampleCount == sampleLength);
// Suffixes store an offset from the text start position
for(ulong i=0; i<sampleLength; i++) {
assert((*positions)[i] < n);
for(ulong i=0; i<sampleLength; i++) {
assert((*positions)[i] < n);
- ulong j = sampled->rank((*positions)[i]);
+ ulong j = sampled->rank1((*positions)[i]);
+
if (j==0) j=sampleLength;
TextPosition textPos = (*tmpSuffix)[i];
(*suffixDocId)[j-1] = DocIdAtTextPos(textStartPos, textPos);
if (j==0) j=sampleLength;
TextPosition textPos = (*tmpSuffix)[i];
(*suffixDocId)[j-1] = DocIdAtTextPos(textStartPos, textPos);
delete positions;
delete textStartPos;
delete positions;
delete textStartPos;
- // std::cerr << "heap usage before Doc: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
-
+#ifdef DEBUG_MEMUSAGE
+ std::cerr << "max heap usage before Doc: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
+ HeapProfiler::ResetMaxHeapConsumption();
+#endif
uint *tmp = new uint[numberOfTexts]; // FIXME Silly...
for (unsigned i = 0; i < numberOfTexts; ++i)
uint *tmp = new uint[numberOfTexts]; // FIXME Silly...
for (unsigned i = 0; i < numberOfTexts; ++i)
static_bitsequence_builder * bmb = new static_bitsequence_builder_brw32(16); // FIXME samplerate?
Doc = new static_sequence_wvtree_noptrs(tmp, numberOfTexts, bmb, am);
delete bmb;
static_bitsequence_builder * bmb = new static_bitsequence_builder_brw32(16); // FIXME samplerate?
Doc = new static_sequence_wvtree_noptrs(tmp, numberOfTexts, bmb, am);
delete bmb;
+ // delete [] tmp; // already deleted!
-// std::cerr << "heap usage after Doc: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
+#ifdef DEBUG_MEMUSAGE
+ std::cerr << "max heap usage after Doc: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
+#endif
/*document_result res = Doc->access(3, 3, 0, 3);
cout << "Access result: ";
/*document_result res = Doc->access(3, 3, 0, 3);
cout << "Access result: ";
#include "BitRank.h"
#include "TextCollection.h"
#include "BlockArray.h"
#include "BitRank.h"
#include "TextCollection.h"
#include "BlockArray.h"
// Include from XMLTree/libcds
#include <basics.h> // Defines W == 32
// Include from XMLTree/libcds
#include <basics.h> // Defines W == 32
static_sequence * alphabetrank;
// Sample structures for texts longer than samplerate
static_sequence * alphabetrank;
// Sample structures for texts longer than samplerate
- BSGAP *sampled; // FIXME Replace with RRR02
+ static_bitsequence * sampled; // FIXME Replace with RRR02
BlockArray *suffixes;
BlockArray *suffixDocId;
BlockArray *suffixes;
BlockArray *suffixDocId;