1 // Test driver for text collection
10 #include "TextCollectionBuilder.h"
11 #include "HeapProfiler.h"
12 using SXSI::TextCollectionBuilder;
13 using SXSI::TextCollection;
15 void printDocumentResult(TextCollection::document_result dr)
17 TextCollection::document_result::iterator it;
18 printf("document result:");
19 for (it = dr.begin(); it != dr.end(); ++it)
25 void printFullResult(TextCollection::full_result fr)
27 TextCollection::full_result::iterator it;
28 printf("full result:");
29 for (it = fr.begin(); it != fr.end(); ++it)
30 printf(" (%i, %lu)", (*it).first, (*it).second);
41 int heap_base = HeapProfiler::GetHeapConsumption();
42 std::cerr << "Initial heap usage : " << heap_base << "\n";
43 TextCollectionBuilder *tcb = new TextCollectionBuilder(5);
44 heap_base = HeapProfiler::GetHeapConsumption ();
45 std::cerr << "Heap usage after InitTextCollection : " << heap_base << "\n";
48 while (not(cin.eof())){
49 getline(cin,str); // Read line by line.
50 // cin >> str; // Read word by word.
51 data = (uchar *) str.c_str();
55 tcb->InsertText(data);
59 if ( i % 100000 == 0) {
60 std::cerr << "Inserted : " << i << " strings\n";
61 std::cerr << "Number of bytes inserted : " << j << "b \n";
62 std::cerr << "Heap usage used for strings: " << HeapProfiler::GetHeapConsumption() - heap_base
64 std::cerr << "Ratio is : " << (float) (HeapProfiler::GetHeapConsumption() - heap_base) / ((float) j) <<"\n";
70 //the whole file as 20 strings:
71 /* uchar *temp = Tools::GetFileContents("data/english.100MB", 0);
72 ulong n = strlen((char *)temp);
73 std::cout << "n = " << n << std::endl;
76 for (i = 0; i < 5; ++i)
80 std::cout << "inserted " << strlen((char *)it) << " bytes." << std::endl;
86 std::cout << "over bounds" << std::endl;
90 std::cerr << "Creating new text collection with " << i << " strings (total " << j/1024 << " kb)\n";
91 std::cerr << "max heap usage: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
92 HeapProfiler::ResetMaxHeapConsumption();
93 std::cerr << "Before MakeStatic() [press enter]\n";
95 // This will print the maximum mem usage during construction time:
96 TextCollection* tc = tcb->InitTextCollection();//SXSI::TextStorage::TYPE_LZ_INDEX);
98 std::cerr << "After MakeStatic() [press enter]\n";
99 // This will print the maximum mem usage during MakeStatic():
100 std::cerr << "max heap usage: " << HeapProfiler::GetMaxHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
102 std::cerr << "heap usage: " << HeapProfiler::GetHeapConsumption()/(1024*1024) << " Mbytes" << std::endl;
104 std::cerr << "After Delete [press enter]\n";
105 std::cerr << "heap usage: " << HeapProfiler::GetHeapConsumption() << " bytes" << std::endl;