1 // Test driver for text collection
14 static struct timeval t1;
15 static struct timeval t2;
18 #define STARTTIMER() (gettimeofday(&t1,NULL))
19 #define STOPTIMER() (gettimeofday(&t2,NULL))
20 #define GETTIME() (((t2.tv_sec - t1.tv_sec) * 1000000.0 + (t2.tv_usec - t1.tv_usec))/1000.0)
22 void * last_brk = NULL;
25 void * current_brk = sbrk(0);
26 long int mem = ((long int) current_brk ) - ((long int) last_brk);
27 //last_brk = current_brk;
28 return (mem/1024/1024);
31 #include "TextCollectionBuilder.h"
32 using SXSI::TextCollection;
33 using SXSI::TextCollectionBuilder;
37 int main(int argc, char**argv)
39 string str = string("");
40 unsigned int text_size = 0;
41 unsigned int max_str = 0;
42 unsigned int num_str = 0;
47 TextCollectionBuilder *tcb =
48 TextCollectionBuilder::create(64, TextCollectionBuilder::index_type_swcsa);
53 std::cerr << "Filling collection\n";
55 while (not(cin.eof()) ){
56 std::getline(cin, str); // Read line by line.
57 if (str.compare("------") != 0 ){
59 tcb->InsertText((unsigned char*) str.c_str());
62 if (num_str % 10000 == 0){
65 std::cerr << "Added " << num_str << " strings in "
66 << time << " ms, max_mem=" << get_mem() << "\n";
72 if (max_str < str.size())
74 text_size += str.size();
81 std::cerr << "Number of bytes inserted : " << text_size << "\n";
83 std::cerr << "Calling InitTextCollection() for sf=64: ";
85 tc = tcb->InitTextCollection();
88 std::cerr << time << "ms, max_mem = " << get_mem() << "\n" ;
92 file = fopen("index_64.tc","w+");
93 std::cerr << "Saving to index_64.tc ";
95 tc->Save(file,"index_64.tc");
98 std::cerr << time << "ms, max_mem = " << get_mem() << "\n" ;
102 std::cerr << "Freeing memory : max_mem = " << get_mem() << "\n";
103 std::cerr << "Loading TextCollection from saved file ... ";
104 file = fopen("index_64.tc","r");
105 tc = TextCollection::Load(file, "index_64.tc", TextCollection::index_mode_default);