X-Git-Url: http://git.nguyen.vg/gitweb/?a=blobdiff_plain;f=XMLTreeBuilder.cpp;h=d9b083248a7082d3f94e706e66e7677c2fef4f22;hb=d79d6498e2d585560d915592ef59f3ad6a57b3c7;hp=27bc862691948186b0b766f090c1e578219a179e;hpb=aa6692a9fd2badf8e8e686b92075f041dc03bbef;p=SXSI%2FXMLTree.git diff --git a/XMLTreeBuilder.cpp b/XMLTreeBuilder.cpp index 27bc862..d9b0832 100644 --- a/XMLTreeBuilder.cpp +++ b/XMLTreeBuilder.cpp @@ -1,152 +1,164 @@ - +#include "common.h" #include "XMLTreeBuilder.h" -#include "basics.h" +#include "timings.h" +using std::string; + +XMLTreeBuilder::~XMLTreeBuilder(){ + //free(par_aux); + free(tags_aux); + //delete other stuff. + +} // OpenDocument(empty_texts): it starts the construction of the data structure for // the XML document. Parameter empty_texts indicates whether we index empty texts // in document or not. Returns a non-zero value upon success, NULLT in case of error. -int XMLTreeBuilder::OpenDocument(bool empty_texts, int sample_rate_text, bool dtc) +int XMLTreeBuilder::OpenDocument(bool empty_texts, + int sample_rate_text, + bool dtc, + TextCollectionBuilder::index_type_t index_type) { - found_attributes = false; npar = 0; parArraySize = 1; - ntagnames = 4; disable_tc = dtc; - - indexing_empty_texts = empty_texts; - + text_index_type = index_type; + STARTTIMER(); + par_aux = (pb *)umalloc(sizeof(pb)*parArraySize); - + tags_aux = (TagType *) umalloc(sizeof(TagType)); - - TagName = (unsigned char **) umalloc(4*sizeof(unsigned char*)); - TagName[0] = (unsigned char *) umalloc(4*sizeof(unsigned char)); - strcpy((char *) TagName[0], "<@>"); - TagName[1] = (unsigned char *) umalloc(4*sizeof(unsigned char)); - strcpy((char *) TagName[1], "<$>"); - TagName[2] = (unsigned char *) umalloc(5*sizeof(unsigned char)); - strcpy((char *) TagName[2], "/<@>"); - TagName[3] = (unsigned char *) umalloc(5*sizeof(unsigned char)); - strcpy((char *) TagName[3], "/<$>"); - - if (!indexing_empty_texts) - empty_texts_aux = (unsigned int *)umalloc(sizeof(unsigned int)); - + + TagName = new vector(); + tIdMap = new std::unordered_map(); + + REGISTER_TAG(TagName,tIdMap,DOCUMENT_OPEN_TAG); + REGISTER_TAG(TagName,tIdMap,ATTRIBUTE_OPEN_TAG); + REGISTER_TAG(TagName,tIdMap,PCDATA_OPEN_TAG); + REGISTER_TAG(TagName,tIdMap,ATTRIBUTE_DATA_OPEN_TAG); + REGISTER_TAG(TagName,tIdMap,CLOSING_TAG); + REGISTER_TAG(TagName,tIdMap,DOCUMENT_CLOSE_TAG); + REGISTER_TAG(TagName,tIdMap,ATTRIBUTE_CLOSE_TAG); + REGISTER_TAG(TagName,tIdMap,PCDATA_CLOSE_TAG); + REGISTER_TAG(TagName,tIdMap,ATTRIBUTE_DATA_CLOSE_TAG); + + if (disable_tc) TextBuilder = 0; - else - TextBuilder = new TextCollectionBuilder((unsigned)sample_rate_text); + else + TextBuilder = TextCollectionBuilder::create((unsigned)sample_rate_text, index_type); + Text = 0; - + empty_texts_aux = (unsigned int *)ucalloc(sizeof(unsigned int),1); + eta_size = sizeof(unsigned int); return 1; // indicates success in the initialization of the data structure } // CloseDocument(): it finishes the construction of the data structure for the XML -// document. Tree and tags are represented in the final form, dynamic data -// structures are made static, and the flag "finished" is set to true. After that, +// document. Tree and tags are represented in the final form, dynamic data +// structures are made static, and the flag "finished" is set to true. After that, // the data structure can be queried. XMLTree *XMLTreeBuilder::CloseDocument() - { - // closing parenthesis for the tree root - par_aux = (pb *)urealloc(par_aux, sizeof(pb)*(1+npar/(8*sizeof(pb)))); - setbit(par_aux, npar, CP); - npar++; - - // makes the text collection static - if (!disable_tc) { - assert(Text == 0); - assert(TextBuilder != 0); - Text = TextBuilder->InitTextCollection(); - delete TextBuilder; - TextBuilder = 0; - } + { + //closing parenthesis for the tree root + //par_aux = (pb *)urealloc(par_aux, sizeof(pb)*(1+npar/(8*sizeof(pb)))); + //setbit(par_aux, npar, CP); + //npar++; - XMLTree *T = new XMLTree(par_aux, npar, TagName, ntagnames, empty_texts_aux, tags_aux, - Text, CachedText, indexing_empty_texts, disable_tc); - return T; + // makes the text collection static + STOPTIMER(Parsing); + PRINTTIME("Parsing XML Document", Parsing); + + XMLTree *T = new XMLTree(par_aux, + npar, + TagName, + tIdMap, + empty_texts_aux, // freed by the constructor + tags_aux, // freed by the constructor + TextBuilder, // freed by the constructor + disable_tc, + text_index_type); + tags_aux = 0; + empty_texts_aux = 0; + return T; } // NewOpenTag(tagname): indicates the event of finding a new opening tag in the document. // Tag name is given. Returns a non-zero value upon success, and returns NULLT // in case of failing when trying to insert the new tag. -int XMLTreeBuilder::NewOpenTag(unsigned char *tagname) +int XMLTreeBuilder::NewOpenTag(string tagname) { int i; - // inserts a new opening parentheses in the bit sequence if (sizeof(pb)*8*parArraySize == npar) { // no space left for the new parenthesis - par_aux = (pb *)urealloc(par_aux, sizeof(pb)*2*parArraySize); - parArraySize *= 2; + + // If array is already 1GB, be gentler when resizing: + if (sizeof(pb)*parArraySize >= 1024*1024*1024) + parArraySize += (128*1024*1024); + else + parArraySize *= 2; + par_aux = (pb *) urealloc(par_aux, sizeof(pb)*parArraySize); + }; + + bp_setbit(par_aux,npar,OP); // marks a new opening parenthesis + + TagIdMapIT tag_id = tIdMap->find(tagname); + + if (tag_id == tIdMap->end()){ + REGISTER_TAG(TagName,tIdMap,tagname); + i = TagName->size() - 1; } - - setbit(par_aux,npar,OP); // marks a new opening parenthesis - - // transforms the tagname into a tag identifier. If the tag is new, we insert - // it in the table. - for (i=0; i") was called - if (i==0) - found_attributes=true; - - if (i==ntagnames) { // the tag is a new one, then we insert it - TagName = (unsigned char **)urealloc(TagName, sizeof(char *)*(ntagnames+1)); - - if (!TagName) { - fprintf(stderr, "Error: not enough memory\n"); - return NULLT; - } - - ntagnames++; - TagName[i] = (unsigned char *)umalloc(sizeof(unsigned char)*(strlen((const char *)tagname)+1)); - strcpy((char *)TagName[i], (const char *)tagname); - } + else + i = tag_id->second; + + if (tagname.compare(PCDATA_OPEN_TAG) == 0 || + tagname.compare(ATTRIBUTE_DATA_OPEN_TAG) == 0){ + }; + + tags_aux = (TagType *) urealloc(tags_aux, sizeof(TagType)*(npar + 1)); tags_aux[npar] = i; // inserts the new tag id within the preorder sequence of tags - + npar++; - - return 1; // success + + return 1; // success } // NewClosingTag(tagname): indicates the event of finding a new closing tag in the document. // Tag name is given. Returns a non-zero value upon success, and returns NULLT // in case of failing when trying to insert the new tag. -int XMLTreeBuilder::NewClosingTag(unsigned char *tagname) +int XMLTreeBuilder::NewClosingTag(string tagname) { int i; - // inserts a new closing parentheses in the bit sequence if (sizeof(pb)*8*parArraySize == npar) { // no space left for the new parenthesis - par_aux = (pb *)urealloc(par_aux, sizeof(pb)*2*parArraySize); - parArraySize *= 2; - } - - setbit(par_aux,npar,CP); // marks a new closing parenthesis - - // transforms the tagname into a tag identifier. If the tag is new, we insert - // it in the table. - for (i=0; i= 1024*1024*1024) + parArraySize += (128*1024*1024); + else + parArraySize *= 2; + par_aux = (pb *)urealloc(par_aux, sizeof(pb)*parArraySize); + }; + + bp_setbit(par_aux,npar,CP); // marks a new closing parenthesis + + //tagname.insert(0,"/"); + + //TagIdMapIT tag_id = tIdMap->find(tagname); + + // if (tag_id == tIdMap->end()){ + // REGISTER_TAG(TagName,tIdMap,tagname); + // i = TagName->size() - 1; + // } + // else + // i = tag_id->second; + + tags_aux = (TagType *)urealloc(tags_aux, sizeof(TagType)*(npar + 1)); + + tags_aux[npar] = CLOSING_TAG_ID; // inserts the new tag id within the preorder sequence of tags - tags_aux[npar] = i; // inserts the new tag id within the preorder sequence of tags - npar++; return 1; // success @@ -156,41 +168,23 @@ int XMLTreeBuilder::NewClosingTag(unsigned char *tagname) // NewText(s): indicates the event of finding a new (non-empty) text s in the document. // The new text is inserted within the text collection. Returns a non-zero value upon // success, NULLT in case of error. -int XMLTreeBuilder::NewText(unsigned char *s) +int XMLTreeBuilder::NewText(string text) { - if (disable_tc) { - XMLTreeBuilder::NewEmptyText(); - return 1; - } + if (!disable_tc) { + if (text.empty()) + TextBuilder->InsertText((uchar *)"\001"); + else + TextBuilder->InsertText((uchar *) text.c_str()); + }; - if (!indexing_empty_texts) { - empty_texts_aux = (unsigned int *)urealloc(empty_texts_aux, sizeof(pb)*(1+(npar-1)/(8*sizeof(pb)))); - bitset(empty_texts_aux, npar-1); // marks the non-empty text with a 1 in the bit vector - } - - TextBuilder->InsertText(s); - string cpps = (char*) s; - CachedText.push_back(cpps); - - return 1; // success - } + int n_eta_size = sizeof(uint)*(1+(npar-1)/(8*sizeof(uint))); + //see basics.h, recalloc resizes and sets the new area to 0. -// NewEmptyText(): indicates the event of finding a new empty text in the document. -// In case of indexing empty and non-empty texts, we insert the empty texts into the -// text collection. In case of indexing only non-empty texts, it just indicates an -// empty text in the bit vector of empty texts. Returns a non-zero value upon -// success, NULLT in case of error. -int XMLTreeBuilder::NewEmptyText() - { - unsigned char c = 0; + empty_texts_aux = (uint *)urecalloc(empty_texts_aux,eta_size,n_eta_size); + eta_size = n_eta_size; + bitset(empty_texts_aux, npar-1); // marks the non-empty text with a 1 in the bit vector - if (!indexing_empty_texts) { - empty_texts_aux = (unsigned int *)urealloc(empty_texts_aux, sizeof(pb)*(1+(npar-1)/(8*sizeof(pb)))); - - bitclean(empty_texts_aux, npar-1); // marks the empty text with a 0 in the bit vector - } - else TextBuilder->InsertText(&c); // we insert the empty text just in case we index all the texts - - return 1; // success + return 1; // success } +