X-Git-Url: http://git.nguyen.vg/gitweb/?a=blobdiff_plain;f=XMLTree.cpp;h=2a1da8446a816ab17117963aea34bc785b4945ca;hb=dc02a833a150dbef202bc14aca74c51360d4a631;hp=4ddec0170a5186dc02c43e98ab59e525c89380c2;hpb=dd9992dcd5366f37820c24ed7cddf24ecbc0d549;p=SXSI%2FXMLTree.git diff --git a/XMLTree.cpp b/XMLTree.cpp index 4ddec01..2a1da84 100644 --- a/XMLTree.cpp +++ b/XMLTree.cpp @@ -67,6 +67,16 @@ inline void * umalloc(size_t size){ return dest; } +void XMLTree::print_stats() { + uint total_space = Tags->size()+sizeof(static_sequence*); + total_space += sizeof(uint*)+sizeof(uint)*(2+uint_len(tags_blen,tags_len)); + cout << "Space usage for XMLTree:" << endl + << " - tags static_sequence: " << Tags->size()+sizeof(static_sequence*) << endl + << " - tags access array: " << sizeof(uint*)+sizeof(uint)*(2+uint_len(tags_blen,tags_len)) << endl + << " ... add Diego structures ... " << endl + << " *total* " << total_space << endl; +} + // Save: saves XML tree data structure to file. void XMLTree::Save(unsigned char *filename) { @@ -101,11 +111,21 @@ void XMLTree::Save(unsigned char *filename) // stores the tags Tags->save(fp); + ufwrite(&tags_blen,sizeof(uint),1,fp); + ufwrite(&tags_len,sizeof(uint),1,fp); + ufwrite(tags_fix,sizeof(uint),uint_len(tags_blen,tags_len),fp); // stores the texts - if (!disable_tc) + if (!disable_tc) { Text->Save(fp); - + int st = CachedText.size(); + ufwrite(&st, sizeof(int),1,fp); + for (int i = 0; i< CachedText.size(); i++){ + st = CachedText.at(i).size(); + ufwrite(&st, sizeof(int),1,fp); + ufwrite(CachedText.at(i).c_str(),sizeof(char),1+CachedText.at(i).size(),fp); + }; + }; fclose(fp); } @@ -184,28 +204,57 @@ XMLTree *XMLTree::Load(unsigned char *filename, int sample_rate_text) // loads the tags XML_Tree->Tags = static_sequence::load(fp); + ufread(&XML_Tree->tags_blen,sizeof(uint),1,fp); + ufread(&XML_Tree->tags_len,sizeof(uint),1,fp); + XML_Tree->tags_fix = new uint[uint_len(XML_Tree->tags_blen,XML_Tree->tags_len)]; + ufread(XML_Tree->tags_fix,sizeof(uint),uint_len(XML_Tree->tags_blen,XML_Tree->tags_len),fp); + s_tree+=2*sizeof(uint)+sizeof(uint)*uint_len(XML_Tree->tags_blen,XML_Tree->tags_len); s_tree+= XML_Tree->Tags->size(); + /// FIXME:UGLY tests! + uint * seq = new uint[XML_Tree->tags_len]; + for(uint i=0;itags_len;i++) + seq[i] = get_field(XML_Tree->tags_fix,XML_Tree->tags_blen,i); + cout << "Tags test: " << XML_Tree->Tags->test(seq,XML_Tree->tags_len) << endl; + delete [] seq; + /// End ugly tests + s_text = ftell(fp); // loads the texts if (!XML_Tree->disable_tc){ - XML_Tree->Text = TextCollection::InitTextCollection(sample_rate_text); - XML_Tree->Text->Load(fp,sample_rate_text); + XML_Tree->Text = TextCollection::Load(fp,sample_rate_text); + int sst; + int st; + ufread(&sst, sizeof(int),1,fp); + for (int i=0;iCachedText.push_back(cppstr); + free(str); + }; + } - else + else { XML_Tree->Text = NULL; - + } s_text = ftell(fp) - s_text; + + + + fclose(fp); - std::cerr << "Tree part is " << s_tree/1024 << " Kbytes,\n" - << "with node->tagid part " << XML_Tree->Tags->size()/1024 << "Kbytes \n" + /*std::cerr << "Tree part is " << s_tree/1024 << " Kbytes,\n" + << "with node->tagid part " << XML_Tree->Tags->size()/1024+(uint_len(XML_Tree->tags_blen,XML_Tree->tags_len)*sizeof(uint))/1024 << "Kbytes \n" << "size of Tag part : " << XML_Tree->Tags->length () << " elements\n" << "sizof(unsigned int)* " << XML_Tree->Tags->length () << " = " << sizeof(unsigned int) * XML_Tree->Tags->length () / 1024 << " Kbytes\n" << "Tag part is " << s_tags/1024 << " Kbytes,\n" - << "Text collection is " << s_text/1024 << " Kbytes \n"; + << "Text collection is " << s_text/1024 << " Kbytes \n";*/ + XML_Tree->print_stats(); return XML_Tree; } @@ -234,7 +283,9 @@ XMLTree::~XMLTree() Tags = NULL; //Text->~TextCollection(); - delete Text; + delete TextBuilder; + TextBuilder = NULL; + delete Text; Text = NULL; initialized = false; @@ -306,7 +357,9 @@ bool XMLTree::IsChild(treeNode x, treeNode y) if (!is_ancestor(Par, x, y)) return false; return depth(Par, x) == (depth(Par, y) + 1); } - +bool XMLTree::IsFirstChild(treeNode x){ + return ((x != NULLT)&&(x==Root() || prev_sibling(Par,x) == NULLT)); +} // NumChildren(x): number of children of node x. Constant time with the data structure // of Sadakane. int XMLTree::NumChildren(treeNode x) @@ -372,7 +425,7 @@ TagType XMLTree::Tag(treeNode x) exit(1); } - return Tags->access(node2tagpos(x)); + return get_field(tags_fix,tags_blen,node2tagpos(x)); //Tags->access(node2tagpos(x)); } // DocIds(x): returns the range of text identifiers that descend from node x. @@ -385,6 +438,14 @@ range XMLTree::DocIds(treeNode x) } range r; + if (x == NULLT) + { + r.min = NULLT; + r.max = NULLT; + return r; + }; + + if (indexing_empty_texts) { // faster, no rank needed r.min = x; r.max = x+2*subtree_size(Par, x)-2; @@ -440,6 +501,14 @@ treeNode XMLTree::FirstChild(treeNode x) return first_child(Par, x); } +treeNode XMLTree::LastChild(treeNode x) +{ + if (x == Root() || isleaf(Par,x) || x == NULLT) + return x; + else + return find_open(Par,find_close(Par,parent(Par,x))-1); +} + // NextSibling(x): returns the next sibling of node x, assuming it exists. treeNode XMLTree::NextSibling(treeNode x) { @@ -447,9 +516,9 @@ treeNode XMLTree::NextSibling(treeNode x) fprintf(stderr, "Error: data structure has not been constructed properly\n"); exit(1); } - if (x == Root()) + if (x == Root() || x==NULLT) return NULLT; - + return next_sibling(Par, x); } @@ -479,7 +548,7 @@ treeNode XMLTree::TaggedChild(treeNode x, int i, TagType tag) child = first_child(Par, x); // starts at first child of node x if (child==(treeNode)-1) return NULLT; // node x is a leaf, there is no such child while (child!=(treeNode)-1) { - if (Tags->access(node2tagpos(child)) == tag) { // current child is labeled with tag of interest + if (get_field(tags_fix,tags_blen,node2tagpos(child)) /*Tags->access(node2tagpos(child))*/ == tag) { // current child is labeled with tag of interest i--; if (i==0) return child; // we have seen i children of x tagged tag, this is the one we are looking for } @@ -497,38 +566,166 @@ treeNode XMLTree::TaggedDesc(treeNode x, TagType tag) exit(1); } - int r, s; + //int r, s; treeNode y; if (isleaf(Par,x)) return NULLT; - r = (int) Tags->rank(tag, node2tagpos(x)); - s = (int) Tags->select(tag, r+1); + int s = (int) Tags->select_next(tag,node2tagpos(x)); + /*r = (int) Tags->rank(tag, node2tagpos(x)); + s = (int) Tags->select(tag, r+1);*/ if (s == -1) return NULLT; // there is no such node y = tagpos2node(s); // transforms the tag position into a node position if (!is_ancestor(Par, x, y)) return NULLT; // the next node tagged tag (in preorder) is not within the subtree of x. else return y; } +treeNode XMLTree::TaggedDescOnly(treeNode x,TagType *desctags, unsigned int dtlen) +{ + + treeNode res,y; + if (isleaf(Par,x)) + return NULLT; + + res=NULLT; + for (unsigned int i = 0; i < dtlen; i ++ ) + { + y = TaggedDesc(x,desctags[i]); + res = (res == NULLT) || (( res != NULLT) && (y =! NULLT) && y < res) ? y : res; + + }; + + return res; + +} + + +treeNode XMLTree::TaggedBelow(treeNode x, TagType *childtags, unsigned int ctlen, + TagType *desctags, unsigned int dtlen) +{ + treeNode fs,y,res; + TagType tag; + + if (isleaf(Par,x)) + return NULLT; + + res = NULLT; + fs = first_child(Par,x); + while (fs != NULLT) { + tag = get_field(tags_fix,tags_blen,node2tagpos(fs)); + + /* Check for first_child */ + for (unsigned int i = 0; i < ctlen; i++) { + if (childtags[i] == tag) + return fs; + }; + + for (unsigned int i = 0; i < dtlen; i++) + if (desctags[i] == tag) + return fs; + + /* check in the descendants */ + res = NULLT; + for (unsigned int i = 0; i < dtlen; i ++ ){ + /* maybe inline by hand */ + y = TaggedDesc(fs,desctags[i]); + res = (res==NULLT || (y != NULLT) &&(y < res)) ? y : res; + }; + if (res != NULLT) + return res; + + fs = next_sibling(Par,fs); + }; + return res; + +} +treeNode XMLTree::TaggedFollOnly(treeNode x,TagType *folltags, unsigned int ftlen,treeNode root) +{ + + treeNode res,y,lim; + lim = find_close(Par,root); + res=NULLT; + for (unsigned int i = 0; i < ftlen; i ++ ) + { + y = TaggedFoll(x,folltags[i]); + res = (res == NULLT) || (( res != NULLT) && (y =! NULLT) && y < res) ? y : res; + + }; + + return res < lim ? res : NULLT; + +} + +treeNode XMLTree::TaggedDescOrFollOnly(treeNode x,TagType *folltags, unsigned int ftlen,treeNode root) +{ + + treeNode res,y,lim; + //int r,s; + lim = find_close(Par,root); + res=NULLT; + for (unsigned int i = 0; i < ftlen; i ++ ) + { + + int s = (int) Tags->select_next(folltags[i],node2tagpos(x)); + /*r = (int) Tags->rank(folltags[i], node2tagpos(x)); + s = (int) Tags->select(folltags[i], r+1);*/ + if (s == -1) + y = NULLT; // there is no such node + else { + y = tagpos2node(s); + if (y >= lim) + y = NULLT; + }; + res = (res == NULLT) || (( res != NULLT) && (y =! NULLT) && y < res) ? y : res; + + }; + + return res < lim ? res : NULLT; + +} + + // TaggedNext(x,tag): returns the first node tagged tag with larger preorder than x // Returns NULLT if there is none. -treeNode XMLTree::TaggedNext(treeNode x, TagType tag) +treeNode XMLTree::TaggedNext(treeNode x, TagType *childtags, unsigned int ctlen, + TagType *folltags, unsigned int flen,treeNode root) { - if (!finished) { - fprintf(stderr, "Error: data structure has not been constructed properly\n"); - exit(1); - } + treeNode y,old_y,lim,res; + TagType tag; + if (x == NULLT || x == Root()) + return NULLT; - int r, s; - treeNode y; - if (x==NULLT) - return NULLT; - r = (int) Tags->rank(tag, node2tagpos(x)); - s = (int) Tags->select(tag, r+1); - if (s == -1) return NULLT; // there is no such node - y = tagpos2node(s); // transforms the tag position into a node position - return (y<=x ? NULLT : y); + lim = find_close(Par,root); + + res = NULLT; + + y = next_sibling(Par,x); + while (y != NULLT) { + tag = get_field(tags_fix,tags_blen,node2tagpos(y)); + for(unsigned int i = 0; i < ctlen;i++) + if (childtags[i] == tag) + return y; + + for(unsigned int i = 0; i < flen;i++) + if (folltags[i] == tag) + return y; + + res = TaggedBelow(y,NULL,0,folltags,flen); + if (res != NULLT) + return res; + + y = next_sibling(Par,y); + }; + //Found nothing in the following sibling of x. + res = NULLT; + for(unsigned int i = 0; i < flen;i++){ + y = TaggedFoll(x,folltags[i]); + res = (y!= x && (res == NULLT || (y != NULLT && y < res)))? y : res; + }; + + return res < lim ? res : NULLT; + } @@ -557,25 +754,91 @@ treeNode XMLTree::TaggedPrec(treeNode x, TagType tag) return NULLT; // there is no such node } + // TaggedFoll(x,tag): returns the first node tagged tag with larger preorder than x and not in // the subtree of x. Returns NULLT if there is none. -treeNode XMLTree::TaggedFoll(treeNode x, TagType tag) +treeNode XMLTree::TaggedFoll(treeNode x, TagType tag) { if (!finished) { fprintf(stderr, "Error: data structure has not been constructed properly\n"); exit(1); } - int r, s; - if (x ==NULLT || x == Root()|| (next_sibling(Par,x) == -1 )) + //int r, s; + if (x ==NULLT || x == Root()) + return NULLT; + + int s = (int) Tags->select_next(tag,find_close(Par,x)); + /*r = (int) Tags->rank(tag, find_close(Par, x)); + s = (int) Tags->select(tag, r+1); // select returns -1 in case that there is no r+1-th tag. */ + if (s==-1) return NULLT; + else return tagpos2node(s); + } + +// TaggedFoll(x,tag): returns the first node tagged tag with larger preorder than x and not in +// the subtree of x. Returns NULLT if there is none. +treeNode XMLTree::TaggedFollBelow(treeNode x, TagType tag, treeNode root) + { + + //int r, s; + int lim = node2tagpos(find_close(Par,root)); + if (x ==NULLT || x == Root()) + return NULLT; + + int s = (int) Tags->select_next(tag,find_close(Par,x)); + /*r = (int) Tags->rank(tag,find_close(Par,x)); + s = (int) Tags->select(tag, r+1); // select returns -1 in case that there is no r+1-th tag.*/ + if (s==-1 || s >= lim) return NULLT; + else + return tagpos2node(s); + } + - r = (int) Tags->rank(tag, node2tagpos(next_sibling(Par, x))-1); - s = (int) Tags->select(tag, r+1); // select returns -1 in case that there is no r+1-th tag. +// TaggedFollowingSibling(x,tag): returns the first node tagged tag with larger preorder than x and not in +// the subtree of x. Returns NULLT if there is none. +treeNode XMLTree::TaggedFollowingSibling(treeNode x, TagType tag) + { + if (!finished) { + fprintf(stderr, "Error: data structure has not been constructed properly\n"); + exit(1); + } + + //int r, s; + treeNode ns = next_sibling(Par,x); + + if (x == NULLT || x == Root() || ns == -1) + return NULLT; + + int s = (int) Tags->select_next(tag,node2tagpos(ns)-1); + /*r = (int) Tags->rank(tag, node2tagpos(ns)-1); + s = (int) Tags->select(tag, r+1); // select returns -1 in case that there is no r+1-th tag.*/ if (s==-1) return NULLT; else return tagpos2node(s); } + +// TaggedAncestor(x, tag): returns the closest ancestor of x tagged tag. Return +// NULLT is there is none. +treeNode XMLTree::TaggedAncestor(treeNode x, TagType tag) + { + if (!finished) { + fprintf(stderr, "Error: data structure has not been constructed properly\n"); + exit(1); + } + + if (x == NULLT || x == Root()) + return NULLT; + + treeNode s = parent(Par, x), r = Root(); + while (s != r) { + if (get_field(tags_fix,tags_blen,node2tagpos(s)) /*Tags->access(node2tagpos(s))*/ == tag) return s; + s = parent(Par, s); + } + return NULLT; + } + + // PrevText(x): returns the document identifier of the text to the left // of node x, or NULLT if x is the root node or the text is empty. // Assumes Doc ids start from 0. @@ -699,6 +962,29 @@ treeNode XMLTree::ParentNode(DocID d) return (treeNode)s; } +treeNode XMLTree::PrevNode(DocID d) + { + if (!finished) { + fprintf(stderr, "Error: data structure has not been constructed properly\n"); + exit(1); + } + + if (d == NULLT) + return NULLT; + + int s; + + if (indexing_empty_texts) s = d; + else s = EBVector->select1(d+1); + if (s == -1) + return NULLT; + + if (inspect(Par,s) == CP) // is a closing parenthesis + return find_open(Par, s); + else // is an opening parenthesis + return NULLT; + + } // OpenDocument(empty_texts): it starts the construction of the data structure for @@ -711,7 +997,7 @@ int XMLTree::OpenDocument(bool empty_texts, int sample_rate_text,bool dtc) found_attributes = false; npar = 0; parArraySize = 1; - ntagnames = 2; + ntagnames = 4; disable_tc = dtc; indexing_empty_texts = empty_texts; @@ -720,7 +1006,7 @@ int XMLTree::OpenDocument(bool empty_texts, int sample_rate_text,bool dtc) tags_aux = (TagType *) umalloc(sizeof(TagType)); - TagName = (unsigned char **) umalloc(2*sizeof(unsigned char*)); + TagName = (unsigned char **) umalloc(4*sizeof(unsigned char*)); TagName[0] = (unsigned char *) umalloc(4*sizeof(unsigned char)); @@ -729,14 +1015,25 @@ int XMLTree::OpenDocument(bool empty_texts, int sample_rate_text,bool dtc) TagName[1] = (unsigned char *) umalloc(4*sizeof(unsigned char)); strcpy((char *) TagName[1], "<$>"); + + //OJO need to put these in the table too. + TagName[2] = (unsigned char *) umalloc(5*sizeof(unsigned char)); + + strcpy((char *) TagName[2], "/<@>"); + + TagName[3] = (unsigned char *) umalloc(5*sizeof(unsigned char)); + + strcpy((char *) TagName[3], "/<$>"); if (!indexing_empty_texts) empty_texts_aux = (unsigned int *)umalloc(sizeof(unsigned int)); - - - Text = TextCollection::InitTextCollection((unsigned)sample_rate_text); + if (disable_tc) + TextBuilder = 0; + else + TextBuilder = new TextCollectionBuilder((unsigned)sample_rate_text); + Text = 0; return 1; // indicates success in the initialization of the data structure } @@ -759,24 +1056,51 @@ int XMLTree::CloseDocument() Par = (bp *)umalloc(sizeof(bp)); bp_construct(Par, npar, par_aux, OPT_DEGREE|0); // creates structure for tags - static_bitsequence_builder * bmb = new static_bitsequence_builder_brw32(20); - static_permutation_builder * pmb = new static_permutation_builder_mrrr(PERM_SAMPLE, bmb); - static_sequence_builder * ssb = new static_sequence_builder_gmr_chunk(bmb, pmb); - // If we found an attribute then "<@>" is present in the tree // if we didn't then it is not. "<$>" is never present in the tree - int ntagsize = found_attributes ? 2*ntagnames-1 : 2*ntagnames - 2; - - Tags = new static_sequence_gmr((uint *) tags_aux, (uint) npar-1,ntagsize, bmb, ssb); + uint max_tag = 0; + for(uint i=0;i<(uint)npar-1;i++) + max_tag = max(max_tag,tags_aux[i]); + //max_tag++; + //tags_aux = (TagType *) urealloc(tags_aux, sizeof(TagType)*(npar + 1)); + //tags_aux[npar++] = max_tag; + //int ntagsize = found_attributes ? 2*ntagnames-1 : 2*ntagnames - 2; + int ntagsize = 2*ntagnames + 2; + + //static_bitsequence_builder * bmb = new static_bitsequence_builder_brw32(20); + //static_permutation_builder * pmb = new static_permutation_builder_mrrr(PERM_SAMPLE, bmb); + //static_sequence_builder * ssb = new static_sequence_builder_gmr_chunk(bmb, pmb); + static_bitsequence_builder * bmb = new static_bitsequence_builder_sdarray(); + alphabet_mapper *am = new alphabet_mapper_none(); + //wt_coder * wc = new wt_coder_huff((uint*)tags_aux,npar,am); + //Tags = new static_sequence_wvtree((uint*)tags_aux,npar,wc ,bmb, am); + //Tags = new static_sequence_gmr((uint *) tags_aux, (uint) npar,ntagsize, bmb, ssb); + Tags = new static_sequence_bs((uint*)tags_aux,npar,am,bmb); + + cout << "Tags test: " << Tags->test((uint*)tags_aux,npar) << endl; + + tags_blen = bits(max_tag); + tags_len = (uint)npar; + tags_fix = new uint[uint_len(tags_blen,tags_len)]; + for(uint i=0;i<(uint)npar;i++) + set_field(tags_fix,tags_blen,i,tags_aux[i]); delete bmb; - delete pmb; - delete ssb; + //delete pmb; + //delete ssb; + + // makes the text collection static if (!disable_tc) - Text->MakeStatic(); - + { + assert(Text == 0); + assert(TextBuilder != 0); + Text = TextBuilder->InitTextCollection(); + delete TextBuilder; + TextBuilder = 0; + } + // creates the data structure marking the non-empty texts (just in the case it is necessary) if (!indexing_empty_texts) { EBVector = new static_bitsequence_rrr02((uint *)empty_texts_aux,(ulong)npar,(uint)32); @@ -790,6 +1114,7 @@ int XMLTree::CloseDocument() tags_aux = NULL; finished = true; + print_stats(); return 1; // indicates success in the inicialization } @@ -842,7 +1167,7 @@ int XMLTree::NewOpenTag(unsigned char *tagname) tags_aux[npar] = i; // inserts the new tag id within the preorder sequence of tags npar++; - + return 1; } @@ -866,17 +1191,16 @@ int XMLTree::NewClosingTag(unsigned char *tagname) parArraySize *= 2; } - setbit(par_aux,npar,CP); // marks a new closing opening parenthesis + setbit(par_aux,npar,CP); // marks a new closing parenthesis // transforms the tagname into a tag identifier. If the tag is new, we insert // it in the table. for (i=0; iInsertText(s); + TextBuilder->InsertText(s); + string cpps = (char*) s; + CachedText.push_back(cpps); return 1; // success } @@ -937,7 +1263,7 @@ int XMLTree::NewEmptyText() bitclean(empty_texts_aux, npar-1); // marks the empty text with a 0 in the bit vector } - else Text->InsertText(&c); // we insert the empty text just in case we index all the texts + else TextBuilder->InsertText(&c); // we insert the empty text just in case we index all the texts return 1; // success } @@ -951,7 +1277,7 @@ TagType XMLTree::GetTagId(unsigned char *tagname) // this should be changed for more efficient processing for (i=0; i= ntagnames) return NULL; // invalid tag identifier s = (unsigned char *)umalloc((strlen((const char *)TagName[tagid])+1)*sizeof(unsigned char)); strcpy((char *)s, (const char *)TagName[tagid]); @@ -973,6 +1299,7 @@ unsigned char *XMLTree::GetTagName(TagType tagid) const unsigned char *XMLTree::GetTagNameByRef(TagType tagid) { + if(tagid==(uint)-1) return NULL; if (tagid >= ntagnames) return NULL; // invalid tag identifier return ((const unsigned char*) TagName[tagid]); }