-\r
/******************************************************************************\r
* Copyright (C) 2008 by Diego Arroyuelo *\r
* Interface for the in-memory XQuery/XPath engine *\r
* along with this program; if not, write to the *\r
* Free Software Foundation, Inc., *\r
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *\r
- ******************************************************************************/ \r
+ ******************************************************************************/\r
\r
#ifndef XMLTREE_H_\r
#define XMLTREE_H_\r
-#include "TextCollection/TextCollection.h"\r
-#include <stdio.h>\r
-#include <stdlib.h>\r
-#include <cstring>\r
+\r
+\r
+#include <unordered_set>\r
+#include <unordered_map>\r
+#include <sstream>\r
+#include "TextCollection/TextCollectionBuilder.h"\r
+\r
+#undef W\r
+#undef WW\r
+#undef Wminusone\r
+\r
#include "bp.h"\r
+#include <libcds/includes/basics.h>\r
#include <static_bitsequence.h>\r
#include <alphabet_mapper.h>\r
#include <static_sequence.h>\r
using SXSI::TextCollection;\r
+using SXSI::TextCollectionBuilder;\r
\r
\r
// this constant is used to efficiently compute the child operation in the tree\r
\r
#define NULLT -1\r
\r
- // sets bit p in e\r
-#define bitset(e,p) ((e)[(p)/W] |= (1<<((p)%W)))\r
- // cleans bit p in e\r
-#define bitclean(e,p) ((e)[(p)/W] &= ~(1<<((p)%W)))\r
+#define PERM_SAMPLE 10\r
\r
\r
typedef int treeNode;\r
int max;\r
} range;\r
\r
+// Encoding of the XML Document :\r
+// The following TAGs and IDs are fixed, "" is the tag of the root.\r
+// a TextNode is represented by a leaf <<$>></<$>> The DocId in the TextCollection\r
+// of that leaf is kept in a bit sequence.\r
+// a TextNode below an attribute is likewise represented by a leaf <<@$>><</@$>>\r
+// An element <e a1="v1" a2="v2" ... an="vn" > ...</e> the representation is:\r
+// <e><<@>> <<@>a1> <<$@>>DocID(v1)</<$@>></<@>a1> ... </<@>> .... </e>\r
+// Hence the attributes (if any) are always below the first child of their element,\r
+// as the children of a fake node <@>.\r
+\r
+\r
+#define DOCUMENT_OPEN_TAG ""\r
+#define DOCUMENT_TAG_ID 0\r
+#define ATTRIBUTE_OPEN_TAG "<@>"\r
+#define ATTRIBUTE_TAG_ID 1\r
+#define PCDATA_OPEN_TAG "<$>"\r
+#define PCDATA_TAG_ID 2\r
+#define ATTRIBUTE_DATA_OPEN_TAG "<@$>"\r
+#define ATTRIBUTE_DATA_TAG_ID 3\r
+#define CLOSING_TAG "</>"\r
+#define CLOSING_TAG_ID 4\r
+#define DOCUMENT_CLOSE_TAG "/"\r
+#define ATTRIBUTE_CLOSE_TAG "/<@>"\r
+#define PCDATA_CLOSE_TAG "/<$>"\r
+#define ATTRIBUTE_DATA_CLOSE_TAG "/<@$>"\r
+\r
+\r
+typedef std::unordered_set<int> TagIdSet;\r
+typedef std::unordered_map<std::string,int> TagIdMap;\r
+typedef TagIdMap::const_iterator TagIdMapIT;\r
+\r
+#define REGISTER_TAG(v,h,t) do { (h)->insert(std::make_pair((t),(v)->size()));\\r
+ (v)->push_back(t); } while (false)\r
+\r
+\r
+// returns NULLT if the test is true\r
+#define NULLT_IF(x) do { if (x) return NULLT; } while (0)\r
+\r
+\r
+\r
+\r
+\r
+\r
+class XMLTreeBuilder;\r
\r
class XMLTree {\r
+\r
+ // Only the builder can access the constructor\r
+ friend class XMLTreeBuilder;\r
+\r
+ private:\r
/** Balanced parentheses representation of the tree */\r
bp *Par;\r
\r
/** Mapping from tag identifer to tag name */ \r
- unsigned char **TagName;\r
+ std::vector<std::string> *TagName;\r
+ TagIdMap * tIdMap;\r
\r
- /** boolean flag indicating whether we are indexing empty texts or not */\r
- bool indexing_empty_texts; \r
- \r
/** Bit vector indicating with a 1 the positions of the non-empty texts. */\r
- static_bitsequence_rrr02 *EBVector; \r
+ static_bitsequence *EBVector; \r
\r
/** Tag sequence represented with a data structure for rank and select */\r
- static_sequence_wvtree *Tags;\r
+ static_sequence *Tags;\r
+ uint * tags_fix;\r
+ uint tags_blen, tags_len;\r
\r
/** The texts in the XML document */\r
TextCollection *Text;\r
- \r
- /** Flag indicating whether the whole data structure has been constructed or \r
- * not. If the value is true, you cannot add more texts, nodes, etc. */\r
- bool finished;\r
-\r
- /** Flag indicating whether the construction of the data structure has been\r
- * initialized or not (by calling method OpenDocument()). If this is true,\r
- * you cannot insert new tags or texts. */\r
- bool initialized;\r
- \r
- /* the following components are used for construction purposes */\r
- pb *par_aux;\r
- TagType *tags_aux;\r
- int npar;\r
- int ntagnames;\r
- unsigned int *empty_texts_aux;\r
- \r
-public:\r
\r
- /** Data structure constructor */\r
- XMLTree() {finished = false; initialized = false;}; \r
- \r
+ // Allows to disable the TextCollection for benchmarkin purposes\r
+ bool disable_tc;\r
+ \r
+ FILE* stream;\r
+ int stream_fd; \r
+ std::string * buffer;\r
+ void myfputs(const char* s, FILE * fp){\r
+ buffer->append(s);\r
+ if (buffer->size() >= 100000){\r
+ fputs(buffer->c_str(),fp);\r
+ buffer->clear();\r
+ };\r
+\r
+ }\r
+ void myfputc(const char c, FILE*fp){\r
+ buffer->append(1,c);\r
+ if (buffer->size() >= 100000){\r
+ fputs(buffer->c_str(),fp);\r
+ buffer->clear();\r
+ };\r
+ }\r
+ void mybufferflush(FILE* fp){\r
+ fputs(buffer->c_str(), fp);\r
+ buffer->clear();\r
+ }\r
+\r
+ size_t myfprintf(const char* s, FILE * fp){\r
+ if (s == NULL)\r
+ return 0;\r
+ size_t i = buffer->size();\r
+ buffer->append(s);\r
+ size_t j = buffer->size();\r
+ if (buffer->size() >= 100000){\r
+ fputs(buffer->c_str(),fp);\r
+ buffer->clear();\r
+ };\r
+ return (j-i);\r
+ }\r
+\r
+ void PrintNode(treeNode n, int fd);\r
+ /** Data structure constructors */\r
+ XMLTree(){ buffer = 0;};\r
+\r
+ // non const pointer are freed by this method.\r
+ XMLTree( pb * const par, uint npar, std::vector<std::string> * const TN, TagIdMap * const tim, uint *empty_texts_bmp, TagType *tags,\r
+ TextCollection * const TC, bool dis_tc);\r
+\r
+public: \r
/** Data structure destructor */\r
~XMLTree();\r
\r
/** root(): returns the tree root. */\r
- treeNode Root();\r
- \r
+ treeNode Root() { return 0; }\r
+\r
+ /** Size() : Number of parenthesis */\r
+ unsigned int Size(){\r
+ return tags_len/2;\r
+ }\r
+\r
/** SubtreeSize(x): the number of nodes (and attributes) in the subtree of \r
* node x. */\r
int SubtreeSize(treeNode x);\r
- \r
+ \r
/** SubtreeTags(x,tag): the number of occurrences of tag within the subtree \r
* of node x. */\r
int SubtreeTags(treeNode x, TagType tag);\r
\r
+ /** SubtreeElements(x) of element nodes in the subtree of x\r
+ */\r
+ int SubtreeElements(treeNode x);\r
+\r
/** IsLeaf(x): returns whether node x is leaf or not. In the succinct \r
* representation this is just a bit inspection. */\r
+\r
bool IsLeaf(treeNode x);\r
- \r
+\r
/** IsAncestor(x,y): returns whether node x is ancestor of node y. */\r
+\r
bool IsAncestor(treeNode x, treeNode y);\r
\r
/** IsChild(x,y): returns whether node x is parent of node y. */\r
bool IsChild(treeNode x, treeNode y);\r
- \r
+\r
+ /** IsFirstChild(x): returns whether node x is the first child of its parent. */\r
+ /* OCAML */\r
+ bool IsFirstChild(treeNode x);\r
+ \r
/** NumChildren(x): number of children of node x. Constant time with the \r
* data structure of Sadakane. */\r
int NumChildren(treeNode x);\r
- \r
+\r
/** ChildNumber(x): returns i if node x is the i-th children of its \r
* parent. */\r
- inline int ChildNumber(treeNode x);\r
+ int ChildNumber(treeNode x);\r
\r
/** Depth(x): depth of node x, a simple binary rank on the parentheses \r
* sequence. */\r
/** Postorder(x): returns the postorder number of node x, just regarding \r
* tree nodes (and not texts). */\r
int Postorder(treeNode x);\r
- \r
+ \r
/** Tag(x): returns the tag identifier of node x. */\r
- TagType Tag(treeNode x);\r
- \r
+ TagType Tag(treeNode x) {\r
+ if (tags_blen == 8)\r
+ return (TagType) (((uchar*)tags_fix)[(int) x]);\r
+ else\r
+ return (TagType) get_field(tags_fix,tags_blen, (int) x);\r
+ }\r
+\r
/** DocIds(x): returns the range (i.e., a pair of integers) of document \r
* identifiers that descend from node x. */\r
range DocIds(treeNode x);\r
- \r
+\r
/** Parent(x): returns the parent node of node x. */\r
treeNode Parent(treeNode x);\r
+ /* Assumes x is neither 0 nor -1 */\r
\r
/** Child(x,i): returns the i-th child of node x, assuming it exists. */ \r
treeNode Child(treeNode x, int i);\r
- \r
- /** FirstChild(x): returns the first child of node x, assuming it exists. \r
- * Very fast in BP. */\r
+\r
+ /** FirstChild(x): returns the first child of node x, or NULLT if the node is a leaf\r
+ */\r
treeNode FirstChild(treeNode x);\r
+\r
+ /** FirstElement(x): returns the first non text, non attribute child of node x, or NULLT\r
+ * if none.\r
+ */\r
+ treeNode FirstElement(treeNode x);\r
+\r
+ /** LastChild(x): returns the last child of node x. */\r
+ treeNode LastChild(treeNode x);\r
\r
- /** NextSibling(x): returns the next sibling of node x, assuming it \r
+ /** NextSibling(x): returns the next sibling of node x, or NULLT if none \r
* exists. */\r
+\r
treeNode NextSibling(treeNode x);\r
- \r
+\r
+ /** NextElement(x): returns the first non text, non attribute sibling of node x, or NULLT\r
+ * if none.\r
+ */\r
+ treeNode NextElement(treeNode x);\r
+\r
/** PrevSibling(x): returns the previous sibling of node x, assuming it \r
* exists. */\r
+\r
treeNode PrevSibling(treeNode x);\r
\r
- /** TaggedChild(x,i,tag): returns the i-th child of node x tagged tag, or \r
+ /** TaggedChild(x,tag): returns the first child of node x tagged tag, or \r
* NULLT if there is none. Because of the balanced-parentheses representation \r
* of the tree, this operation is not supported efficiently, just iterating \r
* among the children of node x until finding the desired child. */\r
- treeNode TaggedChild(treeNode x, int i, TagType tag);\r
+ treeNode TaggedChild(treeNode x, TagType tag);\r
\r
+ treeNode SelectChild(treeNode x, TagIdSet * tags);\r
+\r
+ /** TaggedFollowingSibling(x,tag): returns the first sibling of node x tagged tag, or \r
+ * NULLT if there is none. */\r
+ treeNode TaggedFollowingSibling(treeNode x, TagType tag);\r
+ \r
+ treeNode SelectFollowingSibling(treeNode x, TagIdSet * tags);\r
+\r
/** TaggedDesc(x,tag): returns the first node tagged tag with larger \r
* preorder than x and within the subtree of x. Returns NULT if there \r
* is none. */\r
- treeNode TaggedDesc(treeNode x, TagType tag);\r
+ treeNode TaggedDescendant(treeNode x, TagType tag);\r
+\r
+ treeNode SelectDescendant(treeNode x, TagIdSet * tags);\r
\r
/** TaggedPrec(x,tag): returns the first node tagged tag with smaller \r
* preorder than x and not an ancestor of x. Returns NULLT if there \r
* is none. */\r
- treeNode TaggedPrec(treeNode x, TagType tag);\r
+ treeNode TaggedPreceding(treeNode x, TagType tag);\r
\r
/** TaggedFoll(x,tag): returns the first node tagged tag with larger \r
* preorder than x and not in the subtree of x. Returns NULLT if there \r
* is none. */\r
- treeNode TaggedFoll(treeNode x, TagType tag);\r
- \r
+ treeNode TaggedFollowing(treeNode x, TagType tag);\r
+\r
+ treeNode TaggedFollowingBelow(treeNode x, TagType tag,treeNode ancestor); \r
+\r
+ treeNode SelectFollowingBelow(treeNode x, TagIdSet * tags, treeNode ancestor);\r
+\r
+ treeNode TaggedFollowingBefore(treeNode x, TagType tag,treeNode closing);\r
+\r
+ treeNode SelectFollowingBefore(treeNode x, TagIdSet * tags, treeNode closing);\r
+\r
+ /** TaggedAncestor(x, tag): returns the closest ancestor of x tagged \r
+ * tag. Return NULLT is there is none. */\r
+ treeNode TaggedAncestor(treeNode x, TagType tag);\r
+ \r
/** PrevText(x): returns the document identifier of the text to the left of \r
* node x, or NULLT if x is the root node. */\r
DocID PrevText(treeNode x);\r
/** MyText(x): returns the document identifier of the text below node x, or \r
* NULLT if x is not a leaf node. */\r
DocID MyText(treeNode x);\r
- \r
+ DocID MyTextUnsafe(treeNode x);\r
+\r
/** TextXMLId(d): returns the preorder of document with identifier d in the \r
* tree consisting of all tree nodes and all text nodes. */\r
int TextXMLId(DocID d);\r
\r
/** ParentNode(d): returns the parent node of document identifier d. */\r
treeNode ParentNode(DocID d);\r
-\r
- /** OpenDocument(empty_texts,sample_rate_text): initilizes the construction\r
- * of the data structure for the XML document. Parameter empty_texts \r
- * indicates whether we index empty texts in document or not. Parameter \r
- * sample_rate_text indicates the sampling rate for the text searching data\r
- * structures (small values get faster searching but a bigger space \r
- * requirement). Returns a non-zero value upon success, NULLT in case of \r
- * error. */\r
- int OpenDocument(bool empty_texts, int sample_rate_text);\r
-\r
- /** CloseDocument(): finishes the construction of the data structure for \r
- * the XML document. Tree and tags are represented in the final form, \r
- * dynamic data structures are made static, and the flag "finished" is set \r
- * to true. After that, the data structure can be queried. */\r
- int CloseDocument();\r
-\r
- /** NewOpenTag(tagname): indicates the event of finding a new opening tag \r
- * in the document. Tag name is given. Returns a non-zero value upon \r
- * success, and returns NULLT in case of error. */\r
- int NewOpenTag(unsigned char *tagname);\r
\r
- /** NewClosingTag(tagname): indicates the event of finding a new closing tag\r
- * in the document. Tag name is given. Returns a non-zero value upon \r
- * success, and returns NULLT in case of error. */\r
- int NewClosingTag(unsigned char *tagname);\r
- \r
- /** NewText(s): indicates the event of finding a new (non-empty) text s in \r
- * the document. The new text is inserted within the text collection. \r
- * Returns a non-zero value upon success, NULLT in case of error. */\r
- int NewText(unsigned char *s);\r
-\r
- /** NewEmptyText(): indicates the event of finding a new empty text in the \r
- * document. In case of indexing empty and non-empty texts, we insert the \r
- * empty texts into the text collection. In case of indexing only non-empty\r
- * texts, it just indicates an empty text in the bit vector of empty texts. \r
- * Returns a non-zero value upon success, NULLT in case of error. */\r
- int NewEmptyText();\r
+ treeNode PrevNode(DocID d);\r
\r
/** GetTagId(tagname): returns the tag identifier corresponding to a given \r
* tag name. Returns NULLT in case that the tag name does not exists. */\r
* Returns NULL in case that the tag identifier is not valid.*/\r
unsigned char *GetTagName(TagType tagid);\r
\r
+ /** GetTagName(tagid): returns the tag name of a given tag identifier. \r
+ * The result is just a reference and should not be freed by the caller.\r
+ */\r
+ const unsigned char *GetTagNameByRef(TagType tagid);\r
+\r
+ /** RegisterTag adds a new tag to the tag collection this is needed\r
+ * if the query contains a tag which is not in the document, we need\r
+ * to give this new tag a fresh id and store it somewhere. A logical\r
+ * choice is here.\r
+ * We might want to use a hashtable instead of an array though.\r
+ */\r
+ TagType RegisterTag(unsigned char *tagname);\r
+\r
+ bool EmptyText(DocID i) {\r
+ return Text->EmptyText(i);\r
+ }\r
+\r
/** Prefix(s): search for texts prefixed by string s. */\r
TextCollection::document_result Prefix(uchar const *s) {\r
return Text->Prefix(s);\r
}\r
\r
/** Equal(s): search for texts equal to string s. */\r
- TextCollection::document_result Equal(uchar const *s) {\r
+ TextCollection::document_result Equals(uchar const *s) {\r
return Text->Equal(s);\r
}\r
\r
bool IsLessThan(uchar const *s) {\r
return Text->IsLessThan(s);\r
}\r
+ \r
+ /** Count(s): Global counting */\r
+ unsigned Count(uchar const *s) {\r
+ return Text->Count(s);\r
+ }\r
\r
/** CountPrefix(s): counting version of Prefix(s). */\r
unsigned CountPrefix(uchar const *s) {\r
\r
/** CountLessThan(s): counting version of LessThan(s). */\r
unsigned CountLessThan(uchar const *s) {\r
- return CountLessThan(s);\r
+ return Text->CountLessThan(s);\r
}\r
\r
/** GetText(d): returns the text corresponding to document with\r
* id d. */\r
uchar* GetText(DocID d) {\r
- return Text->GetText(d);\r
+ \r
+ uchar * s = Text->GetText(d);\r
+ return (s[0] == 1 ? (s+1) : s);\r
+ }\r
+\r
+ /** GetText(i, j): returns the texts corresponding to documents with\r
+ * ids i, i+1, ..., j. Texts are separated by '\0' character. */\r
+ // uchar* GetText(DocID i, DocID j) {\r
+ // uchar * s = Text->GetText(i, j);\r
+ // return (s[0] == 1 ? (uchar*)"" : s);\r
+ //}\r
+\r
+ TextCollection *getTextCollection() {\r
+ return Text;\r
}\r
\r
/** Save: saves XML tree data structure to file. */\r
- void Save(unsigned char *filename);\r
+ void Save(int fd);\r
\r
/** Load: loads XML tree data structure from file. sample_rate_text \r
* indicates the sample rate for the text search data structure. */\r
- static XMLTree *Load(unsigned char *filename, int sample_rate_text); \r
+ static XMLTree *Load(int fd,bool load_tc, int sample_factor); \r
+\r
+ void insertTag(TagType tag, uint position);\r
+ \r
+ void print_stats();\r
+\r
+ \r
+ /** Parenthesis functions */\r
+ treeNode Closing(treeNode x);\r
+\r
+ bool IsOpen(treeNode x);\r
+\r
+\r
+ /** Print procedure */\r
+ void Print(int fd,treeNode x, bool no_text);\r
+ void Print(int fd,treeNode x) { Print(fd,x,false); }\r
+\r
};\r
+\r
+\r
+\r
+\r
#endif\r
+\r
projects / SXSI / XMLTree.git / blobdiff