Refactor the build process. Piggyback on remake's dependency tracking

to perform a topological sort of the object files, used during linking.
This is done by generating a .dep file for each object file keeping in
a .dep the transitive closure of its dependencies.
Also, packed modules must now be described by:
- a directory
- a file with the .pack extension, with the same basename as the directory,
listing the module names (relative to the directory containing the .ml files)

diff --git a/.gitignore b/.gitignore
index 804ed5a..c683a0c 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+*.dep

 *.cm*
 *.o
 *.native
 *.cm*
 *.o
 *.native

diff --git a/Remakefile.in b/Remakefile.in
index 8f6dbb7..cb9fc4a 100644 (file)
--- a/Remakefile.in
+++ b/Remakefile.in
@@ -6,6 +6,7 @@ OCAMLFINDFLAGS = -syntax $(OCAMLFINDSYNTAX) -package $(OCAMLFINDPACKAGES)  \
        $(OCAMLFINDPPOPTS) $(OCAMLFINDINCLUDES)
 OCAMLFINDLINKFLAGS = -linkpkg
 PACKAGE = @PACKAGE_TARNAME@
        $(OCAMLFINDPPOPTS) $(OCAMLFINDINCLUDES)
 OCAMLFINDLINKFLAGS = -linkpkg
 PACKAGE = @PACKAGE_TARNAME@

+SRC = src

 BIN = src/@PACKAGE_TARNAME@.native@EXE@
 EXE = @EXE@
 
 BIN = src/@PACKAGE_TARNAME@.native@EXE@
 EXE = @EXE@
 


@@ -34,7 +35,7 @@ clean:
        for dir in src tools; do
                find $dir -name '*.cm*' -o -name '*.o' -o -name '*.byte' -o \
                          -name '*.native' -o -name '*.mll' -o -name '*.mly' -o \
        for dir in src tools; do
                find $dir -name '*.cm*' -o -name '*.o' -o -name '*.byte' -o \
                          -name '*.native' -o -name '*.mll' -o -name '*.mly' -o \

-                         -name '*.class' | while read file; do
+                         -name '*.class' -o -name '*.dep' | while read file; do

                case "$file" in
                        *.mll)
                                rm -f "${file%.mll}.ml"
                case "$file" in
                        *.mll)
                                rm -f "${file%.mll}.ml"


@@ -55,104 +56,126 @@ distclean: clean test_clean
 %.class: %.java
        javac $<
 
 %.class: %.java
        javac $<
 

-%.native$(EXE):
-       base=$*
-       src=${base}.ml
-       obj=${base}.cmx
-       dir=src
-       $(REMAKE) -v OCAMLDEPNATIVE=-native "$obj"
-       srcs=`ls "$dir"/*.cmx | sed 's/\.cmx/.ml/g'`
-       objs=`$(OCAMLDEP) $(OCAMLDEPNATIVE) $(OCAMLFINDFLAGS) -modules $srcs | tools/osort.sh cmx`
-       $(OCAMLOPT) -o "$@"  $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDLINKFLAGS) $(OCAMLFINDFLAGS) $objs
+%.native$(EXE): %.cmx %.dep
+       objects=`cat $*.dep | xargs | sed 's/[.]dep/.cmx/g'`
+       $(OCAMLOPT) -o $@  $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDLINKFLAGS) $(OCAMLFINDFLAGS) $objects $<
+
+%.byte$(EXE): %.cmo %.dep
+       objects=`cat $*.dep | xargs | sed 's/[.]dep/.cmo/g'`
+       $(OCAMLC) -o $@  $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDLINKFLAGS) $(OCAMLFINDFLAGS) $objects $<
+
+%.ml:
+       if test -f $*.mly; then
+               $(OCAMLYACC) $*.mly
+       elif test -f $*.mll; then
+               $(OCAMLLEX) $*.mll
+       fi

 
 

-%.byte$(EXE):
+%.cmx %.dep:
+       REMAKE="$(REMAKE) -v OCAMLNATIVE=-native"

        base=$*
        base=$*

-       src=${base}.ml
-       obj=${base}.cmo
-       dir=src
-       $(REMAKE) "$obj"
-       objs=""
-       for i in "$dir"/*.cmi; do
-               ibase=${i%.cmi}
-               if test -f "${ibase}.ml" -o -d "${ibase}" -o -f "${ibase}.mly" -o -f "${ibase}.mll"; then
-                       objs="$objs ${ibase}.cmo"
-               fi
-       done
-       $(REMAKE) $objs
-       srcs=`ls "$dir"/*.cmo | sed 's/\.cmo/.ml/g'`
-       sorted_objs=`$(OCAMLDEP) $OCAMLFINDFLAGS -modules $srcs | tools/osort.sh cmo`
-       $(OCAMLC) -o "$@"  $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDLINKFLAGS) $(OCAMLFINDFLAGS) $sorted_objs
+       dirname=${base%/*}
+       if test -f "$dirname".pack; then
+               #we are below a pack directory
+               PACKDIR=`basename $dirname`
+               PACKINCLUDE="-I $dirname"
+       elif test -f $*.pack -a -d $*; then
+               $(REMAKE) $*.pack
+               PACKINCLUDE="-I $*"
+               PACKDIR=`basename $*`
+               DOPACK=1
+       fi
+       if test "$PACKDIR"; then
+               PACKNAME=`echo $PACKDIR | cut -b1 | tr a-z A-Z`${PACKDIR#?}
+               FORPACK="-for-pack $PACKNAME"
+       fi

 
 

-%.cmx:
-       target="$@"
-       base="${target%.cmx}"
-       src="${base}.ml"
-       if [ -f "$src" ]; then
-               $(REMAKE) "$src"
-               deps=`$(ODEPS) -native $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"`
-               echo "$deps" | $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -v OCAMLDEPNATIVE=-native -v OCAMLFORPACK="$(OCAMLFORPACK)" -r "$target"
-               $(OCAMLOPT) -o "$target" -c $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFORPACK) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"
-       elif [ -d "$base" ];    then
-               modname=`basename "$base"`
-               packname=`echo "$modname" | sed 's/\(.*\)/\u\1/'`
-               objs=`ls "$base"/*.ml | sed 's/\.ml/.cmx/g'`
-               $(REMAKE) -v PACKINCLUDE="-I $base" -v OCAMLFORPACK="-for-pack $packname" $objs
-               sorted_objs=`$(OCAMLDEP) -native $(OCAMLFINDFLAGS) -I $base -modules "$base"/*.ml | tools/osort.sh cmx`
-               $(OCAMLOPT) -o "$target" -pack $sorted_objs
-       elif [ -f "$base".mly ]; then
-               $(REMAKE) "$base".mly
-               $(OCAMLYACC) "$base".mly
-                $(ODEPS) $(OCAMLDEPNATIVE) $(OCAMLFINDFLAGS) "$src" | \
-                       $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -v OCAMLDEPNATIVE=-native -v OCAMLFORPACK="$(OCAMLFORPACK)" -r "$target"
-               $(OCAMLOPT) -o "$target" -c $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFORPACK) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"
+       if test "$DOPACK"; then
+               modules=`cat $*.pack`
+               objects=`echo $modules | xargs tools/ocamlmoduledep.sh -native $PACKINCLUDE -I $(SRC) `
+       else
+               $REMAKE $*.ml
+               modules=`$(OCAMLDEP) $(OCAMLFINDFLAGS) -modules $*.ml | cut -f 2- -d ':'`
+               objects=`tools/ocamlmoduledep.sh -native $PACKINCLUDE -I $(SRC) $modules`

        fi
 
        fi
 

+       $REMAKE $objects

 
 

-%.cmo:
-       target="$@"
-       base="${target%.cmo}"
-       src="${base}.ml"
-       if [ -f "$src" ]; then
-               $(REMAKE) "$src"
-               deps=`$(ODEPS) $OCAMLFINDFLAGS $PACKINCLUDE "$src"`
-               echo "$deps" | $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -r "$target"
-               $(OCAMLC) -o "$target" -c $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"
-       elif [ -d "$base" ];    then
-               modname=`basename "$base"`
-               packname=`echo "$modname" | sed 's/\(.*\)/\u\1/'`
-               objs=`ls "$base"/*.ml | sed 's/\.ml/.cmo/g'`
-               $(REMAKE) -v PACKINCLUDE="-I $base" $objs
-               sorted_objs=`$(OCAMLDEP) $(OCAMLFINDFLAGS) -I $base -modules "$base"/*.ml | tools/osort.sh cmo`
-               $(OCAMLC) -o "$target" -pack $sorted_objs
-       elif [ -f "$base".mly ]; then
-               $(REMAKE) "$base".mly
-               $(OCAMLYACC) "$base".mly
-                $(ODEPS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src" | $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -r "$target"
-               $(OCAMLC) -o "$target" -c $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"
+       deps=`echo $objects | sed 's/[.]cm[ix]/.dep/g'`
+       rm -f $*.dep; touch $*.dep
+       $(REMAKE) $deps
+       for f in $deps; do
+         for g in `cat $f`; do
+             if grep -q $g $*.dep; then continue; fi
+             echo $g >> $*.dep
+          done
+          if grep -q $f $*.dep; then continue; fi
+          echo $f >> $*.dep
+       done
+       if test -f $*.mli; then $REMAKE -v PACKINCLUDE="$PACKINCLUDE" $*.cmi; fi
+       if test "$DOPACK"; then
+               sorted_objects=`cat $*.dep | grep "$PACKDIR" | sed 's/[.]dep/.cmx/' | xargs`
+               cat $*.dep | grep -v "$PACKDIR" > $*.tmp
+               mv $*.tmp $*.dep
+               $(OCAMLOPT) -o $@ -pack $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDFLAGS) $sorted_objects
+       else
+               $(OCAMLOPT) -o $@ $FORPACK -c $PACKINCLUDE $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDFLAGS) $*.ml

        fi
 
        fi
 

+%.cmo %.dep:
+       REMAKE="$(REMAKE)"
+       base=$*
+       dirname=${base%/*}
+       if test -f "$dirname".pack; then
+               #we are below a pack directory
+               PACKDIR=`basename $dirname`
+               PACKINCLUDE="-I $dirname"
+       elif test -f $*.pack -a -d $*; then
+               PACKINCLUDE="-I $*"
+               PACKDIR=`basename $*`
+               DOPACK=1
+       fi

 
 

+       if test "$DOPACK"; then
+               modules=`cat $*.pack`
+               objects=`echo $modules | xargs tools/ocamlmoduledep.sh $PACKINCLUDE -I $(SRC) `
+               $REMAKE $objects
+       else
+               $REMAKE $*.ml
+               modules=`$(OCAMLDEP) $(OCAMLFINDFLAGS) -modules $*.ml | cut -f 2- -d ':'`
+               objects=`tools/ocamlmoduledep.sh $PACKINCLUDE -I $(SRC) $modules`
+               $REMAKE $objects
+       fi
+
+       deps=`echo $objects | sed 's/[.]cm[io]/.dep/g'`
+       rm -f $*.dep; touch $*.dep
+       $(REMAKE) $deps
+       for f in $deps; do
+          for g in `cat $f`; do
+             if grep -q $g $*.dep; then continue; fi
+             echo $g >> $*.dep
+          done
+          if grep -q $f $*.dep; then continue; fi
+          echo $f >> $*.dep
+       done
+
+       if test -f $*.mli; then $REMAKE -v PACKINCLUDE="$PACKINCLUDE" $*.cmi; fi
+       if test "$DOPACK"; then
+               sorted_objects=`cat $*.dep | grep "$PACKDIR" | sed 's/[.]dep/.cmo/' | xargs`
+               cat $*.dep | grep -v "$PACKDIR" > $*.tmp
+               mv $*.tmp $*.dep
+               $(OCAMLC) -o $@ -pack $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $sorted_objects
+       else
+               $(OCAMLC) -o $@ -c $PACKINCLUDE $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS)  $*.ml
+       fi

 
 %.cmi:
 
 %.cmi:

-       target="$@"
-       base=${target%.cmi}
-       if test -f "$base".mli; then
-               $(REMAKE) "$base".mli
-               src=${base}.mli
-               deps=`$(ODEPS) $(OCAMLDEPNATIVE) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"`
-               echo "$deps" | $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -v OCAMLDEPNATIVE="$(OCAMLDEPNATIVE)" -v OCAMLFORPACK="$(OCAMLFORPACK)" -r "$target"
-               if test -z "$(OCAMLDEPNATIVE)"; then
-                       $(OCAMLC) -o "$target" -c $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"
-               else
-                       $(OCAMLOPT) -o "$target" -c $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"
-               fi
+       $(REMAKE) $*.mli
+       if test -z "$(OCAMLNATIVE)"; then
+               $(OCAMLC) -c $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) $*.mli

        else
        else

-               if test -z "$(OCAMLDEPNATIVE)"; then
-                 obj=${base}.cmo
-               else
-                 obj=${base}.cmx
-               fi
-               $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -v OCAMLDEPNATIVE="$(OCAMLDEPNATIVE)" -v OCAMLFORPACK="$(OCAMLFORPACK)" "$obj"
+               $(OCAMLOPT) -c $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) $*.mli
+

        fi
 
 ## Tests
        fi
 
 ## Tests


@@ -174,7 +197,7 @@ test_clean:
                base=${base%.xml.test1.summary}
                $(REMAKE) "$base".xml "$base".xml.queries
                rm -f "$@"
                base=${base%.xml.test1.summary}
                $(REMAKE) "$base".xml "$base".xml.queries
                rm -f "$@"

-               MSG="Test 1 (single query: tatoo vs java implementation)" 
+               MSG="Test 1 (single query: tatoo vs java implementation)"

                echo "$MSG" >> "$@"
                echo "$MSG"
                cat "$base".xml.queries | grep -v '^#' | while read q query; do
                echo "$MSG" >> "$@"
                echo "$MSG"
                cat "$base".xml.queries | grep -v '^#' | while read q query; do


@@ -240,7 +263,7 @@ test_clean:
                base=${base%.xml.test3.summary}
                $(REMAKE) "$base".xml "$base".xml.queries
                rm -f "$@"
                base=${base%.xml.test3.summary}
                $(REMAKE) "$base".xml "$base".xml.queries
                rm -f "$@"

-               MSG="Test 3 (multiple queries composition: sequential vs parallel)" 
+               MSG="Test 3 (multiple queries composition: sequential vs parallel)"

                echo "$MSG" >> "$@"
                echo "$MSG"
                cat "$base".xml.queries | grep -v '^#' | while read q query; do
                echo "$MSG" >> "$@"
                echo "$MSG"
                cat "$base".xml.queries | grep -v '^#' | while read q query; do


@@ -248,7 +271,7 @@ test_clean:
                        OUTPUTA="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.xml
                        LOG="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.log
                        src/@PACKAGE_TARNAME@.native@EXE@ -s -d -C "$base".xml \
                        OUTPUTA="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.xml
                        LOG="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.log
                        src/@PACKAGE_TARNAME@.native@EXE@ -s -d -C "$base".xml \

-                                -o "$OUTPUTA" $(src/split_path.native$(EXE) "$query") > "$LOG" 2>&1
+                                -o "$OUTPUTA" $(tools/split_path.native$(EXE) "$query") > "$LOG" 2>&1

                        echo "Query: $q : $query" >> "$@"
                        cat  "$LOG" | grep '^STATS' >> "$@"
 
                        echo "Query: $q : $query" >> "$@"
                        cat  "$LOG" | grep '^STATS' >> "$@"
 


@@ -256,7 +279,7 @@ test_clean:
                        OUTPUTB="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.xml
                        LOG="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.log
                        src/@PACKAGE_TARNAME@.native@EXE@ -s -d -p -C "$base".xml \
                        OUTPUTB="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.xml
                        LOG="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.log
                        src/@PACKAGE_TARNAME@.native@EXE@ -s -d -p -C "$base".xml \

-                                -o "$OUTPUTB" $(src/split_path.native$(EXE) "$query") > "$LOG" 2>&1
+                                -o "$OUTPUTB" $(tools/split_path.native$(EXE) "$query") > "$LOG" 2>&1

                        echo "Query: $q : $query" >> "$@"
                        cat  "$LOG" | grep '^STATS' >> "$@"
 
                        echo "Query: $q : $query" >> "$@"
                        cat  "$LOG" | grep '^STATS' >> "$@"
 

diff --git a/src/xpath.pack b/src/xpath.pack
new file mode 100644 (file)
index 0000000..7aeb605
--- /dev/null
+++ b/src/xpath.pack
@@ -0,0 +1,5 @@
+Ast
+Compile
+Parser
+Ulexer
+Xpath_internal_parser

diff --git a/tools/ocamlmoduledep.sh b/tools/ocamlmoduledep.sh
index 3102e1e..3bab186 100755 (executable)
--- a/tools/ocamlmoduledep.sh
+++ b/tools/ocamlmoduledep.sh
@@ -8,7 +8,7 @@ usage() {
 }
 
 INCLUDES=""
 }
 
 INCLUDES=""

-MODULE=""
+MODULES=""

 NATIVE=0
 while true; do
     case $1 in
 NATIVE=0
 while true; do
     case $1 in


@@ -22,12 +22,7 @@ $2"
             NATIVE=1
             ;;
         [A-Z]*)
             NATIVE=1
             ;;
         [A-Z]*)

-            if test -z "$MODULE"; then
-                MODULE=$1
-            else
-                usage
-                exit 1
-            fi
+                MODULES="$MODULES $1"

             ;;
         *)
             echo "ERROR: '$1'"
             ;;
         *)
             echo "ERROR: '$1'"


@@ -38,20 +33,24 @@ $2"
     shift
     if test "$#" -eq 0; then break; fi
 done
     shift
     if test "$#" -eq 0; then break; fi
 done

-module="$(echo $MODULE | cut -b1 | tr A-Z a-z)$(echo $MODULE | cut -b2-)"

 if test "$NATIVE" = "1"; then
     ext=cmx
 else
     ext=cmo
 fi
 
 if test "$NATIVE" = "1"; then
     ext=cmx
 else
     ext=cmo
 fi
 

-echo "$INCLUDES" | while read include; do
-    base="$include/$module"
-    if test -f "$base".ml -o -f "$base".mly -o -f "$base".mll -o -d "$base"; then
-        echo "$base"."$ext"
-        break
-    elif test -f "$base".mli; then
-        echo "$base".cmi
-        break
-    fi
+for MODULE in $MODULES; do
+    module="$(echo $MODULE | cut -b1 | tr A-Z a-z)$(echo $MODULE | cut -b2-)"
+    echo "$INCLUDES" | while read include; do
+        base="$include/$module"
+        if test -z "$include"; then continue; fi
+        if test -f "$base".ml -o -f "$base".mly -o -f "$base".mll -o -f "$base".pack; then
+            echo -n "$base"."$ext "
+            break
+        elif test -f "$base".mli; then
+            echo -n "$base"."cmi "
+            break
+        fi
+    done

 done
 done

+echo