From 31514b441ce29f78c114e9d912ba09f37f91d257 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Kim=20Nguy=E1=BB=85n?= Date: Wed, 24 Jul 2013 22:52:03 +0200 Subject: [PATCH] Refactor the build process. Piggyback on remake's dependency tracking to perform a topological sort of the object files, used during linking. This is done by generating a .dep file for each object file keeping in a .dep the transitive closure of its dependencies. Also, packed modules must now be described by: - a directory - a file with the .pack extension, with the same basename as the directory, listing the module names (relative to the directory containing the .ml files) --- .gitignore | 1 + Remakefile.in | 205 ++++++++++++++++++++++------------------ src/xpath.pack | 5 + tools/ocamlmoduledep.sh | 33 ++++--- 4 files changed, 136 insertions(+), 108 deletions(-) create mode 100644 src/xpath.pack diff --git a/.gitignore b/.gitignore index 804ed5a..c683a0c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.dep *.cm* *.o *.native diff --git a/Remakefile.in b/Remakefile.in index 8f6dbb7..cb9fc4a 100644 --- a/Remakefile.in +++ b/Remakefile.in @@ -6,6 +6,7 @@ OCAMLFINDFLAGS = -syntax $(OCAMLFINDSYNTAX) -package $(OCAMLFINDPACKAGES) \ $(OCAMLFINDPPOPTS) $(OCAMLFINDINCLUDES) OCAMLFINDLINKFLAGS = -linkpkg PACKAGE = @PACKAGE_TARNAME@ +SRC = src BIN = src/@PACKAGE_TARNAME@.native@EXE@ EXE = @EXE@ @@ -34,7 +35,7 @@ clean: for dir in src tools; do find $dir -name '*.cm*' -o -name '*.o' -o -name '*.byte' -o \ -name '*.native' -o -name '*.mll' -o -name '*.mly' -o \ - -name '*.class' | while read file; do + -name '*.class' -o -name '*.dep' | while read file; do case "$file" in *.mll) rm -f "${file%.mll}.ml" @@ -55,104 +56,126 @@ distclean: clean test_clean %.class: %.java javac $< -%.native$(EXE): - base=$* - src=${base}.ml - obj=${base}.cmx - dir=src - $(REMAKE) -v OCAMLDEPNATIVE=-native "$obj" - srcs=`ls "$dir"/*.cmx | sed 's/\.cmx/.ml/g'` - objs=`$(OCAMLDEP) $(OCAMLDEPNATIVE) $(OCAMLFINDFLAGS) -modules $srcs | tools/osort.sh cmx` - $(OCAMLOPT) -o "$@" $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDLINKFLAGS) $(OCAMLFINDFLAGS) $objs +%.native$(EXE): %.cmx %.dep + objects=`cat $*.dep | xargs | sed 's/[.]dep/.cmx/g'` + $(OCAMLOPT) -o $@ $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDLINKFLAGS) $(OCAMLFINDFLAGS) $objects $< + +%.byte$(EXE): %.cmo %.dep + objects=`cat $*.dep | xargs | sed 's/[.]dep/.cmo/g'` + $(OCAMLC) -o $@ $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDLINKFLAGS) $(OCAMLFINDFLAGS) $objects $< + +%.ml: + if test -f $*.mly; then + $(OCAMLYACC) $*.mly + elif test -f $*.mll; then + $(OCAMLLEX) $*.mll + fi -%.byte$(EXE): +%.cmx %.dep: + REMAKE="$(REMAKE) -v OCAMLNATIVE=-native" base=$* - src=${base}.ml - obj=${base}.cmo - dir=src - $(REMAKE) "$obj" - objs="" - for i in "$dir"/*.cmi; do - ibase=${i%.cmi} - if test -f "${ibase}.ml" -o -d "${ibase}" -o -f "${ibase}.mly" -o -f "${ibase}.mll"; then - objs="$objs ${ibase}.cmo" - fi - done - $(REMAKE) $objs - srcs=`ls "$dir"/*.cmo | sed 's/\.cmo/.ml/g'` - sorted_objs=`$(OCAMLDEP) $OCAMLFINDFLAGS -modules $srcs | tools/osort.sh cmo` - $(OCAMLC) -o "$@" $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDLINKFLAGS) $(OCAMLFINDFLAGS) $sorted_objs + dirname=${base%/*} + if test -f "$dirname".pack; then + #we are below a pack directory + PACKDIR=`basename $dirname` + PACKINCLUDE="-I $dirname" + elif test -f $*.pack -a -d $*; then + $(REMAKE) $*.pack + PACKINCLUDE="-I $*" + PACKDIR=`basename $*` + DOPACK=1 + fi + if test "$PACKDIR"; then + PACKNAME=`echo $PACKDIR | cut -b1 | tr a-z A-Z`${PACKDIR#?} + FORPACK="-for-pack $PACKNAME" + fi -%.cmx: - target="$@" - base="${target%.cmx}" - src="${base}.ml" - if [ -f "$src" ]; then - $(REMAKE) "$src" - deps=`$(ODEPS) -native $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"` - echo "$deps" | $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -v OCAMLDEPNATIVE=-native -v OCAMLFORPACK="$(OCAMLFORPACK)" -r "$target" - $(OCAMLOPT) -o "$target" -c $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFORPACK) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src" - elif [ -d "$base" ]; then - modname=`basename "$base"` - packname=`echo "$modname" | sed 's/\(.*\)/\u\1/'` - objs=`ls "$base"/*.ml | sed 's/\.ml/.cmx/g'` - $(REMAKE) -v PACKINCLUDE="-I $base" -v OCAMLFORPACK="-for-pack $packname" $objs - sorted_objs=`$(OCAMLDEP) -native $(OCAMLFINDFLAGS) -I $base -modules "$base"/*.ml | tools/osort.sh cmx` - $(OCAMLOPT) -o "$target" -pack $sorted_objs - elif [ -f "$base".mly ]; then - $(REMAKE) "$base".mly - $(OCAMLYACC) "$base".mly - $(ODEPS) $(OCAMLDEPNATIVE) $(OCAMLFINDFLAGS) "$src" | \ - $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -v OCAMLDEPNATIVE=-native -v OCAMLFORPACK="$(OCAMLFORPACK)" -r "$target" - $(OCAMLOPT) -o "$target" -c $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFORPACK) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src" + if test "$DOPACK"; then + modules=`cat $*.pack` + objects=`echo $modules | xargs tools/ocamlmoduledep.sh -native $PACKINCLUDE -I $(SRC) ` + else + $REMAKE $*.ml + modules=`$(OCAMLDEP) $(OCAMLFINDFLAGS) -modules $*.ml | cut -f 2- -d ':'` + objects=`tools/ocamlmoduledep.sh -native $PACKINCLUDE -I $(SRC) $modules` fi + $REMAKE $objects -%.cmo: - target="$@" - base="${target%.cmo}" - src="${base}.ml" - if [ -f "$src" ]; then - $(REMAKE) "$src" - deps=`$(ODEPS) $OCAMLFINDFLAGS $PACKINCLUDE "$src"` - echo "$deps" | $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -r "$target" - $(OCAMLC) -o "$target" -c $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src" - elif [ -d "$base" ]; then - modname=`basename "$base"` - packname=`echo "$modname" | sed 's/\(.*\)/\u\1/'` - objs=`ls "$base"/*.ml | sed 's/\.ml/.cmo/g'` - $(REMAKE) -v PACKINCLUDE="-I $base" $objs - sorted_objs=`$(OCAMLDEP) $(OCAMLFINDFLAGS) -I $base -modules "$base"/*.ml | tools/osort.sh cmo` - $(OCAMLC) -o "$target" -pack $sorted_objs - elif [ -f "$base".mly ]; then - $(REMAKE) "$base".mly - $(OCAMLYACC) "$base".mly - $(ODEPS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src" | $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -r "$target" - $(OCAMLC) -o "$target" -c $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src" + deps=`echo $objects | sed 's/[.]cm[ix]/.dep/g'` + rm -f $*.dep; touch $*.dep + $(REMAKE) $deps + for f in $deps; do + for g in `cat $f`; do + if grep -q $g $*.dep; then continue; fi + echo $g >> $*.dep + done + if grep -q $f $*.dep; then continue; fi + echo $f >> $*.dep + done + if test -f $*.mli; then $REMAKE -v PACKINCLUDE="$PACKINCLUDE" $*.cmi; fi + if test "$DOPACK"; then + sorted_objects=`cat $*.dep | grep "$PACKDIR" | sed 's/[.]dep/.cmx/' | xargs` + cat $*.dep | grep -v "$PACKDIR" > $*.tmp + mv $*.tmp $*.dep + $(OCAMLOPT) -o $@ -pack $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDFLAGS) $sorted_objects + else + $(OCAMLOPT) -o $@ $FORPACK -c $PACKINCLUDE $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDFLAGS) $*.ml fi +%.cmo %.dep: + REMAKE="$(REMAKE)" + base=$* + dirname=${base%/*} + if test -f "$dirname".pack; then + #we are below a pack directory + PACKDIR=`basename $dirname` + PACKINCLUDE="-I $dirname" + elif test -f $*.pack -a -d $*; then + PACKINCLUDE="-I $*" + PACKDIR=`basename $*` + DOPACK=1 + fi + if test "$DOPACK"; then + modules=`cat $*.pack` + objects=`echo $modules | xargs tools/ocamlmoduledep.sh $PACKINCLUDE -I $(SRC) ` + $REMAKE $objects + else + $REMAKE $*.ml + modules=`$(OCAMLDEP) $(OCAMLFINDFLAGS) -modules $*.ml | cut -f 2- -d ':'` + objects=`tools/ocamlmoduledep.sh $PACKINCLUDE -I $(SRC) $modules` + $REMAKE $objects + fi + + deps=`echo $objects | sed 's/[.]cm[io]/.dep/g'` + rm -f $*.dep; touch $*.dep + $(REMAKE) $deps + for f in $deps; do + for g in `cat $f`; do + if grep -q $g $*.dep; then continue; fi + echo $g >> $*.dep + done + if grep -q $f $*.dep; then continue; fi + echo $f >> $*.dep + done + + if test -f $*.mli; then $REMAKE -v PACKINCLUDE="$PACKINCLUDE" $*.cmi; fi + if test "$DOPACK"; then + sorted_objects=`cat $*.dep | grep "$PACKDIR" | sed 's/[.]dep/.cmo/' | xargs` + cat $*.dep | grep -v "$PACKDIR" > $*.tmp + mv $*.tmp $*.dep + $(OCAMLC) -o $@ -pack $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $sorted_objects + else + $(OCAMLC) -o $@ -c $PACKINCLUDE $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $*.ml + fi %.cmi: - target="$@" - base=${target%.cmi} - if test -f "$base".mli; then - $(REMAKE) "$base".mli - src=${base}.mli - deps=`$(ODEPS) $(OCAMLDEPNATIVE) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src"` - echo "$deps" | $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -v OCAMLDEPNATIVE="$(OCAMLDEPNATIVE)" -v OCAMLFORPACK="$(OCAMLFORPACK)" -r "$target" - if test -z "$(OCAMLDEPNATIVE)"; then - $(OCAMLC) -o "$target" -c $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src" - else - $(OCAMLOPT) -o "$target" -c $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) "$src" - fi + $(REMAKE) $*.mli + if test -z "$(OCAMLNATIVE)"; then + $(OCAMLC) -c $(OCAMLFLAGS) $(OCAMLCFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) $*.mli else - if test -z "$(OCAMLDEPNATIVE)"; then - obj=${base}.cmo - else - obj=${base}.cmx - fi - $(REMAKE) -v PACKINCLUDE="$(PACKINCLUDE)" -v OCAMLDEPNATIVE="$(OCAMLDEPNATIVE)" -v OCAMLFORPACK="$(OCAMLFORPACK)" "$obj" + $(OCAMLOPT) -c $(OCAMLFLAGS) $(OCAMLOPTFLAGS) $(OCAMLFINDFLAGS) $(PACKINCLUDE) $*.mli + fi ## Tests @@ -174,7 +197,7 @@ test_clean: base=${base%.xml.test1.summary} $(REMAKE) "$base".xml "$base".xml.queries rm -f "$@" - MSG="Test 1 (single query: tatoo vs java implementation)" + MSG="Test 1 (single query: tatoo vs java implementation)" echo "$MSG" >> "$@" echo "$MSG" cat "$base".xml.queries | grep -v '^#' | while read q query; do @@ -240,7 +263,7 @@ test_clean: base=${base%.xml.test3.summary} $(REMAKE) "$base".xml "$base".xml.queries rm -f "$@" - MSG="Test 3 (multiple queries composition: sequential vs parallel)" + MSG="Test 3 (multiple queries composition: sequential vs parallel)" echo "$MSG" >> "$@" echo "$MSG" cat "$base".xml.queries | grep -v '^#' | while read q query; do @@ -248,7 +271,7 @@ test_clean: OUTPUTA="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.xml LOG="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.log src/@PACKAGE_TARNAME@.native@EXE@ -s -d -C "$base".xml \ - -o "$OUTPUTA" $(src/split_path.native$(EXE) "$query") > "$LOG" 2>&1 + -o "$OUTPUTA" $(tools/split_path.native$(EXE) "$query") > "$LOG" 2>&1 echo "Query: $q : $query" >> "$@" cat "$LOG" | grep '^STATS' >> "$@" @@ -256,7 +279,7 @@ test_clean: OUTPUTB="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.xml LOG="$base".xml.results/"$q"_"$(PACKAGE)"_test3a.log src/@PACKAGE_TARNAME@.native@EXE@ -s -d -p -C "$base".xml \ - -o "$OUTPUTB" $(src/split_path.native$(EXE) "$query") > "$LOG" 2>&1 + -o "$OUTPUTB" $(tools/split_path.native$(EXE) "$query") > "$LOG" 2>&1 echo "Query: $q : $query" >> "$@" cat "$LOG" | grep '^STATS' >> "$@" diff --git a/src/xpath.pack b/src/xpath.pack new file mode 100644 index 0000000..7aeb605 --- /dev/null +++ b/src/xpath.pack @@ -0,0 +1,5 @@ +Ast +Compile +Parser +Ulexer +Xpath_internal_parser diff --git a/tools/ocamlmoduledep.sh b/tools/ocamlmoduledep.sh index 3102e1e..3bab186 100755 --- a/tools/ocamlmoduledep.sh +++ b/tools/ocamlmoduledep.sh @@ -8,7 +8,7 @@ usage() { } INCLUDES="" -MODULE="" +MODULES="" NATIVE=0 while true; do case $1 in @@ -22,12 +22,7 @@ $2" NATIVE=1 ;; [A-Z]*) - if test -z "$MODULE"; then - MODULE=$1 - else - usage - exit 1 - fi + MODULES="$MODULES $1" ;; *) echo "ERROR: '$1'" @@ -38,20 +33,24 @@ $2" shift if test "$#" -eq 0; then break; fi done -module="$(echo $MODULE | cut -b1 | tr A-Z a-z)$(echo $MODULE | cut -b2-)" if test "$NATIVE" = "1"; then ext=cmx else ext=cmo fi -echo "$INCLUDES" | while read include; do - base="$include/$module" - if test -f "$base".ml -o -f "$base".mly -o -f "$base".mll -o -d "$base"; then - echo "$base"."$ext" - break - elif test -f "$base".mli; then - echo "$base".cmi - break - fi +for MODULE in $MODULES; do + module="$(echo $MODULE | cut -b1 | tr A-Z a-z)$(echo $MODULE | cut -b2-)" + echo "$INCLUDES" | while read include; do + base="$include/$module" + if test -z "$include"; then continue; fi + if test -f "$base".ml -o -f "$base".mly -o -f "$base".mll -o -f "$base".pack; then + echo -n "$base"."$ext " + break + elif test -f "$base".mli; then + echo -n "$base"."cmi " + break + fi + done done +echo -- 2.17.1