diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..0e6dadb --- /dev/null +++ b/Makefile.am @@ -0,0 +1,18 @@ +# not a GNU package. You can remove this line, if +# have all needed files, that a GNU package needs +AUTOMAKE_OPTIONS = foreign + +SUBDIRS = src scripts doc + +EXTRA_DIST = README RELEASE Copyright + +ACLOCAL_AMFLAGS = -I m4 + +LN_S=@LN_S@ + +install-exec-hook: + cd ${exec_prefix}/ && \ + ${LN_S} -f lib lib64 + +dist-hook: + rm -rf `find $(distdir)/doc -type d -name .svn` diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..2b20543 --- /dev/null +++ b/configure.ac @@ -0,0 +1,191 @@ +AC_INIT([irstlm], [5.80.06]) +AM_INIT_AUTOMAKE([-Wall -Werror foreign]) +AC_PROG_CC +AC_PROG_CXX +AC_PROG_LIBTOOL +AC_CONFIG_HEADERS([config.h]) +AC_CONFIG_MACRO_DIR([m4]) + +m4_pattern_allow([AM_PROG_AR],[AM_PROG_AR]) +AM_PROG_AR + +AC_ARG_ENABLE([doc], + [AC_HELP_STRING([--enable-doc|--disable-doc], [Enable or Disable (default) creation of documentation])]) + +AC_ARG_ENABLE([trace], + [AC_HELP_STRING([--enable-trace|--disable-trace], [Enable (default) or Disable trace info at run-time])]) + +AC_ARG_ENABLE([assert], + [AC_HELP_STRING([--enable-assert|--disable-assert], [Enable (default) or Disable assert calls at run-time])]) + +AC_ARG_ENABLE([debugging], + [AC_HELP_STRING([--enable-debugging|--disable-debugging], [Enable or Disable (default) debugging info ("-g -O2")])]) + +AC_ARG_ENABLE(profiling, + [AC_HELP_STRING([--enable-profiling|--disable-profiling], [Enable or Disable (default) profiling info ("-pg")])]) + +AC_ARG_ENABLE(caching, + [AC_HELP_STRING([--enable-caching|--disable-caching], [Enable or Disable (default) internal caches to store probs and other info])]) + +AC_ARG_ENABLE(output, + [AC_HELP_STRING([--enable-output|--disable-output], [Enable (default) or Disable part of the output])]) + +AC_ARG_ENABLE(interpolatedsearch, + [AC_HELP_STRING([--enable-interpolatedsearch|--disable-interpolatedsearch], [Enable or Disable (default) interpolated search for n-grams])]) + +AC_ARG_ENABLE(optimization, + [AC_HELP_STRING([--enable-optimization|--disable-optimization], [Enable or Disable (default) optimization info ("-O3")])]) + +AC_ARG_ENABLE(cxx0, + [AC_HELP_STRING([--enable-cxx0|--disable-cxx0], [Enable or Disable (default) c++0x dialect ("-std=c++0x")])]) + +AC_ARG_WITH(zlib, + [AC_HELP_STRING([--with-zlib=PATH], [(optional) path to zlib])], + [with_zlib=$withval], + [with_zlib=no] + ) + +AC_ARG_WITH(tracelevel, + [AC_HELP_STRING([--with-tracelevel=VAL], [(optional) level of tracing; default 0, tracing disabled])], + [with_tracelevel=$withval], + [with_tracelevel=0] + ) + +AM_CONDITIONAL([DOC_COMPILATION],false) +if test "x$enable_doc" = 'xyes' +then + AC_CHECK_TOOL(PDFLATEX,pdflatex,"no") + AC_CHECK_TOOL(BIBTEX,bibtex,"no") + if test "x$PDFLATEX" != "xno" && test "x$BIBTEX" != "xno" + then + AM_CONDITIONAL([DOC_COMPILATION],true) + AC_MSG_NOTICE([pdflatex and bibtex are available]) + AC_MSG_NOTICE([documentation will be created]) + else + AC_MSG_NOTICE([either pdflatex or bibtex is not available]) + AC_MSG_NOTICE([documentation will not be created (default); get it through the website $WEBSITE]) + fi +else + AC_MSG_NOTICE([documentation will not be created (default); get it through the website $WEBSITE]) +fi + + +#### Use this if you want that the default is yes +#### if test "x$enable_foo" != 'xno' +#### Use this if you want that the default is no +#### if test "x$enable_foo" = 'xyes' + + +if test "x$with_tracelevel" != 'x0' +then +if test "x$enable_trace" != 'xno' +then + TRACELEVEL=${with_tracelevel} + AC_MSG_NOTICE([trace enabled (default); trace level is $TRACELEVEL]) +else + TRACELEVEL=0 + AC_MSG_NOTICE([trace disabled; trace level is overwritten to $TRACELEVEL; most regression tests will fail]) +fi +else +if test "x$enable_trace" != 'xno' +then + TRACELEVEL=1 + AC_MSG_NOTICE([trace enabled (default); trace level is overwritten to default value $TRACELEVEL]) +else + TRACELEVEL=0 + AC_MSG_NOTICE([trace disabled; trace level is $TRACELEVEL; most regression tests will fail]) +fi +fi +CPPFLAGS="$CPPFLAGS -DTRACE_LEVEL=$TRACELEVEL" + +if test "x$enable_assert" != 'xno' +then + AC_MSG_NOTICE([assert enabled (default)]) + CPPFLAGS="$CPPFLAGS -DMY_ASSERT_FLAG" +else + AC_MSG_NOTICE([assert disabled]) + CPPFLAGS="$CPPFLAGS -UMY_ASSERT_FLAG" +fi + +if test "x$enable_debugging" = 'xyes' +then + AC_MSG_NOTICE([generation of debugging symbols enabled, compilation with "-g -O2"]) + CPPFLAGS="$CPPFLAGS -g -O2" +else + AC_MSG_NOTICE([generation of debugging symbols disabled (default), compilation without "-g", only "-O2"]) +fi + +if test "x$enable_profiling" = 'xyes' +then + AC_MSG_NOTICE([profiling enabled, compilation with "-pg"]) + CPPFLAGS="$CPPFLAGS -pg" + LDFLAGS="$LDFLAGS -pg" +else + AC_MSG_NOTICE([profiling disabled (default)]) +fi + +if test "x$enable_caching" = 'xyes' +then + AC_MSG_NOTICE([caching enabled]) + CPPFLAGS="$CPPFLAGS -DPS_CACHE_ENABLE -DLMT_CACHE_ENABLE -DMDIADAPTLM_CACHE_ENABLE"; + LDFLAGS="$LDFLAGS" +else + AC_MSG_NOTICE([caching disabled (default)]) +fi + +if test "x$enable_output" = 'xno' +then + AC_MSG_NOTICE([part of the output on stdout is suppressed]) + CPPFLAGS="$CPPFLAGS -DOUTPUT_SUPPRESSED"; + LDFLAGS="$LDFLAGS" +else + AC_MSG_NOTICE([caching disabled (default)]) +fi + +if test "x$enable_interpolatedsearch" = 'xyes' +then + AC_MSG_NOTICE([interpolated search enabled]) + CPPFLAGS="$CPPFLAGS -DINTERP_SEARCH"; + LDFLAGS="$LDFLAGS" +else + AC_MSG_NOTICE([interpolated search disabled (default)]) +fi + +if test "x$enable_optimization" = 'xyes' +then + AC_MSG_NOTICE([optimization enabled, compilation with "-O3"]) + CPPFLAGS="$CPPFLAGS -O3"; + LDFLAGS="$LDFLAGS -O3" +else + AC_MSG_NOTICE([optimization disabled (default)]) +fi + +if test "x$enable_cxx0" != 'xno' +then + AC_MSG_NOTICE([c++x0 dialect is enabled (default), compilation with "-DHAVE_CXX0 -std=c++0x "]) + CPPFLAGS="$CPPFLAGS -DHAVE_CXX0 -std=c++0x"; +else + AC_MSG_NOTICE([c++x0 dialect is disabled, compilation without "-std=c++0x" and with "-UHAVE_CXX0"]) + CPPFLAGS="$CPPFLAGS -UHAVE_CXX0"; +fi + +if test "x$with_zlib" != 'xno' +then + CPPFLAGS="$CPPFLAGS -I${with_zlib}/include" + LDFLAGS="$LDFLAGS -L${with_zlib}/lib" +fi + +LIBS="$LIBS -lz" + +AC_CONFIG_FILES([ +Makefile +src/Makefile +scripts/Makefile +doc/Makefile +]) + +AC_SUBST(transform,'s/_lm/-lm/') + +AC_OUTPUT() + +AC_MSG_NOTICE([The software will be installed into $prefix]) diff --git a/doc/Makefile.am b/doc/Makefile.am new file mode 100644 index 0000000..a3d449b --- /dev/null +++ b/doc/Makefile.am @@ -0,0 +1,23 @@ +documentationdir = @prefix@/doc + +if DOC_COMPILATION + +irstlm-manual.pdf: irstlm-manual.tex + pdflatex irstlm-manual ; \ + bibtex irstlm-manual ; \ + pdflatex irstlm-manual ; \ + pdflatex irstlm-manual ; \ + rm *.aux *.log *.bbl *.blg + +clean-local: + rm -rf irstlm-manual.pdf + +all: irstlm-manual.pdf + +dist_documentation_SCRIPTS = irstlm-manual.pdf + +endif + +EXTRA_DIST = irstlm-manual.tex + + diff --git a/regenerate-makefiles.sh b/regenerate-makefiles.sh new file mode 100755 index 0000000..68152b6 --- /dev/null +++ b/regenerate-makefiles.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# NOTE: +# Versions 1.9 (or higher) of aclocal and automake are required. +# Version 2.59 (or higher) of autoconf is required. + +# For Mac OSX users: +# Standard distribution usually includes versions 1.6 for aclocal and automake. +# Get versions 1.9 or higher +# Set the following variable to the correct paths +#ACLOCAL="/path/to/aclocal-1.9" +#AUTOMAKE="/path/to/automake-1.9" + +force=$1; +# set parameter force to the value "--force" if you want to recreate all links to the autotools + +die () { + echo "$@" >&2 + exit 1 +} + +if [ -z "$ACLOCAL" ] +then + ACLOCAL=`which aclocal` +fi + +if [ -z "$AUTOMAKE" ] +then + AUTOMAKE=`which automake` +fi + +if [ -z "$AUTORECONF" ] +then + AUTORECONF=`which autoreconf` +fi + +if [ -z "$AUTOCONF" ] +then + AUTOCONF=`which autoconf` +fi + +if [ -z "$LIBTOOLIZE" ] +then + LIBTOOLIZE=`which libtoolize` + + if [ -z "$LIBTOOLIZE" ] + then + LIBTOOLIZE=`which glibtoolize` + fi +fi + +if [ ! -d m4 ] ; +then +mkdir m4 +fi + +echo "Calling $AUTORECONF" +$AUTORECONF + +ret=$? + +if [ $ret -ne 0 ] ; then +echo "autoreconf FAILED" +echo "trying '$LIBTOOLIZE --force; $AUTOMAKE --add-missing ; $AUTORECONF'" +$LIBTOOLIZE --force +$AUTOMAKE --add-missing +$AUTORECONF +if [ ! -e config.guess ] ; then +$AUTOMAKE --add-missing +$AUTORECONF +fi +fi + +#echo "Calling $LIBTOOLIZE $force" +#$LIBTOOLIZE $force || die "libtoolize failed" + +#echo "Calling $ACLOCAL..." +#$ACLOCAL -I m4 || die "aclocal failed" + +#echo "Calling $AUTOCONF..." +#$AUTOCONF || die "autoconf failed" + +#echo "Calling $AUTOMAKE --add-missing..." +#$AUTOMAKE --add-missing || die "automake failed" diff --git a/scripts/Makefile.am b/scripts/Makefile.am new file mode 100644 index 0000000..6893b01 --- /dev/null +++ b/scripts/Makefile.am @@ -0,0 +1,9 @@ +wrapperbindir = @prefix@/bin + +dist_wrapperbin_SCRIPTS = \ +add-start-end.sh build-lm-qsub.sh build-lm.sh rm-start-end.sh split-ngt.sh mdtsel.sh \ +build-sublm.pl goograms2ngrams.pl lm-stat.pl merge-sublm.pl ngram-split.pl sort-lm.pl split-dict.pl \ +plsa.sh qplsa.sh + +EXTRA_DIST = wrapper + diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..1f8a065 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,97 @@ +lib_LTLIBRARIES = libirstlm.la + +AM_CXXFLAGS = -isystem/usr/include -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES $(BOOST_CPPFLAGS) -DMYCODESIZE=3 + +libirstlm_ladir = ${includedir} + +libirstlm_la_HEADERS = \ + cmd.h \ + thpool.h \ + dictionary.h \ + gzfilebuf.h \ + htable.h \ + index.h \ + lmContainer.h \ + lmclass.h \ + lmmacro.h \ + lmtable.h \ + lmInterpolation.h \ + mempool.h \ + mfstream.h \ + n_gram.h \ + ngramcache.h \ + ngramtable.h \ + timer.h \ + util.h \ + crc.h \ + interplm.h \ + linearlm.h \ + mdiadapt.h \ + mixture.h \ + normcache.h \ + shiftlm.h \ + cplsa.h \ + cswam.h \ + doc.h + +libirstlm_la_SOURCES = \ + cmd.c \ + thpool.c \ + dictionary.cpp \ + htable.cpp \ + lmContainer.cpp \ + lmclass.cpp \ + lmmacro.cpp \ + lmtable.cpp \ + lmInterpolation.cpp \ + mempool.cpp \ + mfstream.cpp \ + n_gram.cpp \ + ngramcache.cpp \ + ngramtable.cpp \ + timer.cpp \ + util.cpp \ + crc.cpp \ + interplm.cpp \ + linearlm.cpp \ + mdiadapt.cpp \ + mixture.cpp \ + normcache.cpp \ + shiftlm.cpp \ + cplsa.cpp \ + cswam.cpp \ + doc.cpp + +CLEANFILES = $(BUILT_SOURCES) + +libirstlm_la_LIBADD = $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) + +LDADD = -lirstlm -lpthread +DEPENDENCIES = libirstlm.la + +bin_PROGRAMS = dict ngt dtsel compile-lm interpolate-lm prune-lm quantize-lm prune-lm score-lm tlm plsa verify-caching cswa + +dict_SOURCES = dict.cpp +dict_DEPENDENCIES = $(DEPENDENCIES) +ngt_SOURCES = ngt.cpp +ngt_DEPENDENCIES = $(DEPENDENCIES) +dtsel_SOURCES = dtsel.cpp +dtsel_DEPENDENCIES = $(DEPENDENCIES) +compile_lm_SOURCES = compile-lm.cpp +compile_lm_DEPENDENCIES = $(DEPENDENCIES) +interpolate_lm_SOURCES = interpolate-lm.cpp +interpolate_lm_DEPENDENCIES = $(DEPENDENCIES) +prune_lm_SOURCES = prune-lm.cpp +prune_lm_DEPENDENCIES = $(DEPENDENCIES) +quantize_lm_SOURCES = quantize-lm.cpp +quantize_lm_DEPENDENCIES = $(DEPENDENCIES) +score_lm_SOURCES = score-lm.cpp +score_lm_DEPENDENCIES = $(DEPENDENCIES) +tlm_SOURCES = tlm.cpp +tlm_DEPENDENCIES = $(DEPENDENCIES) +plsa_SOURCES = plsa.cpp +plsa_DEPENDENCIES = $(DEPENDENCIES) +verify_caching_SOURCES = verify-caching.cpp +verify_caching_DEPENDENCIES = $(DEPENDENCIES) +cswa_SOURCES = cswa.cpp +cswa_DEPENDENCIES = $(DEPENDENCIES)