From f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2 Mon Sep 17 00:00:00 2001 From: Ian Hodson Date: Wed, 30 May 2012 21:27:06 +0100 Subject: Add openfst to external, as used by GoogleTTS Moved from GoogleTTS Change-Id: I6bc6bdadaa53bd0f810b88443339f6d899502cc8 --- src/extensions/far/Makefile.am | 34 ++ src/extensions/far/Makefile.in | 673 ++++++++++++++++++++++++++++++++ src/extensions/far/compile-strings.cc | 36 ++ src/extensions/far/farcompilestrings.cc | 84 ++++ src/extensions/far/farcreate.cc | 62 +++ src/extensions/far/farextract.cc | 62 +++ src/extensions/far/farinfo.cc | 56 +++ src/extensions/far/farprintstrings.cc | 70 ++++ src/extensions/far/farscript.cc | 113 ++++++ src/extensions/far/main.cc | 118 ++++++ src/extensions/far/stlist.cc | 31 ++ src/extensions/far/sttable.cc | 31 ++ 12 files changed, 1370 insertions(+) create mode 100644 src/extensions/far/Makefile.am create mode 100644 src/extensions/far/Makefile.in create mode 100644 src/extensions/far/compile-strings.cc create mode 100644 src/extensions/far/farcompilestrings.cc create mode 100644 src/extensions/far/farcreate.cc create mode 100644 src/extensions/far/farextract.cc create mode 100644 src/extensions/far/farinfo.cc create mode 100644 src/extensions/far/farprintstrings.cc create mode 100644 src/extensions/far/farscript.cc create mode 100644 src/extensions/far/main.cc create mode 100644 src/extensions/far/stlist.cc create mode 100644 src/extensions/far/sttable.cc (limited to 'src/extensions/far') diff --git a/src/extensions/far/Makefile.am b/src/extensions/far/Makefile.am new file mode 100644 index 0000000..61f7cc8 --- /dev/null +++ b/src/extensions/far/Makefile.am @@ -0,0 +1,34 @@ +AM_CPPFLAGS = -I$(srcdir)/../../include $(ICU_CPPFLAGS) + +libfstdir = @libfstdir@ + +if HAVE_SCRIPT +libfst_LTLIBRARIES = libfstfarscript.la libfstfar.la +else +libfst_LTLIBRARIES = libfstfar.la +endif + +libfstfar_la_SOURCES = sttable.cc stlist.cc +libfstfar_la_LDFLAGS = -version-info 0:0:0 + +if HAVE_SCRIPT +libfstfarscript_la_SOURCES = farscript.cc compile-strings.cc main.cc +libfstfarscript_la_LDFLAGS = -version-info 0:0:0 +endif + +if HAVE_BIN +bin_PROGRAMS = farcompilestrings farcreate farextract farinfo farprintstrings + +LDADD = libfstfarscript.la libfstfar.la ../../script/libfstscript.la \ + ../../lib/libfst.la -lm -ldl + +farcompilestrings_SOURCES = farcompilestrings.cc + +farcreate_SOURCES = farcreate.cc + +farextract_SOURCES = farextract.cc + +farinfo_SOURCES = farinfo.cc + +farprintstrings_SOURCES = farprintstrings.cc +endif diff --git a/src/extensions/far/Makefile.in b/src/extensions/far/Makefile.in new file mode 100644 index 0000000..8e05cab --- /dev/null +++ b/src/extensions/far/Makefile.in @@ -0,0 +1,673 @@ +# Makefile.in generated by automake 1.11.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +@HAVE_BIN_TRUE@bin_PROGRAMS = farcompilestrings$(EXEEXT) \ +@HAVE_BIN_TRUE@ farcreate$(EXEEXT) farextract$(EXEEXT) \ +@HAVE_BIN_TRUE@ farinfo$(EXEEXT) farprintstrings$(EXEEXT) +subdir = src/extensions/far +DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_icu.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h \ + $(top_builddir)/src/include/fst/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__installdirs = "$(DESTDIR)$(libfstdir)" "$(DESTDIR)$(bindir)" +LTLIBRARIES = $(libfst_LTLIBRARIES) +libfstfar_la_LIBADD = +am_libfstfar_la_OBJECTS = sttable.lo stlist.lo +libfstfar_la_OBJECTS = $(am_libfstfar_la_OBJECTS) +libfstfar_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ + $(CXXFLAGS) $(libfstfar_la_LDFLAGS) $(LDFLAGS) -o $@ +@HAVE_SCRIPT_FALSE@am_libfstfar_la_rpath = -rpath $(libfstdir) +@HAVE_SCRIPT_TRUE@am_libfstfar_la_rpath = -rpath $(libfstdir) +libfstfarscript_la_LIBADD = +am__libfstfarscript_la_SOURCES_DIST = farscript.cc compile-strings.cc \ + main.cc +@HAVE_SCRIPT_TRUE@am_libfstfarscript_la_OBJECTS = farscript.lo \ +@HAVE_SCRIPT_TRUE@ compile-strings.lo main.lo +libfstfarscript_la_OBJECTS = $(am_libfstfarscript_la_OBJECTS) +libfstfarscript_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ + $(CXXFLAGS) $(libfstfarscript_la_LDFLAGS) $(LDFLAGS) -o $@ +@HAVE_SCRIPT_TRUE@am_libfstfarscript_la_rpath = -rpath $(libfstdir) +PROGRAMS = $(bin_PROGRAMS) +am__farcompilestrings_SOURCES_DIST = farcompilestrings.cc +@HAVE_BIN_TRUE@am_farcompilestrings_OBJECTS = \ +@HAVE_BIN_TRUE@ farcompilestrings.$(OBJEXT) +farcompilestrings_OBJECTS = $(am_farcompilestrings_OBJECTS) +farcompilestrings_LDADD = $(LDADD) +@HAVE_BIN_TRUE@farcompilestrings_DEPENDENCIES = libfstfarscript.la \ +@HAVE_BIN_TRUE@ libfstfar.la ../../script/libfstscript.la \ +@HAVE_BIN_TRUE@ ../../lib/libfst.la +am__farcreate_SOURCES_DIST = farcreate.cc +@HAVE_BIN_TRUE@am_farcreate_OBJECTS = farcreate.$(OBJEXT) +farcreate_OBJECTS = $(am_farcreate_OBJECTS) +farcreate_LDADD = $(LDADD) +@HAVE_BIN_TRUE@farcreate_DEPENDENCIES = libfstfarscript.la \ +@HAVE_BIN_TRUE@ libfstfar.la ../../script/libfstscript.la \ +@HAVE_BIN_TRUE@ ../../lib/libfst.la +am__farextract_SOURCES_DIST = farextract.cc +@HAVE_BIN_TRUE@am_farextract_OBJECTS = farextract.$(OBJEXT) +farextract_OBJECTS = $(am_farextract_OBJECTS) +farextract_LDADD = $(LDADD) +@HAVE_BIN_TRUE@farextract_DEPENDENCIES = libfstfarscript.la \ +@HAVE_BIN_TRUE@ libfstfar.la ../../script/libfstscript.la \ +@HAVE_BIN_TRUE@ ../../lib/libfst.la +am__farinfo_SOURCES_DIST = farinfo.cc +@HAVE_BIN_TRUE@am_farinfo_OBJECTS = farinfo.$(OBJEXT) +farinfo_OBJECTS = $(am_farinfo_OBJECTS) +farinfo_LDADD = $(LDADD) +@HAVE_BIN_TRUE@farinfo_DEPENDENCIES = libfstfarscript.la libfstfar.la \ +@HAVE_BIN_TRUE@ ../../script/libfstscript.la \ +@HAVE_BIN_TRUE@ ../../lib/libfst.la +am__farprintstrings_SOURCES_DIST = farprintstrings.cc +@HAVE_BIN_TRUE@am_farprintstrings_OBJECTS = farprintstrings.$(OBJEXT) +farprintstrings_OBJECTS = $(am_farprintstrings_OBJECTS) +farprintstrings_LDADD = $(LDADD) +@HAVE_BIN_TRUE@farprintstrings_DEPENDENCIES = libfstfarscript.la \ +@HAVE_BIN_TRUE@ libfstfar.la ../../script/libfstscript.la \ +@HAVE_BIN_TRUE@ ../../lib/libfst.la +DEFAULT_INCLUDES = +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=link $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \ + $(LDFLAGS) -o $@ +SOURCES = $(libfstfar_la_SOURCES) $(libfstfarscript_la_SOURCES) \ + $(farcompilestrings_SOURCES) $(farcreate_SOURCES) \ + $(farextract_SOURCES) $(farinfo_SOURCES) \ + $(farprintstrings_SOURCES) +DIST_SOURCES = $(libfstfar_la_SOURCES) \ + $(am__libfstfarscript_la_SOURCES_DIST) \ + $(am__farcompilestrings_SOURCES_DIST) \ + $(am__farcreate_SOURCES_DIST) $(am__farextract_SOURCES_DIST) \ + $(am__farinfo_SOURCES_DIST) \ + $(am__farprintstrings_SOURCES_DIST) +ETAGS = etags +CTAGS = ctags +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +GREP = @GREP@ +ICU_CFLAGS = @ICU_CFLAGS@ +ICU_CONFIG = @ICU_CONFIG@ +ICU_CPPFLAGS = @ICU_CPPFLAGS@ +ICU_CXXFLAGS = @ICU_CXXFLAGS@ +ICU_LIBS = @ICU_LIBS@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +libfstdir = @libfstdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +lt_ECHO = @lt_ECHO@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +AM_CPPFLAGS = -I$(srcdir)/../../include $(ICU_CPPFLAGS) +@HAVE_SCRIPT_FALSE@libfst_LTLIBRARIES = libfstfar.la +@HAVE_SCRIPT_TRUE@libfst_LTLIBRARIES = libfstfarscript.la libfstfar.la +libfstfar_la_SOURCES = sttable.cc stlist.cc +libfstfar_la_LDFLAGS = -version-info 0:0:0 +@HAVE_SCRIPT_TRUE@libfstfarscript_la_SOURCES = farscript.cc compile-strings.cc main.cc +@HAVE_SCRIPT_TRUE@libfstfarscript_la_LDFLAGS = -version-info 0:0:0 +@HAVE_BIN_TRUE@LDADD = libfstfarscript.la libfstfar.la ../../script/libfstscript.la \ +@HAVE_BIN_TRUE@ ../../lib/libfst.la -lm -ldl + +@HAVE_BIN_TRUE@farcompilestrings_SOURCES = farcompilestrings.cc +@HAVE_BIN_TRUE@farcreate_SOURCES = farcreate.cc +@HAVE_BIN_TRUE@farextract_SOURCES = farextract.cc +@HAVE_BIN_TRUE@farinfo_SOURCES = farinfo.cc +@HAVE_BIN_TRUE@farprintstrings_SOURCES = farprintstrings.cc +all: all-am + +.SUFFIXES: +.SUFFIXES: .cc .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/extensions/far/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/extensions/far/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): +install-libfstLTLIBRARIES: $(libfst_LTLIBRARIES) + @$(NORMAL_INSTALL) + test -z "$(libfstdir)" || $(MKDIR_P) "$(DESTDIR)$(libfstdir)" + @list='$(libfst_LTLIBRARIES)'; test -n "$(libfstdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(libfstdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(libfstdir)"; \ + } + +uninstall-libfstLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(libfst_LTLIBRARIES)'; test -n "$(libfstdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(libfstdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(libfstdir)/$$f"; \ + done + +clean-libfstLTLIBRARIES: + -test -z "$(libfst_LTLIBRARIES)" || rm -f $(libfst_LTLIBRARIES) + @list='$(libfst_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +libfstfar.la: $(libfstfar_la_OBJECTS) $(libfstfar_la_DEPENDENCIES) + $(libfstfar_la_LINK) $(am_libfstfar_la_rpath) $(libfstfar_la_OBJECTS) $(libfstfar_la_LIBADD) $(LIBS) +libfstfarscript.la: $(libfstfarscript_la_OBJECTS) $(libfstfarscript_la_DEPENDENCIES) + $(libfstfarscript_la_LINK) $(am_libfstfarscript_la_rpath) $(libfstfarscript_la_OBJECTS) $(libfstfarscript_la_LIBADD) $(LIBS) +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)" + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p || test -f $$p1; \ + then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +clean-binPROGRAMS: + @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +farcompilestrings$(EXEEXT): $(farcompilestrings_OBJECTS) $(farcompilestrings_DEPENDENCIES) + @rm -f farcompilestrings$(EXEEXT) + $(CXXLINK) $(farcompilestrings_OBJECTS) $(farcompilestrings_LDADD) $(LIBS) +farcreate$(EXEEXT): $(farcreate_OBJECTS) $(farcreate_DEPENDENCIES) + @rm -f farcreate$(EXEEXT) + $(CXXLINK) $(farcreate_OBJECTS) $(farcreate_LDADD) $(LIBS) +farextract$(EXEEXT): $(farextract_OBJECTS) $(farextract_DEPENDENCIES) + @rm -f farextract$(EXEEXT) + $(CXXLINK) $(farextract_OBJECTS) $(farextract_LDADD) $(LIBS) +farinfo$(EXEEXT): $(farinfo_OBJECTS) $(farinfo_DEPENDENCIES) + @rm -f farinfo$(EXEEXT) + $(CXXLINK) $(farinfo_OBJECTS) $(farinfo_LDADD) $(LIBS) +farprintstrings$(EXEEXT): $(farprintstrings_OBJECTS) $(farprintstrings_DEPENDENCIES) + @rm -f farprintstrings$(EXEEXT) + $(CXXLINK) $(farprintstrings_OBJECTS) $(farprintstrings_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/compile-strings.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farcompilestrings.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farcreate.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farextract.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farinfo.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farprintstrings.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farscript.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stlist.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sttable.Plo@am__quote@ + +.cc.o: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cc.lo: +@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(PROGRAMS) +installdirs: + for dir in "$(DESTDIR)$(libfstdir)" "$(DESTDIR)$(bindir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-binPROGRAMS clean-generic clean-libfstLTLIBRARIES \ + clean-libtool mostlyclean-am + +distclean: distclean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-libfstLTLIBRARIES + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-binPROGRAMS + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-binPROGRAMS uninstall-libfstLTLIBRARIES + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS all all-am check check-am clean clean-binPROGRAMS \ + clean-generic clean-libfstLTLIBRARIES clean-libtool ctags \ + distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-binPROGRAMS \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-libfstLTLIBRARIES \ + install-man install-pdf install-pdf-am install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags uninstall \ + uninstall-am uninstall-binPROGRAMS uninstall-libfstLTLIBRARIES + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/extensions/far/compile-strings.cc b/src/extensions/far/compile-strings.cc new file mode 100644 index 0000000..e8a99cc --- /dev/null +++ b/src/extensions/far/compile-strings.cc @@ -0,0 +1,36 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#include +#include +#include + +namespace fst { + +// Compute the minimal length required to +// encode each line number as a decimal number +int KeySize(const char *filename) { + ifstream istrm(filename); + istrm.seekg(0); + string s; + int nline = 0; + while (getline(istrm, s)) + ++nline; + istrm.seekg(0); + return nline ? ceil(log10(nline + 1)) : 1; +} + +} // namespace fst diff --git a/src/extensions/far/farcompilestrings.cc b/src/extensions/far/farcompilestrings.cc new file mode 100644 index 0000000..0c9b352 --- /dev/null +++ b/src/extensions/far/farcompilestrings.cc @@ -0,0 +1,84 @@ +// farcompilestrings.cc + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: allauzen@google.com (Cyril Allauzen) +// Modified: jpr@google.com (Jake Ratkiewicz) to use new arc-type dispatching +// +// \file +// Compiles a set of stings as FSTs and stores them in a finite-state +// archive. +// + +#include +#include +#include +#include + +DEFINE_string(key_prefix, "", "Prefix to append to keys"); +DEFINE_string(key_suffix, "", "Suffix to append to keys"); +DEFINE_int32(generate_keys, 0, + "Generate N digit numeric keys (def: use file basenames)"); +DEFINE_string(far_type, "default", "FAR file format type: one of: "); +DEFINE_bool(allow_negative_labels, false, + "Allow negative labels (not recommended; may cause conflicts)"); +DEFINE_string(arc_type, "standard", "Output arc type"); +DEFINE_string(entry_type, "line", "Entry type: one of : " + "\"file\" (one FST per file), \"line\" (one FST per line)"); +DEFINE_string(fst_type, "vector", "Output FST type"); +DEFINE_string(token_type, "symbol", "Token type: one of : " + "\"symbol\", \"byte\", \"utf8\""); +DEFINE_string(symbols, "", "Label symbol table"); +DEFINE_string(unknown_symbol, "", ""); +DEFINE_bool(file_list_input, false, + "Each input files contains a list of files to be processed"); + + +int main(int argc, char **argv) { + namespace s = fst::script; + + string usage = "Compiles a set of strings as FSTs and stores them in"; + usage += " a finite-state archive.\n\n Usage:"; + usage += argv[0]; + usage += " in1.txt [in2.txt ...] out.far\n"; + + std::set_new_handler(FailedNewHandler); + SetFlags(usage.c_str(), &argc, &argv, true); + + if (argc < 3) { + ShowUsage(); + return 1; + } + + vector in_fnames(argc - 2); + + for (unsigned i = 1; i < argc - 1; ++i) { + in_fnames[i - 1] = argv[i]; + } + + string out_fname = argv[argc - 1]; + + fst::FarEntryType fet = fst::StringToFarEntryType(FLAGS_entry_type); + fst::FarTokenType ftt = fst::StringToFarTokenType(FLAGS_token_type); + fst::FarType far_type = fst::FarTypeFromString(FLAGS_far_type); + + s::FarCompileStrings(in_fnames, out_fname, FLAGS_arc_type, FLAGS_fst_type, + far_type, FLAGS_generate_keys, fet, ftt, + FLAGS_symbols, FLAGS_unknown_symbol, + FLAGS_allow_negative_labels, + FLAGS_file_list_input, FLAGS_key_prefix, + FLAGS_key_suffix); + + return 0; +} diff --git a/src/extensions/far/farcreate.cc b/src/extensions/far/farcreate.cc new file mode 100644 index 0000000..009fc34 --- /dev/null +++ b/src/extensions/far/farcreate.cc @@ -0,0 +1,62 @@ +// farcreate.cc + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: riley@google.com (Michael Riley) +// Modified: jpr@google.com (Jake Ratkiewicz) to use new dispatch +// +// \file +// Creates a finite-state archive from input FSTs. +// + +#include +#include +#include + +DEFINE_string(key_prefix, "", "Prefix to append to keys"); +DEFINE_string(key_suffix, "", "Suffix to append to keys"); +DEFINE_int32(generate_keys, 0, + "Generate N digit numeric keys (def: use file basenames)"); +DEFINE_string(far_type, "default", + "FAR file format type: one of: \"default\", \"stlist\", \"sstable\", \"sttable\""); +DEFINE_bool(file_list_input, false, + "Each input files contains a list of files to be processed"); + +int main(int argc, char **argv) { + namespace s = fst::script; + + string usage = "Creates a finite-state archive from input FSTs.\n\n Usage:"; + usage += argv[0]; + usage += " in1.fst [in2.fst ...] out.far\n"; + + std::set_new_handler(FailedNewHandler); + SetFlags(usage.c_str(), &argc, &argv, true); + + if (argc < 3) { + ShowUsage(); + return 1; + } + + vector in_fnames; + for (int i = 1; i < argc - 1; ++i) + in_fnames.push_back(argv[i]); + + string out_fname = argv[argc - 1]; + string arc_type = fst::LoadArcTypeFromFst(in_fnames[0]); + fst::FarType far_type = fst::FarTypeFromString(FLAGS_far_type); + + s::FarCreate(in_fnames, out_fname, arc_type, FLAGS_generate_keys, + FLAGS_file_list_input, far_type, FLAGS_key_prefix, + FLAGS_key_suffix); +} diff --git a/src/extensions/far/farextract.cc b/src/extensions/far/farextract.cc new file mode 100644 index 0000000..72b2d15 --- /dev/null +++ b/src/extensions/far/farextract.cc @@ -0,0 +1,62 @@ +// farextract.cc + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: riley@google.com (Michael Riley) +// Modified: jpr@google.com (Jake Ratkiewicz) to use new arc dispatch +// +// \file +// Extracts component FSTs from an finite-state archive. +// + +#include +#include + +DEFINE_string(filename_prefix, "", "Prefix to append to filenames"); +DEFINE_string(filename_suffix, "", "Suffix to append to filenames"); +DEFINE_int32(generate_filenames, 0, + "Generate N digit numeric filenames (def: use keys)"); +DEFINE_string(begin_key, "", + "First key to extract (def: first key in archive)"); +DEFINE_string(end_key, "", + "Last key to extract (def: last key in archive)"); + + +int main(int argc, char **argv) { + namespace s = fst::script; + + string usage = "Extracts FSTs from a finite-state archive.\n\n Usage:"; + usage += argv[0]; + usage += " in1.far [in2.far...]\n"; + + std::set_new_handler(FailedNewHandler); + SetFlags(usage.c_str(), &argc, &argv, true); + + if (argc < 2) { + ShowUsage(); + return 1; + } + + vector ifilenames; + for (int i = 1; i < argc; ++i) + ifilenames.push_back(argv[i]); + + const string &arc_type = fst::LoadArcTypeFromFar(ifilenames[0]); + + s::FarExtract(ifilenames, arc_type, FLAGS_generate_filenames, + FLAGS_begin_key, FLAGS_end_key, FLAGS_filename_prefix, + FLAGS_filename_suffix); + + return 0; +} diff --git a/src/extensions/far/farinfo.cc b/src/extensions/far/farinfo.cc new file mode 100644 index 0000000..e8036a9 --- /dev/null +++ b/src/extensions/far/farinfo.cc @@ -0,0 +1,56 @@ +// farinfo.cc + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: allauzen@google.com (Cyril Allauzen) +// Modified: jpr@google.com (Jake Ratkiewicz) to use new arc dispatching +// +// \file +// Prints some basic information about the FSTs in an FST archive. +// + +#include +#include + +DEFINE_string(begin_key, "", + "First key to extract (def: first key in archive)"); +DEFINE_string(end_key, "", + "Last key to extract (def: last key in archive)"); + +DEFINE_bool(list_fsts, false, "Display FST information for each key"); + +int main(int argc, char **argv) { + namespace s = fst::script; + + string usage = "Prints some basic information about the FSTs in an FST "; + usage += "archive.\n\n Usage:"; + usage += argv[0]; + usage += " in1.far [in2.far...]\n"; + usage += " Flags: begin_key end_key list_fsts"; + + std::set_new_handler(FailedNewHandler); + SetFlags(usage.c_str(), &argc, &argv, true); + + if (argc < 2) { + ShowUsage(); + return 1; + } + + vector filenames; + for (int i = 1; i < argc; ++i) + filenames.push_back(argv[i]); + + s::FarInfo(filenames, fst::LoadArcTypeFromFar(filenames[0]), + FLAGS_begin_key, FLAGS_end_key, FLAGS_list_fsts); +} diff --git a/src/extensions/far/farprintstrings.cc b/src/extensions/far/farprintstrings.cc new file mode 100644 index 0000000..ab7d52b --- /dev/null +++ b/src/extensions/far/farprintstrings.cc @@ -0,0 +1,70 @@ +// farprintstrings.cc + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: allauzen@google.com (Cyril Allauzen) +// +// \file +// Output as strings the string FSTs in a finite-state archive. +// + +#include + +DEFINE_string(filename_prefix, "", "Prefix to append to filenames"); +DEFINE_string(filename_suffix, "", "Suffix to append to filenames"); +DEFINE_int32(generate_filenames, 0, + "Generate N digit numeric filenames (def: use keys)"); +DEFINE_string(begin_key, "", + "First key to extract (def: first key in archive)"); +DEFINE_string(end_key, "", + "Last key to extract (def: last key in archive)"); +// PrintStringsMain specific flag definitions. +DEFINE_bool(print_key, false, "Prefix each string by its key"); +DEFINE_string(entry_type, "line", "Entry type: one of : " + "\"file\" (one FST per file), \"line\" (one FST per line)"); +DEFINE_string(token_type, "symbol", "Token type: one of : " + "\"symbol\", \"byte\", \"utf8\""); +DEFINE_string(symbols, "", "Label symbol table"); + + +int main(int argc, char **argv) { + namespace s = fst::script; + + string usage = "Print as string the string FSTs in an archive.\n\n Usage:"; + usage += argv[0]; + usage += " in1.far [in2.far ...]\n"; + + std::set_new_handler(FailedNewHandler); + SetFlags(usage.c_str(), &argc, &argv, true); + + if (argc < 2) { + ShowUsage(); + return 1; + } + + vector ifilenames; + for (int i = 1; i < argc; ++i) + ifilenames.push_back(argv[i]); + + string arc_type = fst::LoadArcTypeFromFar(ifilenames[0]); + + s::FarPrintStrings(ifilenames, arc_type, + fst::StringToFarEntryType(FLAGS_entry_type), + fst::StringToFarTokenType(FLAGS_token_type), + FLAGS_begin_key, FLAGS_end_key, FLAGS_print_key, + FLAGS_symbols, FLAGS_generate_filenames, + FLAGS_filename_prefix, FLAGS_filename_suffix); + + return 0; +} diff --git a/src/extensions/far/farscript.cc b/src/extensions/far/farscript.cc new file mode 100644 index 0000000..a04645d --- /dev/null +++ b/src/extensions/far/farscript.cc @@ -0,0 +1,113 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +// Definitions of 'scriptable' versions of FAR operations, that is, +// those that can be called with FstClass-type arguments. + +#include +#include +#include + +namespace fst { +namespace script { + +void FarCompileStrings(const vector &in_fnames, + const string &out_fname, + const string &arc_type, + const string &fst_type, + const FarType &far_type, + int32 generate_keys, + FarEntryType fet, + FarTokenType tt, + const string &symbols_fname, + const string &unknown_symbol, + bool allow_negative_labels, + bool file_list_input, + const string &key_prefix, + const string &key_suffix) { + FarCompileStringsArgs args(in_fnames, out_fname, fst_type, far_type, + generate_keys, fet, tt, symbols_fname, + unknown_symbol, allow_negative_labels, + file_list_input, key_prefix, key_suffix); + + Apply >("FarCompileStrings", arc_type, + &args); +} + +void FarCreate(const vector &in_fnames, + const string &out_fname, + const string &arc_type, + const int32 generate_keys, + const bool file_list_input, + const FarType &far_type, + const string &key_prefix, + const string &key_suffix) { + FarCreateArgs args(in_fnames, out_fname, generate_keys, file_list_input, + far_type, key_prefix, key_suffix); + + Apply >("FarCreate", arc_type, &args); +} + +void FarExtract(const vector &ifilenames, + const string &arc_type, + int32 generate_filenames, const string &begin_key, + const string &end_key, const string &filename_prefix, + const string &filename_suffix) { + FarExtractArgs args(ifilenames, generate_filenames, begin_key, end_key, + filename_prefix, filename_suffix); + + Apply >("FarExtract", arc_type, &args); +} + +void FarInfo(const vector &filenames, + const string &arc_type, + const string &begin_key, + const string &end_key, + const bool list_fsts) { + FarInfoArgs args(filenames, begin_key, end_key, list_fsts); + + Apply >("FarInfo", arc_type, &args); +} + +void FarPrintStrings(const vector &ifilenames, + const string &arc_type, + const FarEntryType entry_type, + const FarTokenType token_type, + const string &begin_key, + const string &end_key, + const bool print_key, + const string &symbols_fname, + const int32 generate_filenames, + const string &filename_prefix, + const string &filename_suffix) { + FarPrintStringsArgs args(ifilenames, entry_type, token_type, begin_key, + end_key, print_key, symbols_fname, + generate_filenames, + filename_prefix, + filename_suffix); + + Apply >("FarPrintStrings", arc_type, + &args); +} + +// Instantiate all templates for common arc types. + +REGISTER_FST_FAR_OPERATIONS(StdArc); +REGISTER_FST_FAR_OPERATIONS(LogArc); +REGISTER_FST_FAR_OPERATIONS(Log64Arc); + +} // namespace script +} // namespace fst diff --git a/src/extensions/far/main.cc b/src/extensions/far/main.cc new file mode 100644 index 0000000..b01d639 --- /dev/null +++ b/src/extensions/far/main.cc @@ -0,0 +1,118 @@ +// main.cc + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: riley@google.com (Michael Riley) +// Modified: jpr@google.com (Jake Ratkiewicz) to not use new arc-dispatch +// +// \file +// Definitions and functions for invoking and using Far main +// functions that support multiple and extensible arc types. + +#include +#include +using std::vector; + +#include +#include +#include + +namespace fst { + +// Return the 'FarType' value corresponding to a far type name. +FarType FarTypeFromString(const string &str) { + FarType type = FAR_DEFAULT; + if (str == "stlist") + type = FAR_STLIST; + else if (str == "sttable") + type = FAR_STTABLE; + else if (str == "default") + type = FAR_DEFAULT; + return type; +} + + +// Return the textual name corresponding to a 'FarType;. +string FarTypeToString(FarType type) { + switch (type) { + case FAR_STLIST: + return "stlist"; + case FAR_STTABLE: + return "sttable"; + case FAR_DEFAULT: + return "default"; + default: + return ""; + } +} + +FarEntryType StringToFarEntryType(const string &s) { + if (s == "line") { + return FET_LINE; + } else if (s == "file") { + return FET_FILE; + } else { + FSTERROR() << "Unknown FAR entry type: " << s; + return FET_LINE; // compiler requires return + } +} + +FarTokenType StringToFarTokenType(const string &s) { + if (s == "symbol") { + return FTT_SYMBOL; + } else if (s == "byte") { + return FTT_BYTE; + } else if (s == "utf8") { + return FTT_UTF8; + } else { + FSTERROR() << "Unknown FAR entry type: " << s; + return FTT_SYMBOL; // compiler requires return + } +} + + +string LoadArcTypeFromFar(const string &far_fname) { + FarHeader hdr; + + if (far_fname.empty()) { + LOG(ERROR) << "Reading FAR from standard in not supported"; + return ""; + } + + if (!hdr.Read(far_fname)) { + LOG(ERROR) << "Error reading FAR: " << far_fname; + return ""; + } + + string atype = hdr.ArcType(); + if (atype == "unknown") { + LOG(ERROR) << "Empty FST archive: " << far_fname; + return ""; + } + + return atype; +} + +string LoadArcTypeFromFst(const string &fst_fname) { + FstHeader hdr; + ifstream in(fst_fname.c_str(), ifstream::in | ifstream::binary); + if (!hdr.Read(in, fst_fname)) { + LOG(ERROR) << "Error reading FST: " << fst_fname; + return ""; + } + + return hdr.ArcType(); +} + +} // namespace fst diff --git a/src/extensions/far/stlist.cc b/src/extensions/far/stlist.cc new file mode 100644 index 0000000..f75b592 --- /dev/null +++ b/src/extensions/far/stlist.cc @@ -0,0 +1,31 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: allauzen@google.com (Cyril Allauzen) + +#include + +namespace fst { + +bool IsSTList(const string &filename) { + ifstream strm(filename.c_str()); + if (!strm) + return false; + + int32 magic_number = 0; + ReadType(strm, &magic_number); + return magic_number == kSTListMagicNumber; +} + +} // namespace fst diff --git a/src/extensions/far/sttable.cc b/src/extensions/far/sttable.cc new file mode 100644 index 0000000..76e91b8 --- /dev/null +++ b/src/extensions/far/sttable.cc @@ -0,0 +1,31 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: allauzen@google.com (Cyril Allauzen) + +#include + +namespace fst { + +bool IsSTTable(const string &filename) { + ifstream strm(filename.c_str()); + if (!strm) + return false; + + int32 magic_number = 0; + ReadType(strm, &magic_number); + return magic_number == kSTTableMagicNumber; +} + +} // namespace fst -- cgit v1.2.3