Updated OpenFST version to openfst-1.3.2-CL32004048 from Greco3.

Change-Id: I19b0db718256b35c0e3e5a7315f1ed6335e6dcac

5 files changed, 110 insertions, 47 deletions

diff --git a/src/lib/flags.cc b/src/lib/flags.cc
index d05fd73..222d66b 100644
--- a/src/lib/flags.cc
+++ b/src/lib/flags.cc
@@ -24,14 +24,18 @@
 
 DEFINE_int32(v, 0, "verbose level");
 DEFINE_bool(help, false, "show usage information");
+DEFINE_bool(helpshort, false, "show brief usage information");
 DEFINE_string(tmpdir, "/tmp/", "temporary directory");
 
 using namespace std;
 
 static string flag_usage;
+static string prog_src;
 
-void SetFlags(const char *usage, int *argc, char ***argv, bool remove_flags) {
+void SetFlags(const char *usage, int *argc, char ***argv,
+              bool remove_flags, const char *src) {
   flag_usage = usage;
+  prog_src = src;
   int index = 1;
   for (; index < *argc; ++index) {
     string argval = (*argv)[index];
@@ -82,22 +86,75 @@ void SetFlags(const char *usage, int *argc, char ***argv, bool remove_flags) {
   }
 
   if (FLAGS_help) {
-    ShowUsage();
+    ShowUsage(true);
+    exit(1);
+  }
+
+  if (FLAGS_helpshort) {
+    ShowUsage(false);
     exit(1);
   }
 }
 
-void ShowUsage() {
+// If flag is defined in file 'src' and 'in_src' true or is not
+// defined in file 'src' and 'in_src' is false, then print usage.
+static void
+ShowUsageRestrict(const std::set< pair<string, string> > &usage_set,
+		  const string &src, bool in_src, bool show_file) {
+  string old_file;
+  bool file_out = false;
+  bool usage_out = false;
+  for (std::set< pair<string, string> >::const_iterator it =
+	 usage_set.begin();
+       it != usage_set.end();
+       ++it) {
+    const string &file = it->first;
+    const string &usage = it->second;
+
+    bool match = file == src;
+    if ((match && !in_src) || (!match && in_src))
+      continue;
+
+    if (file != old_file) {
+      if (show_file) {
+        if (file_out) cout << "\n";
+	cout << "Flags from: " << file << "\n";
+        file_out = true;
+      }
+      old_file = file;
+    }
+    cout << usage << "\n";
+    usage_out = true;
+  }
+  if (usage_out) cout << "\n";
+}
+
+void ShowUsage(bool long_usage) {
+  std::set< pair<string, string> > usage_set;
+
   cout << flag_usage << "\n";
-  cout << "  Flags Description:\n";
+
   FlagRegister<bool> *bool_register = FlagRegister<bool>::GetRegister();
-  bool_register->ShowUsage();
+  bool_register->GetUsage(&usage_set);
   FlagRegister<string> *string_register = FlagRegister<string>::GetRegister();
-  string_register->ShowUsage();
+  string_register->GetUsage(&usage_set);
   FlagRegister<int32> *int32_register = FlagRegister<int32>::GetRegister();
-  int32_register->ShowUsage();
+  int32_register->GetUsage(&usage_set);
   FlagRegister<int64> *int64_register = FlagRegister<int64>::GetRegister();
-  int64_register->ShowUsage();
+  int64_register->GetUsage(&usage_set);
   FlagRegister<double> *double_register = FlagRegister<double>::GetRegister();
-  double_register->ShowUsage();
+  double_register->GetUsage(&usage_set);
+
+  if (!prog_src.empty()) {
+    cout << "PROGRAM FLAGS:\n\n";
+    ShowUsageRestrict(usage_set, prog_src, true, false);
+  }
+
+  if (!long_usage)
+    return;
+
+  if (!prog_src.empty())
+    cout << "LIBRARY FLAGS:\n\n";
+
+  ShowUsageRestrict(usage_set, prog_src, false, true);
 }
diff --git a/src/lib/properties.cc b/src/lib/properties.cc
index db0e2c8..9e03d59 100644
--- a/src/lib/properties.cc
+++ b/src/lib/properties.cc
@@ -117,8 +117,10 @@ uint64 DeterminizeProperties(uint64 inprops, bool has_subsequential_label) {
   uint64 outprops = kAccessible;
   if (((kAcceptor | kNoIEpsilons) & inprops) || has_subsequential_label)
     outprops |= kIDeterministic;
-  outprops |= (kError | kAcceptor | kNoEpsilons | kAcyclic |
+  outprops |= (kError | kAcceptor | kAcyclic |
                kInitialAcyclic | kCoAccessible | kString) & inprops;
+  if (inprops & kNoIEpsilons)
+    outprops |= kNoEpsilons & inprops;
   if (inprops & kAccessible)
      outprops |= (kNotAcceptor | kEpsilons | kIEpsilons | kOEpsilons |
                   kCyclic) & inprops;
@@ -375,6 +377,7 @@ uint64 UnionProperties(uint64 inprops1, uint64 inprops2, bool delayed) {
   uint64 outprops = (kAcceptor | kUnweighted | kAcyclic | kAccessible)
                     & inprops1 & inprops2;
   outprops |= kError & (inprops1 | inprops2);
+  outprops |= kInitialAcyclic;
 
   bool empty1 = delayed;  // Can fst1 be the empty machine?
   bool empty2 = delayed;  // Can fst2 be the empty machine?
diff --git a/src/lib/symbol-table.cc b/src/lib/symbol-table.cc
index 8b35cdf..a195a7c 100644
--- a/src/lib/symbol-table.cc
+++ b/src/lib/symbol-table.cc
@@ -20,6 +20,7 @@
 // Classes to provide symbol-to-integer and integer-to-symbol mappings.
 
 #include <fst/symbol-table.h>
+
 #include <fst/util.h>
 
 DEFINE_bool(fst_compat_symbols, true,
@@ -35,9 +36,12 @@ const int kLineLen = 8096;
 // Identifies stream data as a symbol table (and its endianity)
 static const int32 kSymbolTableMagicNumber = 2125658996;
 
+SymbolTableTextOptions::SymbolTableTextOptions()
+    : allow_negative(false), fst_field_separator(FLAGS_fst_field_separator) { }
+
 SymbolTableImpl* SymbolTableImpl::ReadText(istream &strm,
                                            const string &filename,
-                                           bool allow_negative) {
+                                           const SymbolTableTextOptions &opts) {
   SymbolTableImpl* impl = new SymbolTableImpl(filename);
 
   int64 nline = 0;
@@ -45,7 +49,7 @@ SymbolTableImpl* SymbolTableImpl::ReadText(istream &strm,
   while (strm.getline(line, kLineLen)) {
     ++nline;
     vector<char *> col;
-    string separator = FLAGS_fst_field_separator + "\n";
+    string separator = opts.fst_field_separator + "\n";
     SplitToVector(line, separator.c_str(), &col, true);
     if (col.size() == 0)  // empty line
       continue;
@@ -61,7 +65,7 @@ SymbolTableImpl* SymbolTableImpl::ReadText(istream &strm,
     char *p;
     int64 key = strtoll(value, &p, 10);
     if (p < value + strlen(value) ||
-        (!allow_negative && key < 0) || key == -1) {
+        (!opts.allow_negative && key < 0) || key == -1) {
       LOG(ERROR) << "SymbolTable::ReadText: Bad non-negative integer \""
                  << value << "\" (skipping), "
                  << "file = " << filename << ", line = " << nline;
@@ -74,21 +78,30 @@ SymbolTableImpl* SymbolTableImpl::ReadText(istream &strm,
 }
 
 void SymbolTableImpl::MaybeRecomputeCheckSum() const {
-  if (check_sum_finalized_)
-    return;
+  {
+    ReaderMutexLock check_sum_lock(&check_sum_mutex_);
+    if (check_sum_finalized_)
+      return;
+  }
+
+  // We'll aquire an exclusive lock to recompute the checksums.
+  MutexLock check_sum_lock(&check_sum_mutex_);
+  if (check_sum_finalized_)  // Another thread (coming in around the same time
+    return;                  // might have done it already).  So we recheck.
 
   // Calculate the original label-agnostic check sum.
-  check_sum_.Reset();
+  CheckSummer check_sum;
   for (int64 i = 0; i < symbols_.size(); ++i)
-    check_sum_.Update(symbols_[i], strlen(symbols_[i]) + 1);
-  check_sum_string_ = check_sum_.Digest();
+    check_sum.Update(symbols_[i], strlen(symbols_[i]) + 1);
+  check_sum_string_ = check_sum.Digest();
 
   // Calculate the safer, label-dependent check sum.
-  labeled_check_sum_.Reset();
+  CheckSummer labeled_check_sum;
   for (int64 key = 0; key < dense_key_limit_; ++key) {
     ostringstream line;
     line << symbols_[key] << '\t' << key;
-    labeled_check_sum_.Update(line.str()); }
+    labeled_check_sum.Update(line.str().data(), line.str().size());
+  }
   for (map<int64, const char*>::const_iterator it =
        key_map_.begin();
        it != key_map_.end();
@@ -96,10 +109,10 @@ void SymbolTableImpl::MaybeRecomputeCheckSum() const {
     if (it->first >= dense_key_limit_) {
       ostringstream line;
       line << it->second << '\t' << it->first;
-      labeled_check_sum_.Update(line.str());
+      labeled_check_sum.Update(line.str().data(), line.str().size());
     }
   }
-  labeled_check_sum_string_ = labeled_check_sum_.Digest();
+  labeled_check_sum_string_ = labeled_check_sum.Digest();
 
   check_sum_finalized_ = true;
 }
@@ -231,12 +244,22 @@ void SymbolTable::AddTable(const SymbolTable& table) {
     impl_->AddSymbol(iter.Symbol());
 }
 
-bool SymbolTable::WriteText(ostream &strm) const {
+bool SymbolTable::WriteText(ostream &strm,
+                            const SymbolTableTextOptions &opts) const {
+  if (opts.fst_field_separator.empty()) {
+    LOG(ERROR) << "Missing required field separator";
+    return false;
+  }
+  bool once_only = false;
   for (SymbolTableIterator iter(*this); !iter.Done(); iter.Next()) {
     ostringstream line;
-    line << iter.Symbol() << FLAGS_fst_field_separator[0] << iter.Value()
+    if (iter.Value() < 0 && !opts.allow_negative && !once_only) {
+      LOG(WARNING) << "Negative symbol table entry when not allowed";
+      once_only = true;
+    }
+    line << iter.Symbol() << opts.fst_field_separator[0] << iter.Value()
          << '\n';
-    strm.write(line.str().c_str(), line.str().length());
+    strm.write(line.str().data(), line.str().length());
   }
   return true;
 }
diff --git a/src/lib/temp_Android.temp_mk b/src/lib/temp_Android.temp_mk
deleted file mode 100644
index 7a6936e..0000000
--- a/src/lib/temp_Android.temp_mk
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Copyright 2012 Google Inc. All Rights Reserved.
-# Author: npereira@google.com (Nicole Pereira)
-#
-# Android makefile for openfst library.
-#
-
-OPENFST_DIR := $(call my-dir)
-
-LOCAL_PATH := $(call my-dir)
-include $(CLEAR_VARS)
-
-LOCAL_MODULE := libopenfst_patts
-
-LOCAL_CPP_EXTENSION := .cc
-LOCAL_C_INCLUDES += $(OPENFST_DIR)/src/include/
-
-#LOCAL_SRC_FILES := $(call private-function-all-cpp-files-under, src) 
-
-include $(BUILD_STATIC_LIBRARY)
diff --git a/src/lib/util.cc b/src/lib/util.cc
index eeba92b..f754da5 100644
--- a/src/lib/util.cc
+++ b/src/lib/util.cc
@@ -26,7 +26,7 @@
 
 DEFINE_bool(fst_error_fatal, true,
             "FST errors are fatal; o.w. return objects flagged as bad: "
-            " e.g., FSTs - kError prop. true, FST weights - not  a member()");
+            " e.g., FSTs - kError prop. true, FST weights - not  a Member()");
 
 namespace fst {
 
@@ -49,7 +49,7 @@ int64 StrToInt64(const string &s, const string &src, size_t nline,
 void Int64ToStr(int64 n, string *s) {
   ostringstream nstr;
   nstr << n;
-  *s = nstr.str();
+  s->append(nstr.str().data(), nstr.str().size());
 }
 
 void ConvertToLegalCSymbol(string *s) {