From 5b6dc79427b8f7eeb6a7ff68034ab8548ce670ea Mon Sep 17 00:00:00 2001
From: Alexander Gutkin <agutkin@google.com>
Date: Thu, 28 Feb 2013 00:24:20 +0000
Subject: Bumped OpenFST implementation to openfst-1.3.3-CL41851770.

Updated OpenFST implementation to the most recent version
used by Greco3 (corresponds to nlp::fst exported at Perforce
CL 41851770).

In particular this version has an improved PDT support.

Change-Id: I5aadfc962297eef73922c67e7d57866f11ee7d81
---
 src/include/fst/extensions/far/extract.h | 119 ++++++++++++++++++++++---------
 1 file changed, 87 insertions(+), 32 deletions(-)

(limited to 'src/include/fst/extensions/far/extract.h')

diff --git a/src/include/fst/extensions/far/extract.h b/src/include/fst/extensions/far/extract.h
index d6f92ff..95866de 100644
--- a/src/include/fst/extensions/far/extract.h
+++ b/src/include/fst/extensions/far/extract.h
@@ -31,52 +31,107 @@ using std::vector;
 
 namespace fst {
 
+template<class Arc>
+inline void FarWriteFst(const Fst<Arc>* fst, string key,
+                        string* okey, int* nrep,
+                        const int32 &generate_filenames, int i,
+                        const string &filename_prefix,
+                        const string &filename_suffix) {
+  if (key == *okey)
+    ++*nrep;
+  else
+    *nrep = 0;
+
+  *okey = key;
+
+  string ofilename;
+  if (generate_filenames) {
+    ostringstream tmp;
+    tmp.width(generate_filenames);
+    tmp.fill('0');
+    tmp << i;
+    ofilename = tmp.str();
+  } else {
+    if (*nrep > 0) {
+      ostringstream tmp;
+      tmp << '.' << nrep;
+      key.append(tmp.str().data(), tmp.str().size());
+    }
+    ofilename = key;
+  }
+  fst->Write(filename_prefix + ofilename + filename_suffix);
+}
+
 template<class Arc>
 void FarExtract(const vector<string> &ifilenames,
                 const int32 &generate_filenames,
-                const string &begin_key,
-                const string &end_key,
+                const string &keys,
+                const string &key_separator,
+                const string &range_delimiter,
                 const string &filename_prefix,
                 const string &filename_suffix) {
   FarReader<Arc> *far_reader = FarReader<Arc>::Open(ifilenames);
   if (!far_reader) return;
 
-  if (!begin_key.empty())
-    far_reader->Find(begin_key);
-
   string okey;
   int nrep = 0;
-  for (int i = 1; !far_reader->Done(); far_reader->Next(), ++i) {
-    string key = far_reader->GetKey();
-    if (!end_key.empty() && end_key < key)
-      break;
-    const Fst<Arc> &fst = far_reader->GetFst();
-
-    if (key == okey)
-      ++nrep;
-    else
-      nrep = 0;
 
-    okey = key;
-
-    string ofilename;
-    if (generate_filenames) {
-      ostringstream tmp;
-      tmp.width(generate_filenames);
-      tmp.fill('0');
-      tmp << i;
-      ofilename = tmp.str();
-    } else {
-      if (nrep > 0) {
-        ostringstream tmp;
-        tmp << '.' << nrep;
-        key.append(tmp.str().data(), tmp.str().size());
+  vector<char *> key_vector;
+  // User has specified a set of fsts to extract, where some of the "fsts" could
+  // be ranges.
+  if (!keys.empty()) {
+    char *keys_cstr = new char[keys.size()+1];
+    strcpy(keys_cstr, keys.c_str());
+    SplitToVector(keys_cstr, key_separator.c_str(), &key_vector, true);
+    int i = 0;
+    for (int k = 0; k < key_vector.size(); ++k, ++i) {
+      string key = string(key_vector[k]);
+      char *key_cstr = new char[key.size()+1];
+      strcpy(key_cstr, key.c_str());
+      vector<char *> range_vector;
+      SplitToVector(key_cstr, range_delimiter.c_str(), &range_vector, false);
+      if (range_vector.size() == 1) {  // Not a range
+        if (!far_reader->Find(key)) {
+          LOG(ERROR) << "FarExtract: Cannot find key: " << key;
+          return;
+        }
+        const Fst<Arc> &fst = far_reader->GetFst();
+        FarWriteFst(&fst, key, &okey, &nrep, generate_filenames, i,
+                    filename_prefix, filename_suffix);
+      } else if (range_vector.size() == 2) {  // A legal range
+        string begin_key = string(range_vector[0]);
+        string end_key = string(range_vector[1]);
+        if (begin_key.empty() || end_key.empty()) {
+          LOG(ERROR) << "FarExtract: Illegal range specification: " << key;
+          return;
+        }
+        if (!far_reader->Find(begin_key)) {
+          LOG(ERROR) << "FarExtract: Cannot find key: " << begin_key;
+          return;
+        }
+        for ( ; !far_reader->Done(); far_reader->Next(), ++i) {
+          string ikey = far_reader->GetKey();
+          if (end_key < ikey) break;
+          const Fst<Arc> &fst = far_reader->GetFst();
+          FarWriteFst(&fst, ikey, &okey, &nrep, generate_filenames, i,
+                      filename_prefix, filename_suffix);
+        }
+      } else {
+        LOG(ERROR) << "FarExtract: Illegal range specification: " << key;
+        return;
       }
-      ofilename = key;
+      delete key_cstr;
     }
-    fst.Write(filename_prefix + ofilename + filename_suffix);
+    delete keys_cstr;
+    return;
+  }
+  // Nothing specified: extract everything.
+  for (int i = 1; !far_reader->Done(); far_reader->Next(), ++i) {
+    string key = far_reader->GetKey();
+    const Fst<Arc> &fst = far_reader->GetFst();
+    FarWriteFst(&fst, key, &okey, &nrep, generate_filenames, i,
+                filename_prefix, filename_suffix);
   }
-
   return;
 }
 
-- 
cgit v1.2.3