1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
// compile.h
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright 2005-2010 Google, Inc.
// Author: riley@google.com (Michael Riley)
//
// \file
// Class to to compile a binary Fst from textual input.
#ifndef FST_SCRIPT_COMPILE_IMPL_H_
#define FST_SCRIPT_COMPILE_IMPL_H_
#include <unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
#include <sstream>
#include <string>
#include <vector>
using std::vector;
#include <iostream>
#include <fstream>
#include <fst/fst.h>
#include <fst/util.h>
#include <fst/vector-fst.h>
DECLARE_string(fst_field_separator);
namespace fst {
// Compile a binary Fst from textual input, helper class for fstcompile.cc
// WARNING: Stand-alone use of this class not recommended, most code should
// read/write using the binary format which is much more efficient.
template <class A> class FstCompiler {
public:
typedef A Arc;
typedef typename A::StateId StateId;
typedef typename A::Label Label;
typedef typename A::Weight Weight;
// WARNING: use of 'allow_negative_labels = true' not recommended; may
// cause conflicts
FstCompiler(istream &istrm, const string &source,
const SymbolTable *isyms, const SymbolTable *osyms,
const SymbolTable *ssyms, bool accep, bool ikeep,
bool okeep, bool nkeep, bool allow_negative_labels = false)
: nline_(0), source_(source),
isyms_(isyms), osyms_(osyms), ssyms_(ssyms),
nstates_(0), keep_state_numbering_(nkeep),
allow_negative_labels_(allow_negative_labels) {
char line[kLineLen];
while (istrm.getline(line, kLineLen)) {
++nline_;
vector<char *> col;
string separator = FLAGS_fst_field_separator + "\n";
SplitToVector(line, separator.c_str(), &col, true);
if (col.size() == 0 || col[0][0] == '\0') // empty line
continue;
if (col.size() > 5 ||
(col.size() > 4 && accep) ||
(col.size() == 3 && !accep)) {
FSTERROR() << "FstCompiler: Bad number of columns, source = "
<< source_
<< ", line = " << nline_;
fst_.SetProperties(kError, kError);
return;
}
StateId s = StrToStateId(col[0]);
while (s >= fst_.NumStates())
fst_.AddState();
if (nline_ == 1)
fst_.SetStart(s);
Arc arc;
StateId d = s;
switch (col.size()) {
case 1:
fst_.SetFinal(s, Weight::One());
break;
case 2:
fst_.SetFinal(s, StrToWeight(col[1], true));
break;
case 3:
arc.nextstate = d = StrToStateId(col[1]);
arc.ilabel = StrToILabel(col[2]);
arc.olabel = arc.ilabel;
arc.weight = Weight::One();
fst_.AddArc(s, arc);
break;
case 4:
arc.nextstate = d = StrToStateId(col[1]);
arc.ilabel = StrToILabel(col[2]);
if (accep) {
arc.olabel = arc.ilabel;
arc.weight = StrToWeight(col[3], false);
} else {
arc.olabel = StrToOLabel(col[3]);
arc.weight = Weight::One();
}
fst_.AddArc(s, arc);
break;
case 5:
arc.nextstate = d = StrToStateId(col[1]);
arc.ilabel = StrToILabel(col[2]);
arc.olabel = StrToOLabel(col[3]);
arc.weight = StrToWeight(col[4], false);
fst_.AddArc(s, arc);
}
while (d >= fst_.NumStates())
fst_.AddState();
}
if (ikeep)
fst_.SetInputSymbols(isyms);
if (okeep)
fst_.SetOutputSymbols(osyms);
}
const VectorFst<A> &Fst() const {
return fst_;
}
private:
// Maximum line length in text file.
static const int kLineLen = 8096;
int64 StrToId(const char *s, const SymbolTable *syms,
const char *name, bool allow_negative = false) const {
int64 n = 0;
if (syms) {
n = syms->Find(s);
if (n == -1 || (!allow_negative && n < 0)) {
FSTERROR() << "FstCompiler: Symbol \"" << s
<< "\" is not mapped to any integer " << name
<< ", symbol table = " << syms->Name()
<< ", source = " << source_ << ", line = " << nline_;
fst_.SetProperties(kError, kError);
}
} else {
char *p;
n = strtoll(s, &p, 10);
if (p < s + strlen(s) || (!allow_negative && n < 0)) {
FSTERROR() << "FstCompiler: Bad " << name << " integer = \"" << s
<< "\", source = " << source_ << ", line = " << nline_;
fst_.SetProperties(kError, kError);
}
}
return n;
}
StateId StrToStateId(const char *s) {
StateId n = StrToId(s, ssyms_, "state ID");
if (keep_state_numbering_)
return n;
// remap state IDs to make dense set
typename unordered_map<StateId, StateId>::const_iterator it = states_.find(n);
if (it == states_.end()) {
states_[n] = nstates_;
return nstates_++;
} else {
return it->second;
}
}
StateId StrToILabel(const char *s) const {
return StrToId(s, isyms_, "arc ilabel", allow_negative_labels_);
}
StateId StrToOLabel(const char *s) const {
return StrToId(s, osyms_, "arc olabel", allow_negative_labels_);
}
Weight StrToWeight(const char *s, bool allow_zero) const {
Weight w;
istringstream strm(s);
strm >> w;
if (!strm || (!allow_zero && w == Weight::Zero())) {
FSTERROR() << "FstCompiler: Bad weight = \"" << s
<< "\", source = " << source_ << ", line = " << nline_;
fst_.SetProperties(kError, kError);
w = Weight::NoWeight();
}
return w;
}
mutable VectorFst<A> fst_;
size_t nline_;
string source_; // text FST source name
const SymbolTable *isyms_; // ilabel symbol table
const SymbolTable *osyms_; // olabel symbol table
const SymbolTable *ssyms_; // slabel symbol table
unordered_map<StateId, StateId> states_; // state ID map
StateId nstates_; // number of seen states
bool keep_state_numbering_;
bool allow_negative_labels_; // not recommended; may cause conflicts
DISALLOW_COPY_AND_ASSIGN(FstCompiler);
};
} // namespace fst
#endif // FST_SCRIPT_COMPILE_IMPL_H_
|
