diff options
Diffstat (limited to 'src/include/fst/script')
50 files changed, 5111 insertions, 0 deletions
diff --git a/src/include/fst/script/arcsort.h b/src/include/fst/script/arcsort.h new file mode 100644 index 0000000..4277332 --- /dev/null +++ b/src/include/fst/script/arcsort.h @@ -0,0 +1,49 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_ARCSORT_H_ +#define FST_SCRIPT_ARCSORT_H_ + +#include <fst/arcsort.h> +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> + +namespace fst { +namespace script { + +enum ArcSortType { ILABEL_COMPARE, OLABEL_COMPARE }; + +typedef args::Package<MutableFstClass*, const ArcSortType> ArcSortArgs; + +template<class Arc> +void ArcSort(ArcSortArgs *args) { + MutableFst<Arc> *fst = args->arg1->GetMutableFst<Arc>(); + + if (args->arg2 == ILABEL_COMPARE) { + ILabelCompare<Arc> icomp; + ArcSort(fst, icomp); + } else { // OLABEL_COMPARE + OLabelCompare<Arc> ocomp; + ArcSort(fst, ocomp); + } +} + +void ArcSort(MutableFstClass *ofst, ArcSortType sort_type); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_ARCSORT_H_ diff --git a/src/include/fst/script/arg-packs.h b/src/include/fst/script/arg-packs.h new file mode 100644 index 0000000..8ebf8d8 --- /dev/null +++ b/src/include/fst/script/arg-packs.h @@ -0,0 +1,240 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +// Convenience templates for defining arg packs for the FstClass operations. + +// See operation-templates.h for a discussion about why these are needed; the +// short story is that all FstClass operations must be implemented by a version +// that takes one argument, most likely a struct bundling all the +// logical arguments together. These template structs provide convenient ways +// to specify these bundles (e.g. by means of appropriate typedefs). + +// The ArgPack template is sufficient for bundling together all the args for +// a particular function. The function is assumed to be void-returning. If +// you want a space for a return value, use the WithReturnValue template +// as follows: + +// WithReturnValue<bool, ArgPack<...> > + +#ifndef FST_SCRIPT_ARG_PACKS_H_ +#define FST_SCRIPT_ARG_PACKS_H_ + +namespace fst { +namespace script { +namespace args { + +// Sentinel value that means "no arg here." +class none_type { }; + +// Base arg pack template class. Specializations follow that allow +// fewer numbers of arguments (down to 2). If the maximum number of arguments +// increases, you will need to change three things: +// 1) Add more template parameters to this template +// 2) Add more specializations to allow fewer numbers of parameters than +// the new max. +// 3) Add extra none_types to all existing specializations to fill +// the new slots. + + +// 9 args (max) +template<class T1, + class T2 = none_type, + class T3 = none_type, + class T4 = none_type, + class T5 = none_type, + class T6 = none_type, + class T7 = none_type, + class T8 = none_type, + class T9 = none_type> +struct Package { + T1 arg1; + T2 arg2; + T3 arg3; + T4 arg4; + T5 arg5; + T6 arg6; + T7 arg7; + T8 arg8; + T9 arg9; + + Package(T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5, T6 arg6, + T7 arg7, T8 arg8, T9 arg9) : + arg1(arg1), arg2(arg2), arg3(arg3), arg4(arg4), arg5(arg5), + arg6(arg6), arg7(arg7), arg8(arg8), arg9(arg9) { } +}; + +// 8 args +template<class T1, + class T2, + class T3, + class T4, + class T5, + class T6, + class T7, + class T8> +struct Package<T1, T2, T3, T4, T5, T6, T7, T8, none_type> { + T1 arg1; + T2 arg2; + T3 arg3; + T4 arg4; + T5 arg5; + T6 arg6; + T7 arg7; + T8 arg8; + + Package(T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5, T6 arg6, + T7 arg7, T8 arg8) : + arg1(arg1), arg2(arg2), arg3(arg3), arg4(arg4), arg5(arg5), + arg6(arg6), arg7(arg7), arg8(arg8) { } +}; + +// 7 args +template<class T1, + class T2, + class T3, + class T4, + class T5, + class T6, + class T7> +struct Package<T1, T2, T3, T4, T5, T6, T7, + none_type, none_type> { + T1 arg1; + T2 arg2; + T3 arg3; + T4 arg4; + T5 arg5; + T6 arg6; + T7 arg7; + + Package(T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5, T6 arg6, + T7 arg7) : + arg1(arg1), arg2(arg2), arg3(arg3), arg4(arg4), arg5(arg5), + arg6(arg6), arg7(arg7) { } +}; + +// 6 args +template<class T1, + class T2, + class T3, + class T4, + class T5, + class T6> +struct Package<T1, T2, T3, T4, T5, T6, none_type, + none_type, none_type> { + T1 arg1; + T2 arg2; + T3 arg3; + T4 arg4; + T5 arg5; + T6 arg6; + + Package(T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5, T6 arg6) : + arg1(arg1), arg2(arg2), arg3(arg3), arg4(arg4), arg5(arg5), + arg6(arg6) { } +}; + +// 5 args +template<class T1, + class T2, + class T3, + class T4, + class T5> +struct Package<T1, T2, T3, T4, T5, none_type, none_type, + none_type, none_type> { + T1 arg1; + T2 arg2; + T3 arg3; + T4 arg4; + T5 arg5; + + Package(T1 arg1, T2 arg2, T3 arg3, T4 arg4, T5 arg5) : + arg1(arg1), arg2(arg2), arg3(arg3), arg4(arg4), arg5(arg5) { } +}; + +// 4 args +template<class T1, + class T2, + class T3, + class T4> +struct Package<T1, T2, T3, T4, none_type, none_type, + none_type, none_type, none_type> { + T1 arg1; + T2 arg2; + T3 arg3; + T4 arg4; + + Package(T1 arg1, T2 arg2, T3 arg3, T4 arg4) : + arg1(arg1), arg2(arg2), arg3(arg3), arg4(arg4) { } +}; + +// 3 args +template<class T1, + class T2, + class T3> +struct Package<T1, T2, T3, none_type, none_type, + none_type, none_type, none_type, + none_type> { + T1 arg1; + T2 arg2; + T3 arg3; + + Package(T1 arg1, T2 arg2, T3 arg3) : + arg1(arg1), arg2(arg2), arg3(arg3) { } +}; + +// 2 args (minimum) +template<class T1, + class T2> +struct Package<T1, T2, none_type, none_type, + none_type, none_type, none_type, + none_type, none_type> { + T1 arg1; + T2 arg2; + + Package(T1 arg1, T2 arg2) : + arg1(arg1), arg2(arg2) { } +}; + +// Tack this on to an existing arg pack to add a return value. +// The syntax for accessing the args is then slightly more stilted, +// as you must do an extra member access (since the args are stored +// as a member of this class). +// The alternative is to declare another slew of templates for functions +// that return a value, analogous to the above. + +template<class Retval, class ArgPackage> +struct WithReturnValue { + Retval retval; + const ArgPackage &args; + + explicit WithReturnValue(const ArgPackage &args) : args(args) { } +}; + +// We don't want to store a reference to a reference, if ArgPackage is +// already some reference type. +template<class Retval, class ArgPackage> +struct WithReturnValue<Retval, ArgPackage&> { + Retval retval; + const ArgPackage &args; + + explicit WithReturnValue(const ArgPackage &args) : args(args) { } +}; + +} // namespace args +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_ARG_PACKS_H_ diff --git a/src/include/fst/script/closure.h b/src/include/fst/script/closure.h new file mode 100644 index 0000000..93b5ec3 --- /dev/null +++ b/src/include/fst/script/closure.h @@ -0,0 +1,41 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_CLOSURE_H_ +#define FST_SCRIPT_CLOSURE_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/closure.h> + +namespace fst { +namespace script { + +typedef args::Package<MutableFstClass*, const ClosureType> ClosureArgs; + +template<class Arc> +void Closure(ClosureArgs *args) { + MutableFst<Arc> *fst = args->arg1->GetMutableFst<Arc>(); + + Closure(fst, args->arg2); +} + +void Closure(MutableFstClass *ofst, ClosureType closure_type); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_CLOSURE_H_ diff --git a/src/include/fst/script/compile-impl.h b/src/include/fst/script/compile-impl.h new file mode 100644 index 0000000..4aab15b --- /dev/null +++ b/src/include/fst/script/compile-impl.h @@ -0,0 +1,215 @@ +// compile.h + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: riley@google.com (Michael Riley) +// +// \file +// Class to to compile a binary Fst from textual input. + +#ifndef FST_SCRIPT_COMPILE_IMPL_H_ +#define FST_SCRIPT_COMPILE_IMPL_H_ + +#include <unordered_map> +using std::tr1::unordered_map; +using std::tr1::unordered_multimap; +#include <sstream> +#include <string> +#include <vector> +using std::vector; + +#include <iostream> +#include <fstream> +#include <fst/fst.h> +#include <fst/util.h> +#include <fst/vector-fst.h> + +DECLARE_string(fst_field_separator); + +namespace fst { + +// Compile a binary Fst from textual input, helper class for fstcompile.cc +// WARNING: Stand-alone use of this class not recommended, most code should +// read/write using the binary format which is much more efficient. +template <class A> class FstCompiler { + public: + typedef A Arc; + typedef typename A::StateId StateId; + typedef typename A::Label Label; + typedef typename A::Weight Weight; + + // WARNING: use of 'allow_negative_labels = true' not recommended; may + // cause conflicts + FstCompiler(istream &istrm, const string &source, + const SymbolTable *isyms, const SymbolTable *osyms, + const SymbolTable *ssyms, bool accep, bool ikeep, + bool okeep, bool nkeep, bool allow_negative_labels = false) + : nline_(0), source_(source), + isyms_(isyms), osyms_(osyms), ssyms_(ssyms), + nstates_(0), keep_state_numbering_(nkeep), + allow_negative_labels_(allow_negative_labels) { + char line[kLineLen]; + while (istrm.getline(line, kLineLen)) { + ++nline_; + vector<char *> col; + string separator = FLAGS_fst_field_separator + "\n"; + SplitToVector(line, separator.c_str(), &col, true); + if (col.size() == 0 || col[0][0] == '\0') // empty line + continue; + if (col.size() > 5 || + (col.size() > 4 && accep) || + (col.size() == 3 && !accep)) { + FSTERROR() << "FstCompiler: Bad number of columns, source = " + << source_ + << ", line = " << nline_; + fst_.SetProperties(kError, kError); + return; + } + StateId s = StrToStateId(col[0]); + while (s >= fst_.NumStates()) + fst_.AddState(); + if (nline_ == 1) + fst_.SetStart(s); + + Arc arc; + StateId d = s; + switch (col.size()) { + case 1: + fst_.SetFinal(s, Weight::One()); + break; + case 2: + fst_.SetFinal(s, StrToWeight(col[1], true)); + break; + case 3: + arc.nextstate = d = StrToStateId(col[1]); + arc.ilabel = StrToILabel(col[2]); + arc.olabel = arc.ilabel; + arc.weight = Weight::One(); + fst_.AddArc(s, arc); + break; + case 4: + arc.nextstate = d = StrToStateId(col[1]); + arc.ilabel = StrToILabel(col[2]); + if (accep) { + arc.olabel = arc.ilabel; + arc.weight = StrToWeight(col[3], false); + } else { + arc.olabel = StrToOLabel(col[3]); + arc.weight = Weight::One(); + } + fst_.AddArc(s, arc); + break; + case 5: + arc.nextstate = d = StrToStateId(col[1]); + arc.ilabel = StrToILabel(col[2]); + arc.olabel = StrToOLabel(col[3]); + arc.weight = StrToWeight(col[4], false); + fst_.AddArc(s, arc); + } + while (d >= fst_.NumStates()) + fst_.AddState(); + } + if (ikeep) + fst_.SetInputSymbols(isyms); + if (okeep) + fst_.SetOutputSymbols(osyms); + } + + const VectorFst<A> &Fst() const { + return fst_; + } + + private: + // Maximum line length in text file. + static const int kLineLen = 8096; + + int64 StrToId(const char *s, const SymbolTable *syms, + const char *name, bool allow_negative = false) const { + int64 n = 0; + + if (syms) { + n = syms->Find(s); + if (n == -1 || (!allow_negative && n < 0)) { + FSTERROR() << "FstCompiler: Symbol \"" << s + << "\" is not mapped to any integer " << name + << ", symbol table = " << syms->Name() + << ", source = " << source_ << ", line = " << nline_; + fst_.SetProperties(kError, kError); + } + } else { + char *p; + n = strtoll(s, &p, 10); + if (p < s + strlen(s) || (!allow_negative && n < 0)) { + FSTERROR() << "FstCompiler: Bad " << name << " integer = \"" << s + << "\", source = " << source_ << ", line = " << nline_; + fst_.SetProperties(kError, kError); + } + } + return n; + } + + StateId StrToStateId(const char *s) { + StateId n = StrToId(s, ssyms_, "state ID"); + + if (keep_state_numbering_) + return n; + + // remap state IDs to make dense set + typename unordered_map<StateId, StateId>::const_iterator it = states_.find(n); + if (it == states_.end()) { + states_[n] = nstates_; + return nstates_++; + } else { + return it->second; + } + } + + StateId StrToILabel(const char *s) const { + return StrToId(s, isyms_, "arc ilabel", allow_negative_labels_); + } + + StateId StrToOLabel(const char *s) const { + return StrToId(s, osyms_, "arc olabel", allow_negative_labels_); + } + + Weight StrToWeight(const char *s, bool allow_zero) const { + Weight w; + istringstream strm(s); + strm >> w; + if (!strm || (!allow_zero && w == Weight::Zero())) { + FSTERROR() << "FstCompiler: Bad weight = \"" << s + << "\", source = " << source_ << ", line = " << nline_; + fst_.SetProperties(kError, kError); + w = Weight::NoWeight(); + } + return w; + } + + mutable VectorFst<A> fst_; + size_t nline_; + string source_; // text FST source name + const SymbolTable *isyms_; // ilabel symbol table + const SymbolTable *osyms_; // olabel symbol table + const SymbolTable *ssyms_; // slabel symbol table + unordered_map<StateId, StateId> states_; // state ID map + StateId nstates_; // number of seen states + bool keep_state_numbering_; + bool allow_negative_labels_; // not recommended; may cause conflicts + + DISALLOW_COPY_AND_ASSIGN(FstCompiler); +}; + +} // namespace fst + +#endif // FST_SCRIPT_COMPILE_IMPL_H_ diff --git a/src/include/fst/script/compile.h b/src/include/fst/script/compile.h new file mode 100644 index 0000000..bb6ea56 --- /dev/null +++ b/src/include/fst/script/compile.h @@ -0,0 +1,92 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_COMPILE_H_ +#define FST_SCRIPT_COMPILE_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/compile-impl.h> + +namespace fst { +namespace script { + +// Note: it is safe to pass these strings as references because +// this struct is only used to pass them deeper in the call graph. +// Be sure you understand why this is so before using this struct +// for anything else! +struct FstCompileArgs { + fst::istream &istrm; + const string &source; + const string &dest; + const string &fst_type; + const fst::SymbolTable *isyms; + const fst::SymbolTable *osyms; + const fst::SymbolTable *ssyms; + const bool accep; + const bool ikeep; + const bool okeep; + const bool nkeep; + const bool allow_negative_labels; + + FstCompileArgs(istream &istrm, const string &source, const string &dest, + const string &fst_type, const fst::SymbolTable *isyms, + const fst::SymbolTable *osyms, + const fst::SymbolTable *ssyms, + bool accep, bool ikeep, bool okeep, bool nkeep, + bool allow_negative_labels = false) : + istrm(istrm), source(source), dest(dest), fst_type(fst_type), + isyms(isyms), osyms(osyms), ssyms(ssyms), accep(accep), ikeep(ikeep), + okeep(okeep), nkeep(nkeep), + allow_negative_labels(allow_negative_labels) { } +}; + +template<class Arc> +void CompileFst(FstCompileArgs *args) { + using fst::FstCompiler; + using fst::Convert; + using fst::Fst; + + FstCompiler<Arc> fstcompiler(args->istrm, args->source, args->isyms, + args->osyms, args->ssyms, + args->accep, args->ikeep, + args->okeep, args->nkeep, + args->allow_negative_labels); + + const Fst<Arc> *fst = &fstcompiler.Fst(); + if (args->fst_type != "vector") { + fst = Convert<Arc>(*fst, args->fst_type); + if (!fst) { + FSTERROR() << "Failed to convert FST to desired type: " + << args->fst_type; + return; + } + } + + fst->Write(args->dest); +} + +void CompileFst(istream &istrm, const string &source, const string &dest, + const string &fst_type, const string &arc_type, + const SymbolTable *isyms, + const SymbolTable *osyms, const SymbolTable *ssyms, + bool accep, bool ikeep, bool okeep, bool nkeep, + bool allow_negative_labels); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_COMPILE_H_ diff --git a/src/include/fst/script/compose.h b/src/include/fst/script/compose.h new file mode 100644 index 0000000..96375f7 --- /dev/null +++ b/src/include/fst/script/compose.h @@ -0,0 +1,63 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_COMPOSE_H_ +#define FST_SCRIPT_COMPOSE_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/compose.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass&, const FstClass&, + MutableFstClass*, ComposeFilter> ComposeArgs1; + +template<class Arc> +void Compose(ComposeArgs1 *args) { + const Fst<Arc> &ifst1 = *(args->arg1.GetFst<Arc>()); + const Fst<Arc> &ifst2 = *(args->arg2.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg3->GetMutableFst<Arc>(); + + Compose(ifst1, ifst2, ofst, args->arg4); +} + +typedef fst::ComposeOptions ComposeOptions; + +typedef args::Package<const FstClass&, const FstClass&, + MutableFstClass*, const ComposeOptions &> ComposeArgs2; + +template<class Arc> +void Compose(ComposeArgs2 *args) { + const Fst<Arc> &ifst1 = *(args->arg1.GetFst<Arc>()); + const Fst<Arc> &ifst2 = *(args->arg2.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg3->GetMutableFst<Arc>(); + + Compose(ifst1, ifst2, ofst, args->arg4); +} + +void Compose(const FstClass &ifst1, const FstClass &ifst2, + MutableFstClass *ofst, + const ComposeOptions &opts = fst::script::ComposeOptions()); + +void Compose(const FstClass &ifst1, const FstClass &ifst2, + MutableFstClass *ofst, ComposeFilter compose_filter); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_COMPOSE_H_ diff --git a/src/include/fst/script/concat.h b/src/include/fst/script/concat.h new file mode 100644 index 0000000..46c4407 --- /dev/null +++ b/src/include/fst/script/concat.h @@ -0,0 +1,54 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_CONCAT_H_ +#define FST_SCRIPT_CONCAT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/concat.h> + +namespace fst { +namespace script { + +typedef args::Package<MutableFstClass*, const FstClass&> ConcatArgs1; +typedef args::Package<const FstClass&, MutableFstClass*> ConcatArgs2; + +template<class Arc> +void Concat(ConcatArgs1 *args) { + MutableFst<Arc> *ofst = args->arg1->GetMutableFst<Arc>(); + const Fst<Arc> &ifst = *(args->arg2.GetFst<Arc>()); + + Concat(ofst, ifst); +} + +template<class Arc> +void Concat(ConcatArgs2 *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + + Concat(ifst, ofst); +} + +void Concat(MutableFstClass *ofst, const FstClass &ifst); +void Concat(const FstClass &ifst, MutableFstClass *ofst); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_CONCAT_H_ diff --git a/src/include/fst/script/connect.h b/src/include/fst/script/connect.h new file mode 100644 index 0000000..19c4390 --- /dev/null +++ b/src/include/fst/script/connect.h @@ -0,0 +1,45 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_CONNECT_H_ +#define FST_SCRIPT_CONNECT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/dfs-visit.h> +#include <fst/connect.h> + +namespace fst { +namespace script { + +// This function confuses SWIG, because both versions have the same args +#ifndef SWIG +template<class Arc> +void Connect(MutableFstClass *fst) { + MutableFst<Arc> *typed_fst = fst->GetMutableFst<Arc>(); + + Connect(typed_fst); +} +#endif + +void Connect(MutableFstClass *fst); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_CONNECT_H_ diff --git a/src/include/fst/script/convert.h b/src/include/fst/script/convert.h new file mode 100644 index 0000000..2c70a70 --- /dev/null +++ b/src/include/fst/script/convert.h @@ -0,0 +1,49 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_CONVERT_H_ +#define FST_SCRIPT_CONVERT_H_ + +#include <string> + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass&, const string&> ConvertInnerArgs; +typedef args::WithReturnValue<FstClass*, ConvertInnerArgs> ConvertArgs; + +template<class Arc> +void Convert(ConvertArgs *args) { + const Fst<Arc> &fst = *(args->args.arg1.GetFst<Arc>()); + const string &new_type = args->args.arg2; + + Fst<Arc> *result = Convert(fst, new_type); + args->retval = new FstClass(result); + delete result; +} + +#ifdef SWIG +%newobject Convert; +#endif +FstClass *Convert(const FstClass& f, const string &new_type); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_CONVERT_H_ diff --git a/src/include/fst/script/decode.h b/src/include/fst/script/decode.h new file mode 100644 index 0000000..1064ad5 --- /dev/null +++ b/src/include/fst/script/decode.h @@ -0,0 +1,46 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_DECODE_H_ +#define FST_SCRIPT_DECODE_H_ + +#include <string> + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/encode.h> + +namespace fst { +namespace script { + +typedef args::Package<MutableFstClass*, const string&> DecodeArgs; + +template<class Arc> +void Decode(DecodeArgs *args) { + MutableFst<Arc> *ofst = args->arg1->GetMutableFst<Arc>(); + + EncodeMapper<Arc> *decoder = EncodeMapper<Arc>::Read(args->arg2, DECODE); + Decode(ofst, *decoder); + + delete decoder; +} + +void Decode(MutableFstClass *fst, const string &coder_fname); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_DECODE_H_ diff --git a/src/include/fst/script/determinize.h b/src/include/fst/script/determinize.h new file mode 100644 index 0000000..38fd7ad --- /dev/null +++ b/src/include/fst/script/determinize.h @@ -0,0 +1,68 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_DETERMINIZE_H_ +#define FST_SCRIPT_DETERMINIZE_H_ + +#include <fst/determinize.h> +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> + +namespace fst { +namespace script { + +struct DeterminizeOptions { + float delta; + WeightClass weight_threshold; + int64 state_threshold; + int64 subsequential_label; + + explicit DeterminizeOptions(float d = fst::kDelta, + WeightClass w = + fst::script::WeightClass::Zero(), + int64 n = fst::kNoStateId, int64 l = 0) + : delta(d), weight_threshold(w), state_threshold(n), + subsequential_label(l) {} +}; + +typedef args::Package<const FstClass&, MutableFstClass*, + const DeterminizeOptions &> DeterminizeArgs; + +template<class Arc> +void Determinize(DeterminizeArgs *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + const DeterminizeOptions &opts = args->arg3; + + fst::DeterminizeOptions<Arc> detargs; + detargs.delta = opts.delta; + detargs.weight_threshold = + *(opts.weight_threshold.GetWeight<typename Arc::Weight>()); + detargs.state_threshold = opts.state_threshold; + detargs.subsequential_label = opts.subsequential_label; + + Determinize(ifst, ofst, detargs); +} + +void Determinize(const FstClass &ifst, MutableFstClass *ofst, + const DeterminizeOptions &opts = + fst::script::DeterminizeOptions()); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_DETERMINIZE_H_ diff --git a/src/include/fst/script/difference.h b/src/include/fst/script/difference.h new file mode 100644 index 0000000..76490d4 --- /dev/null +++ b/src/include/fst/script/difference.h @@ -0,0 +1,67 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_DIFFERENCE_H_ +#define FST_SCRIPT_DIFFERENCE_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/compose.h> // for ComposeFilter +#include <fst/difference.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass&, const FstClass&, + MutableFstClass*, ComposeFilter> DifferenceArgs1; + +template<class Arc> +void Difference(DifferenceArgs1 *args) { + const Fst<Arc> &ifst1 = *(args->arg1.GetFst<Arc>()); + const Fst<Arc> &ifst2 = *(args->arg2.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg3->GetMutableFst<Arc>(); + + Difference(ifst1, ifst2, ofst, args->arg4); +} + +typedef args::Package<const FstClass&, const FstClass&, + MutableFstClass*, const ComposeOptions &> DifferenceArgs2; + +template<class Arc> +void Difference(DifferenceArgs2 *args) { + const Fst<Arc> &ifst1 = *(args->arg1.GetFst<Arc>()); + const Fst<Arc> &ifst2 = *(args->arg2.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg3->GetMutableFst<Arc>(); + + Difference(ifst1, ifst2, ofst, args->arg4); +} + + +void Difference(const FstClass &ifst1, const FstClass &ifst2, + MutableFstClass *ofst, + ComposeFilter compose_filter); + +void Difference(const FstClass &ifst1, const FstClass &ifst2, + MutableFstClass *ofst, + const ComposeOptions &opts = fst::script::ComposeOptions()); + + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_DIFFERENCE_H_ diff --git a/src/include/fst/script/draw-impl.h b/src/include/fst/script/draw-impl.h new file mode 100644 index 0000000..e346649 --- /dev/null +++ b/src/include/fst/script/draw-impl.h @@ -0,0 +1,234 @@ +// draw.h + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: allauzen@google.com (Cyril Allauzen) +// +// \file +// Class to draw a binary FST by producing a text file in dot format, +// helper class to fstdraw.cc + +#ifndef FST_SCRIPT_DRAW_IMPL_H_ +#define FST_SCRIPT_DRAW_IMPL_H_ + +#include <sstream> +#include <string> + +#include <fst/script/fst-class.h> +#include <fst/fst.h> +#include <fst/util.h> + +namespace fst { + +// Print a binary Fst in the dot textual format, helper class for fstdraw.cc +// WARNING: Stand-alone use not recommend. +template <class A> class FstDrawer { + public: + typedef A Arc; + typedef typename A::StateId StateId; + typedef typename A::Label Label; + typedef typename A::Weight Weight; + + FstDrawer(const Fst<A> &fst, + const SymbolTable *isyms, + const SymbolTable *osyms, + const SymbolTable *ssyms, + bool accep, + string title, + float width, + float height, + bool portrait, + bool vertical, + float ranksep, + float nodesep, + int fontsize, + int precision, + bool show_weight_one) + : fst_(fst), isyms_(isyms), osyms_(osyms), ssyms_(ssyms), + accep_(accep && fst.Properties(kAcceptor, true)), ostrm_(0), + title_(title), width_(width), height_(height), portrait_(portrait), + vertical_(vertical), ranksep_(ranksep), nodesep_(nodesep), + fontsize_(fontsize), precision_(precision), + show_weight_one_(show_weight_one) {} + + // Draw Fst to an output buffer (or stdout if buf = 0) + void Draw(ostream *strm, const string &dest) { + ostrm_ = strm; + dest_ = dest; + StateId start = fst_.Start(); + if (start == kNoStateId) + return; + + PrintString("digraph FST {\n"); + if (vertical_) + PrintString("rankdir = BT;\n"); + else + PrintString("rankdir = LR;\n"); + PrintString("size = \""); + Print(width_); + PrintString(","); + Print(height_); + PrintString("\";\n"); + if (!dest_.empty()) + PrintString("label = \"" + title_ + "\";\n"); + PrintString("center = 1;\n"); + if (portrait_) + PrintString("orientation = Portrait;\n"); + else + PrintString("orientation = Landscape;\n"); + PrintString("ranksep = \""); + Print(ranksep_); + PrintString("\";\n"); + PrintString("nodesep = \""); + Print(nodesep_); + PrintString("\";\n"); + // initial state first + DrawState(start); + for (StateIterator< Fst<A> > siter(fst_); + !siter.Done(); + siter.Next()) { + StateId s = siter.Value(); + if (s != start) + DrawState(s); + } + PrintString("}\n"); + } + + private: + // Maximum line length in text file. + static const int kLineLen = 8096; + + void PrintString(const string &s) const { + *ostrm_ << s; + } + + // Escapes backslash and double quote if these occur in the string. Dot will + // not deal gracefully with these if they are not escaped. + inline void EscapeChars(const string &s, string* ns) const { + const char* c = s.c_str(); + while (*c) { + if (*c == '\\' || *c == '"') ns->push_back('\\'); + ns->push_back(*c); + ++c; + } + } + + void PrintId(int64 id, const SymbolTable *syms, + const char *name) const { + if (syms) { + string symbol = syms->Find(id); + if (symbol == "") { + FSTERROR() << "FstDrawer: Integer " << id + << " is not mapped to any textual symbol" + << ", symbol table = " << syms->Name() + << ", destination = " << dest_; + symbol = "?"; + } + string nsymbol; + EscapeChars(symbol, &nsymbol); + PrintString(nsymbol); + } else { + ostringstream sid; + sid << id; + PrintString(sid.str()); + } + } + + void PrintStateId(StateId s) const { + PrintId(s, ssyms_, "state ID"); + } + + void PrintILabel(Label l) const { + PrintId(l, isyms_, "arc input label"); + } + + void PrintOLabel(Label l) const { + PrintId(l, osyms_, "arc output label"); + } + + template <class T> + void Print(T t) const { + *ostrm_ << t; + } + + void DrawState(StateId s) const { + Print(s); + PrintString(" [label = \""); + PrintStateId(s); + Weight final = fst_.Final(s); + if (final != Weight::Zero()) { + if (show_weight_one_ || (final != Weight::One())) { + PrintString("/"); + Print(final); + } + PrintString("\", shape = doublecircle,"); + } else { + PrintString("\", shape = circle,"); + } + if (s == fst_.Start()) + PrintString(" style = bold,"); + else + PrintString(" style = solid,"); + PrintString(" fontsize = "); + Print(fontsize_); + PrintString("]\n"); + for (ArcIterator< Fst<A> > aiter(fst_, s); + !aiter.Done(); + aiter.Next()) { + Arc arc = aiter.Value(); + PrintString("\t"); + Print(s); + PrintString(" -> "); + Print(arc.nextstate); + PrintString(" [label = \""); + PrintILabel(arc.ilabel); + if (!accep_) { + PrintString(":"); + PrintOLabel(arc.olabel); + } + if (show_weight_one_ || (arc.weight != Weight::One())) { + PrintString("/"); + Print(arc.weight); + } + PrintString("\", fontsize = "); + Print(fontsize_); + PrintString("];\n"); + } + } + + const Fst<A> &fst_; + const SymbolTable *isyms_; // ilabel symbol table + const SymbolTable *osyms_; // olabel symbol table + const SymbolTable *ssyms_; // slabel symbol table + bool accep_; // print as acceptor when possible + ostream *ostrm_; // drawn FST destination + string dest_; // drawn FST destination name + + string title_; + float width_; + float height_; + bool portrait_; + bool vertical_; + float ranksep_; + float nodesep_; + int fontsize_; + int precision_; + bool show_weight_one_; + + DISALLOW_COPY_AND_ASSIGN(FstDrawer); +}; + +} // namespace fst + +#endif // FST_SCRIPT_DRAW_IMPL_H_ diff --git a/src/include/fst/script/draw.h b/src/include/fst/script/draw.h new file mode 100644 index 0000000..1611ad1 --- /dev/null +++ b/src/include/fst/script/draw.h @@ -0,0 +1,113 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_DRAW_H_ +#define FST_SCRIPT_DRAW_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/draw-impl.h> +#include <iostream> +#include <fstream> + +namespace fst { +namespace script { + +// Note: it is safe to pass these strings as references because +// this struct is only used to pass them deeper in the call graph. +// Be sure you understand why this is so before using this struct +// for anything else! +struct FstDrawerArgs { + const FstClass &fst; + const SymbolTable *isyms; + const SymbolTable *osyms; + const SymbolTable *ssyms; + const bool accep; + const string& title; + const float width; + const float height; + const bool portrait; + const bool vertical; + const float ranksep; + const float nodesep; + const int fontsize; + const int precision; + const bool show_weight_one; + ostream *ostrm; + const string &dest; + + FstDrawerArgs(const FstClass &fst, + const SymbolTable *isyms, + const SymbolTable *osyms, + const SymbolTable *ssyms, + bool accep, + const string &title, + float width, + float height, + bool portrait, + bool vertical, + float ranksep, + float nodesep, + int fontsize, + int precision, + bool show_weight_one, + ostream *ostrm, + const string &dest) : + fst(fst), isyms(isyms), osyms(osyms), ssyms(ssyms), accep(accep), + title(title), width(width), height(height), portrait(portrait), + vertical(vertical), ranksep(ranksep), nodesep(nodesep), + fontsize(fontsize), precision(precision), + show_weight_one(show_weight_one), ostrm(ostrm), dest(dest) { } +}; + + +template<class Arc> +void DrawFst(FstDrawerArgs *args) { + const Fst<Arc> &fst = *(args->fst.GetFst<Arc>()); + + FstDrawer<Arc> fstdrawer(fst, args->isyms, args->osyms, args->ssyms, + args->accep, args->title, args->width, + args->height, args->portrait, + args->vertical, args->ranksep, + args->nodesep, args->fontsize, + args->precision, args->show_weight_one); + fstdrawer.Draw(args->ostrm, args->dest); +} + +void DrawFst(const FstClass &fst, + const SymbolTable *isyms, + const SymbolTable *osyms, + const SymbolTable *ssyms, + bool accep, + const string &title, + float width, + float height, + bool portrait, + bool vertical, + float ranksep, + float nodesep, + int fontsize, + int precision, + bool show_weight_one, + ostream *ostrm, + const string &dest); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_DRAW_H_ diff --git a/src/include/fst/script/encode.h b/src/include/fst/script/encode.h new file mode 100644 index 0000000..dc1a290 --- /dev/null +++ b/src/include/fst/script/encode.h @@ -0,0 +1,58 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_ENCODE_H_ +#define FST_SCRIPT_ENCODE_H_ + +#include <string> + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/encode.h> + +namespace fst { +namespace script { + +typedef args::Package<MutableFstClass*, uint32, bool, + const string &> EncodeArgs; + +template<class Arc> +void Encode(EncodeArgs *args) { + MutableFst<Arc> *ofst = args->arg1->GetMutableFst<Arc>(); + bool reuse_encoder = args->arg3; + const string &coder_fname = args->arg4; + uint32 flags = args->arg2; + + EncodeMapper<Arc> *encoder = reuse_encoder + ? EncodeMapper<Arc>::Read(coder_fname, ENCODE) + : new EncodeMapper<Arc>(flags, ENCODE); + + Encode(ofst, encoder); + if (!args->arg3) + encoder->Write(coder_fname); + + delete encoder; +} + +void Encode(MutableFstClass *fst, uint32 flags, bool reuse_encoder, + const string &coder_fname); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_ENCODE_H_ diff --git a/src/include/fst/script/epsnormalize.h b/src/include/fst/script/epsnormalize.h new file mode 100644 index 0000000..50b12da --- /dev/null +++ b/src/include/fst/script/epsnormalize.h @@ -0,0 +1,44 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_EPSNORMALIZE_H_ +#define FST_SCRIPT_EPSNORMALIZE_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/epsnormalize.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass&, MutableFstClass*, + EpsNormalizeType> EpsNormalizeArgs; + +template<class Arc> +void EpsNormalize(EpsNormalizeArgs *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + + EpsNormalize(ifst, ofst, args->arg3); +} + +void EpsNormalize(const FstClass &ifst, MutableFstClass *ofst, + EpsNormalizeType norm_type = EPS_NORM_INPUT); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_EPSNORMALIZE_H_ diff --git a/src/include/fst/script/equal.h b/src/include/fst/script/equal.h new file mode 100644 index 0000000..9fb2d3c --- /dev/null +++ b/src/include/fst/script/equal.h @@ -0,0 +1,45 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_EQUAL_H_ +#define FST_SCRIPT_EQUAL_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/equal.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass&, const FstClass&, float> EqualInnerArgs; +typedef args::WithReturnValue<bool, EqualInnerArgs> EqualArgs; + +template<class Arc> +void Equal(EqualArgs *args) { + const Fst<Arc> &fst1 = *(args->args.arg1.GetFst<Arc>()); + const Fst<Arc> &fst2 = *(args->args.arg2.GetFst<Arc>()); + + args->retval = Equal(fst1, fst2, args->args.arg3); +} + +bool Equal(const FstClass &fst1, const FstClass &fst2, + float delta = kDelta); + +} // namespace script +} // namespace fst + + +#endif // FST_SCRIPT_EQUAL_H_ diff --git a/src/include/fst/script/equivalent.h b/src/include/fst/script/equivalent.h new file mode 100644 index 0000000..43460c6 --- /dev/null +++ b/src/include/fst/script/equivalent.h @@ -0,0 +1,47 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_EQUIVALENT_H_ +#define FST_SCRIPT_EQUIVALENT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/equivalent.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass &, const FstClass &, + float> EquivalentInnerArgs; +typedef args::WithReturnValue<bool, EquivalentInnerArgs> EquivalentArgs; + +template<class Arc> +void Equivalent(EquivalentArgs *args) { + const Fst<Arc> &fst1 = *(args->args.arg1.GetFst<Arc>()); + const Fst<Arc> &fst2 = *(args->args.arg2.GetFst<Arc>()); + + args->retval = Equivalent(fst1, fst2, args->args.arg3); +} + +bool Equivalent(const FstClass &fst1, const FstClass &fst2, + float delta = kDelta); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_EQUIVALENT_H_ diff --git a/src/include/fst/script/fst-class.h b/src/include/fst/script/fst-class.h new file mode 100644 index 0000000..3eacab4 --- /dev/null +++ b/src/include/fst/script/fst-class.h @@ -0,0 +1,343 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_FST_CLASS_H_ +#define FST_SCRIPT_FST_CLASS_H_ + +#include <string> + +#include <fst/fst.h> +#include <fst/mutable-fst.h> +#include <fst/vector-fst.h> +#include <iostream> +#include <fstream> + +// Classes to support "boxing" all existing types of FST arcs in a single +// FstClass which hides the arc types. This allows clients to load +// and work with FSTs without knowing the arc type. + +// These classes are only recommended for use in high-level scripting +// applications. Most users should use the lower-level templated versions +// corresponding to these classes. + +namespace fst { +namespace script { + +// +// Abstract base class defining the set of functionalities implemented +// in all impls, and passed through by all bases Below FstClassBase +// the class hierarchy bifurcates; FstClassImplBase serves as the base +// class for all implementations (of which FstClassImpl is currently +// the only one) and FstClass serves as the base class for all +// interfaces. +// +class FstClassBase { + public: + virtual const string &ArcType() const = 0; + virtual const string &FstType() const = 0; + virtual const string &WeightType() const = 0; + virtual const SymbolTable *InputSymbols() const = 0; + virtual const SymbolTable *OutputSymbols() const = 0; + virtual void Write(const string& fname) const = 0; + virtual uint64 Properties(uint64 mask, bool test) const = 0; + virtual ~FstClassBase() { } +}; + +class FstClassImplBase : public FstClassBase { + public: + virtual FstClassImplBase *Copy() = 0; + virtual void SetInputSymbols(SymbolTable *is) = 0; + virtual void SetOutputSymbols(SymbolTable *is) = 0; + virtual ~FstClassImplBase() { } +}; + + +// +// CONTAINER CLASS +// Wraps an Fst<Arc>, hiding its arc type. Whether this Fst<Arc> +// pointer refers to a special kind of FST (e.g. a MutableFst) is +// known by the type of interface class that owns the pointer to this +// container. +// + +template<class Arc> +class FstClassImpl : public FstClassImplBase { + public: + explicit FstClassImpl(Fst<Arc> *impl, + bool should_own = false) : + impl_(should_own ? impl : impl->Copy()) { } + + virtual const string &ArcType() const { + return Arc::Type(); + } + + virtual const string &FstType() const { + return impl_->Type(); + } + + virtual const string &WeightType() const { + return Arc::Weight::Type(); + } + + virtual const SymbolTable *InputSymbols() const { + return impl_->InputSymbols(); + } + + virtual const SymbolTable *OutputSymbols() const { + return impl_->OutputSymbols(); + } + + // Warning: calling this method casts the FST to a mutable FST. + virtual void SetInputSymbols(SymbolTable *is) { + static_cast<MutableFst<Arc> *>(impl_)->SetInputSymbols(is); + } + + // Warning: calling this method casts the FST to a mutable FST. + virtual void SetOutputSymbols(SymbolTable *os) { + static_cast<MutableFst<Arc> *>(impl_)->SetOutputSymbols(os); + } + + virtual void Write(const string &fname) const { + impl_->Write(fname); + } + + virtual uint64 Properties(uint64 mask, bool test) const { + return impl_->Properties(mask, test); + } + + virtual ~FstClassImpl() { delete impl_; } + + Fst<Arc> *GetImpl() { return impl_; } + + virtual FstClassImpl *Copy() { + return new FstClassImpl<Arc>(impl_); + } + + private: + Fst<Arc> *impl_; +}; + +// +// BASE CLASS DEFINITIONS +// + +class MutableFstClass; + +class FstClass : public FstClassBase { + public: + template<class Arc> + static FstClass *Read(istream &stream, + const FstReadOptions &opts) { + if (!opts.header) { + FSTERROR() << "FstClass::Read: options header not specified"; + return 0; + } + const FstHeader &hdr = *opts.header; + + if (hdr.Properties() & kMutable) { + return ReadTypedFst<MutableFstClass, MutableFst<Arc> >(stream, opts); + } else { + return ReadTypedFst<FstClass, Fst<Arc> >(stream, opts); + } + } + + template<class Arc> + explicit FstClass(Fst<Arc> *fst) : impl_(new FstClassImpl<Arc>(fst)) { } + + explicit FstClass(const FstClass &other) : impl_(other.impl_->Copy()) { } + + static FstClass *Read(const string &fname); + + virtual const string &ArcType() const { + return impl_->ArcType(); + } + + virtual const string& FstType() const { + return impl_->FstType(); + } + + virtual const SymbolTable *InputSymbols() const { + return impl_->InputSymbols(); + } + + virtual const SymbolTable *OutputSymbols() const { + return impl_->OutputSymbols(); + } + + virtual const string& WeightType() const { + return impl_->WeightType(); + } + + virtual void Write(const string &fname) const { + impl_->Write(fname); + } + + virtual uint64 Properties(uint64 mask, bool test) const { + return impl_->Properties(mask, test); + } + + template<class Arc> + const Fst<Arc> *GetFst() const { + if (Arc::Type() != ArcType()) { + return NULL; + } else { + FstClassImpl<Arc> *typed_impl = static_cast<FstClassImpl<Arc> *>(impl_); + return typed_impl->GetImpl(); + } + } + + virtual ~FstClass() { delete impl_; } + + // These methods are required by IO registration + template<class Arc> + static FstClassImplBase *Convert(const FstClass &other) { + LOG(ERROR) << "Doesn't make sense to convert any class to type FstClass."; + return 0; + } + + template<class Arc> + static FstClassImplBase *Create() { + LOG(ERROR) << "Doesn't make sense to create an FstClass with a " + << "particular arc type."; + return 0; + } + protected: + explicit FstClass(FstClassImplBase *impl) : impl_(impl) { } + + // Generic template method for reading an arc-templated FST of type + // UnderlyingT, and returning it wrapped as FstClassT, with appropriate + // error checking. Called from arc-templated Read() static methods. + template<class FstClassT, class UnderlyingT> + static FstClassT* ReadTypedFst(istream &stream, + const FstReadOptions &opts) { + UnderlyingT *u = UnderlyingT::Read(stream, opts); + if (!u) { + return 0; + } else { + FstClassT *r = new FstClassT(u); + delete u; + return r; + } + } + + FstClassImplBase *GetImpl() { return impl_; } + private: + FstClassImplBase *impl_; +}; + +// +// Specific types of FstClass with special properties +// + +class MutableFstClass : public FstClass { + public: + template<class Arc> + explicit MutableFstClass(MutableFst<Arc> *fst) : + FstClass(fst) { } + + template<class Arc> + MutableFst<Arc> *GetMutableFst() { + Fst<Arc> *fst = const_cast<Fst<Arc> *>(this->GetFst<Arc>()); + MutableFst<Arc> *mfst = static_cast<MutableFst<Arc> *>(fst); + + return mfst; + } + + template<class Arc> + static MutableFstClass *Read(istream &stream, + const FstReadOptions &opts) { + MutableFst<Arc> *mfst = MutableFst<Arc>::Read(stream, opts); + if (!mfst) { + return 0; + } else { + MutableFstClass *retval = new MutableFstClass(mfst); + delete mfst; + return retval; + } + } + + static MutableFstClass *Read(const string &fname, bool convert = false); + + virtual void SetInputSymbols(SymbolTable *is) { + GetImpl()->SetInputSymbols(is); + } + + virtual void SetOutputSymbols(SymbolTable *os) { + GetImpl()->SetOutputSymbols(os); + } + + // These methods are required by IO registration + template<class Arc> + static FstClassImplBase *Convert(const FstClass &other) { + LOG(ERROR) << "Doesn't make sense to convert any class to type " + << "MutableFstClass."; + return 0; + } + + template<class Arc> + static FstClassImplBase *Create() { + LOG(ERROR) << "Doesn't make sense to create a MutableFstClass with a " + << "particular arc type."; + return 0; + } + + protected: + explicit MutableFstClass(FstClassImplBase *impl) : FstClass(impl) { } +}; + + +class VectorFstClass : public MutableFstClass { + public: + explicit VectorFstClass(const FstClass &other); + explicit VectorFstClass(const string &arc_type); + + template<class Arc> + explicit VectorFstClass(VectorFst<Arc> *fst) : + MutableFstClass(fst) { } + + template<class Arc> + static VectorFstClass *Read(istream &stream, + const FstReadOptions &opts) { + VectorFst<Arc> *vfst = VectorFst<Arc>::Read(stream, opts); + if (!vfst) { + return 0; + } else { + VectorFstClass *retval = new VectorFstClass(vfst); + delete vfst; + return retval; + } + } + + static VectorFstClass *Read(const string &fname); + + // Converter / creator for known arc types + template<class Arc> + static FstClassImplBase *Convert(const FstClass &other) { + return new FstClassImpl<Arc>(new VectorFst<Arc>( + *other.GetFst<Arc>()), true); + } + + template<class Arc> + static FstClassImplBase *Create() { + return new FstClassImpl<Arc>(new VectorFst<Arc>(), true); + } +}; + +} // namespace script +} // namespace fst + + +#endif // FST_SCRIPT_FST_CLASS_H_ diff --git a/src/include/fst/script/fstscript-decl.h b/src/include/fst/script/fstscript-decl.h new file mode 100644 index 0000000..fee813e --- /dev/null +++ b/src/include/fst/script/fstscript-decl.h @@ -0,0 +1,35 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +// Forward declarations for the FST and FST-script classes. + +#ifndef FST_SCRIPT_FSTSCRIPT_DECL_H_ +#define FST_SCRIPT_FSTSCRIPT_DECL_H_ + +#include <fst/fst-decl.h> + +namespace fst { +namespace script { + +class FstClass; +class MutableFstClass; +class VectorFstClass; +class WeightClass; + +} // namespace script +} // namespace fst; + +#endif // FST_SCRIPT_FSTSCRIPT_DECL_H_ diff --git a/src/include/fst/script/fstscript.h b/src/include/fst/script/fstscript.h new file mode 100644 index 0000000..90e1e75 --- /dev/null +++ b/src/include/fst/script/fstscript.h @@ -0,0 +1,154 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +// Convenience file that includes all FstScript functionality + +#ifndef FST_SCRIPT_FSTSCRIPT_H_ +#define FST_SCRIPT_FSTSCRIPT_H_ + +// Major classes +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> +#include <fst/script/text-io.h> + +// Templates like Operation< >, Apply< > +#include <fst/script/script-impl.h> + +// Operations +#include <fst/script/arcsort.h> +#include <fst/script/closure.h> +#include <fst/script/compile.h> +#include <fst/script/compose.h> +#include <fst/script/concat.h> +#include <fst/script/connect.h> +#include <fst/script/convert.h> +#include <fst/script/decode.h> +#include <fst/script/determinize.h> +#include <fst/script/difference.h> +#include <fst/script/draw.h> +#include <fst/script/encode.h> +#include <fst/script/epsnormalize.h> +#include <fst/script/equal.h> +#include <fst/script/equivalent.h> +#include <fst/script/info.h> +#include <fst/script/intersect.h> +#include <fst/script/invert.h> +#include <fst/script/map.h> +#include <fst/script/minimize.h> +#include <fst/script/print.h> +#include <fst/script/project.h> +#include <fst/script/prune.h> +#include <fst/script/push.h> +#include <fst/script/randequivalent.h> +#include <fst/script/randgen.h> +#include <fst/script/relabel.h> +#include <fst/script/replace.h> +#include <fst/script/reverse.h> +#include <fst/script/reweight.h> +#include <fst/script/rmepsilon.h> +#include <fst/script/shortest-distance.h> +#include <fst/script/shortest-path.h> +#include <fst/script/symbols.h> +#include <fst/script/synchronize.h> +#include <fst/script/topsort.h> +#include <fst/script/union.h> +#include <fst/script/verify.h> + +// +// REGISTER OPERATIONS +// + + +// This class is necessary because registering each of the operations +// separately overfills the stack, as there's so many of them. +namespace fst { +namespace script { +template<class Arc> +class AllFstOperationsRegisterer { + public: + AllFstOperationsRegisterer() { + RegisterBatch1(); + RegisterBatch2(); + } + + private: + void RegisterBatch1() { + REGISTER_FST_OPERATION(ArcSort, Arc, ArcSortArgs); + REGISTER_FST_OPERATION(Closure, Arc, ClosureArgs); + REGISTER_FST_OPERATION(CompileFst, Arc, FstCompileArgs); + REGISTER_FST_OPERATION(Compose, Arc, ComposeArgs1); + REGISTER_FST_OPERATION(Compose, Arc, ComposeArgs2); + REGISTER_FST_OPERATION(Concat, Arc, ConcatArgs1); + REGISTER_FST_OPERATION(Concat, Arc, ConcatArgs2); + REGISTER_FST_OPERATION(Connect, Arc, MutableFstClass); + REGISTER_FST_OPERATION(Convert, Arc, ConvertArgs); + REGISTER_FST_OPERATION(Decode, Arc, DecodeArgs); + REGISTER_FST_OPERATION(Determinize, Arc, DeterminizeArgs); + REGISTER_FST_OPERATION(Difference, Arc, DifferenceArgs1); + REGISTER_FST_OPERATION(Difference, Arc, DifferenceArgs2); + REGISTER_FST_OPERATION(DrawFst, Arc, FstDrawerArgs); + REGISTER_FST_OPERATION(Encode, Arc, EncodeArgs); + REGISTER_FST_OPERATION(EpsNormalize, Arc, EpsNormalizeArgs); + REGISTER_FST_OPERATION(Equal, Arc, EqualArgs); + REGISTER_FST_OPERATION(Equivalent, Arc, EquivalentArgs); + REGISTER_FST_OPERATION(PrintFstInfo, Arc, InfoArgs); + REGISTER_FST_OPERATION(Intersect, Arc, IntersectArgs1); + REGISTER_FST_OPERATION(Intersect, Arc, IntersectArgs2); + REGISTER_FST_OPERATION(Invert, Arc, MutableFstClass); + REGISTER_FST_OPERATION(Map, Arc, MapArgs); + REGISTER_FST_OPERATION(Minimize, Arc, MinimizeArgs); + } + + void RegisterBatch2() { + REGISTER_FST_OPERATION(PrintFst, Arc, FstPrinterArgs); + REGISTER_FST_OPERATION(Project, Arc, ProjectArgs); + REGISTER_FST_OPERATION(Prune, Arc, PruneArgs1); + REGISTER_FST_OPERATION(Prune, Arc, PruneArgs2); + REGISTER_FST_OPERATION(Prune, Arc, PruneArgs3); + REGISTER_FST_OPERATION(Prune, Arc, PruneArgs4); + REGISTER_FST_OPERATION(Push, Arc, PushArgs1); + REGISTER_FST_OPERATION(Push, Arc, PushArgs2); + REGISTER_FST_OPERATION(RandEquivalent, Arc, RandEquivalentArgs1); + REGISTER_FST_OPERATION(RandEquivalent, Arc, RandEquivalentArgs2); + REGISTER_FST_OPERATION(RandGen, Arc, RandGenArgs); + REGISTER_FST_OPERATION(Relabel, Arc, RelabelArgs1); + REGISTER_FST_OPERATION(Relabel, Arc, RelabelArgs2); + REGISTER_FST_OPERATION(Relabel, Arc, RelabelArgs3); + REGISTER_FST_OPERATION(Replace, Arc, ReplaceArgs); + REGISTER_FST_OPERATION(Reverse, Arc, ReverseArgs); + REGISTER_FST_OPERATION(Reweight, Arc, ReweightArgs); + REGISTER_FST_OPERATION(RmEpsilon, Arc, RmEpsilonArgs1); + REGISTER_FST_OPERATION(RmEpsilon, Arc, RmEpsilonArgs2); + REGISTER_FST_OPERATION(RmEpsilon, Arc, RmEpsilonArgs3); + REGISTER_FST_OPERATION(ShortestDistance, Arc, ShortestDistanceArgs1); + REGISTER_FST_OPERATION(ShortestDistance, Arc, ShortestDistanceArgs2); + REGISTER_FST_OPERATION(ShortestDistance, Arc, ShortestDistanceArgs3); + REGISTER_FST_OPERATION(ShortestPath, Arc, ShortestPathArgs1); + REGISTER_FST_OPERATION(ShortestPath, Arc, ShortestPathArgs2); + REGISTER_FST_OPERATION(Synchronize, Arc, SynchronizeArgs); + REGISTER_FST_OPERATION(TopSort, Arc, TopSortArgs); + REGISTER_FST_OPERATION(Union, Arc, UnionArgs); + REGISTER_FST_OPERATION(Verify, Arc, VerifyArgs); + } +}; +} // namespace script +} // namespace fst + + +#define REGISTER_FST_OPERATIONS(Arc) \ + AllFstOperationsRegisterer<Arc> register_all_fst_operations ## Arc; + +#endif // FST_SCRIPT_FSTSCRIPT_H_ diff --git a/src/include/fst/script/info-impl.h b/src/include/fst/script/info-impl.h new file mode 100644 index 0000000..408fbcd --- /dev/null +++ b/src/include/fst/script/info-impl.h @@ -0,0 +1,325 @@ +// info.h + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: riley@google.com (Michael Riley) +// +// \file +// Class to compute various information about FSTs, helper class for fstinfo.cc + +#ifndef FST_SCRIPT_INFO_IMPL_H_ +#define FST_SCRIPT_INFO_IMPL_H_ + +#include <string> +#include <vector> +using std::vector; + +#include <fst/connect.h> +#include <fst/dfs-visit.h> +#include <fst/fst.h> +#include <fst/lookahead-matcher.h> +#include <fst/matcher.h> +#include <fst/queue.h> +#include <fst/test-properties.h> +#include <fst/verify.h> +#include <fst/visit.h> + +namespace fst { + +// Compute various information about FSTs, helper class for fstinfo.cc. +// WARNING: Stand-alone use of this class is not recommended, most code +// should call directly the relevant library functions: Fst<A>::NumStates, +// Fst<A>::NumArcs, TestProperties, ... +template <class A> class FstInfo { + public: + typedef A Arc; + typedef typename A::StateId StateId; + typedef typename A::Label Label; + typedef typename A::Weight Weight; + + // When info_type is "short" (or "auto" and not an ExpandedFst) + // then only minimal info is computed and can be requested. + FstInfo(const Fst<A> &fst, bool test_properties, + const string &arc_filter_type = "any", + string info_type = "auto", bool verify = true) + : fst_type_(fst.Type()), + input_symbols_(fst.InputSymbols() ? + fst.InputSymbols()->Name() : "none"), + output_symbols_(fst.OutputSymbols() ? + fst.OutputSymbols()->Name() : "none"), + nstates_(0), narcs_(0), start_(kNoStateId), nfinal_(0), + nepsilons_(0), niepsilons_(0), noepsilons_(0), + naccess_(0), ncoaccess_(0), nconnect_(0), ncc_(0), nscc_(0), + input_match_type_(MATCH_NONE), output_match_type_(MATCH_NONE), + input_lookahead_(false), output_lookahead_(false), + properties_(0), arc_filter_type_(arc_filter_type), long_info_(true) { + if (info_type == "long") { + long_info_ = true; + } else if (info_type == "short") { + long_info_ = false; + } else if (info_type == "auto") { + long_info_ = fst.Properties(kExpanded, false); + } else { + FSTERROR() << "Bad info type: " << info_type; + return; + } + + if (!long_info_) + return; + + // If the FST is not sane, we return. + if (verify && !Verify(fst)) { + FSTERROR() << "FstInfo: Verify: FST not well-formed."; + return; + } + + start_ = fst.Start(); + properties_ = fst.Properties(kFstProperties, test_properties); + + for (StateIterator< Fst<A> > siter(fst); + !siter.Done(); + siter.Next()) { + ++nstates_; + StateId s = siter.Value(); + if (fst.Final(s) != Weight::Zero()) + ++nfinal_; + for (ArcIterator< Fst<A> > aiter(fst, s); + !aiter.Done(); + aiter.Next()) { + const A &arc = aiter.Value(); + ++narcs_; + if (arc.ilabel == 0 && arc.olabel == 0) + ++nepsilons_; + if (arc.ilabel == 0) + ++niepsilons_; + if (arc.olabel == 0) + ++noepsilons_; + } + } + + { + vector<StateId> cc; + CcVisitor<Arc> cc_visitor(&cc); + FifoQueue<StateId> fifo_queue; + if (arc_filter_type == "any") { + Visit(fst, &cc_visitor, &fifo_queue); + } else if (arc_filter_type == "epsilon") { + Visit(fst, &cc_visitor, &fifo_queue, EpsilonArcFilter<Arc>()); + } else if (arc_filter_type == "iepsilon") { + Visit(fst, &cc_visitor, &fifo_queue, InputEpsilonArcFilter<Arc>()); + } else if (arc_filter_type == "oepsilon") { + Visit(fst, &cc_visitor, &fifo_queue, OutputEpsilonArcFilter<Arc>()); + } else { + FSTERROR() << "Bad arc filter type: " << arc_filter_type; + return; + } + + for (StateId s = 0; s < cc.size(); ++s) { + if (cc[s] >= ncc_) + ncc_ = cc[s] + 1; + } + } + + { + vector<StateId> scc; + vector<bool> access, coaccess; + uint64 props = 0; + SccVisitor<Arc> scc_visitor(&scc, &access, &coaccess, &props); + if (arc_filter_type == "any") { + DfsVisit(fst, &scc_visitor); + } else if (arc_filter_type == "epsilon") { + DfsVisit(fst, &scc_visitor, EpsilonArcFilter<Arc>()); + } else if (arc_filter_type == "iepsilon") { + DfsVisit(fst, &scc_visitor, InputEpsilonArcFilter<Arc>()); + } else if (arc_filter_type == "oepsilon") { + DfsVisit(fst, &scc_visitor, OutputEpsilonArcFilter<Arc>()); + } else { + FSTERROR() << "Bad arc filter type: " << arc_filter_type; + return; + } + + for (StateId s = 0; s < scc.size(); ++s) { + if (access[s]) + ++naccess_; + if (coaccess[s]) + ++ncoaccess_; + if (access[s] && coaccess[s]) + ++nconnect_; + if (scc[s] >= nscc_) + nscc_ = scc[s] + 1; + } + } + + LookAheadMatcher< Fst<A> > imatcher(fst, MATCH_INPUT); + input_match_type_ = imatcher.Type(test_properties); + input_lookahead_ = imatcher.Flags() & kInputLookAheadMatcher; + + LookAheadMatcher< Fst<A> > omatcher(fst, MATCH_OUTPUT); + output_match_type_ = omatcher.Type(test_properties); + output_lookahead_ = omatcher.Flags() & kOutputLookAheadMatcher; + } + + // Short info + const string& FstType() const { return fst_type_; } + const string& ArcType() const { return A::Type(); } + const string& InputSymbols() const { return input_symbols_; } + const string& OutputSymbols() const { return output_symbols_; } + const bool LongInfo() const { return long_info_; } + const string& ArcFilterType() const { return arc_filter_type_; } + + // Long info + MatchType InputMatchType() const { CheckLong(); return input_match_type_; } + MatchType OutputMatchType() const { CheckLong(); return output_match_type_; } + bool InputLookAhead() const { CheckLong(); return input_lookahead_; } + bool OutputLookAhead() const { CheckLong(); return output_lookahead_; } + int64 NumStates() const { CheckLong(); return nstates_; } + int64 NumArcs() const { CheckLong(); return narcs_; } + int64 Start() const { CheckLong(); return start_; } + int64 NumFinal() const { CheckLong(); return nfinal_; } + int64 NumEpsilons() const { CheckLong(); return nepsilons_; } + int64 NumInputEpsilons() const { CheckLong(); return niepsilons_; } + int64 NumOutputEpsilons() const { CheckLong(); return noepsilons_; } + int64 NumAccessible() const { CheckLong(); return naccess_; } + int64 NumCoAccessible() const { CheckLong(); return ncoaccess_; } + int64 NumConnected() const { CheckLong(); return nconnect_; } + int64 NumCc() const { CheckLong(); return ncc_; } + int64 NumScc() const { CheckLong(); return nscc_; } + uint64 Properties() const { CheckLong(); return properties_; } + + private: + void CheckLong() const { + if (!long_info_) + FSTERROR() << "FstInfo: method only available with long info version"; + } + + string fst_type_; + string input_symbols_; + string output_symbols_; + int64 nstates_; + int64 narcs_; + int64 start_; + int64 nfinal_; + int64 nepsilons_; + int64 niepsilons_; + int64 noepsilons_; + int64 naccess_; + int64 ncoaccess_; + int64 nconnect_; + int64 ncc_; + int64 nscc_; + MatchType input_match_type_; + MatchType output_match_type_; + bool input_lookahead_; + bool output_lookahead_; + uint64 properties_; + string arc_filter_type_; + bool long_info_; + DISALLOW_COPY_AND_ASSIGN(FstInfo); +}; + +template <class A> +void PrintFstInfo(const FstInfo<A> &fstinfo, bool pipe = false) { + ostream &os = pipe ? cerr : cout; + + ios_base::fmtflags old = os.setf(ios::left); + os.width(50); + os << "fst type" << fstinfo.FstType() << endl; + os.width(50); + os << "arc type" << fstinfo.ArcType() << endl; + os.width(50); + os << "input symbol table" << fstinfo.InputSymbols() << endl; + os.width(50); + os << "output symbol table" << fstinfo.OutputSymbols() << endl; + + if (!fstinfo.LongInfo()) { + os.setf(old); + return; + } + + os.width(50); + os << "# of states" << fstinfo.NumStates() << endl; + os.width(50); + os << "# of arcs" << fstinfo.NumArcs() << endl; + os.width(50); + os << "initial state" << fstinfo.Start() << endl; + os.width(50); + os << "# of final states" << fstinfo.NumFinal() << endl; + os.width(50); + os << "# of input/output epsilons" << fstinfo.NumEpsilons() << endl; + os.width(50); + os << "# of input epsilons" << fstinfo.NumInputEpsilons() << endl; + os.width(50); + os << "# of output epsilons" << fstinfo.NumOutputEpsilons() << endl; + os.width(50); + + string arc_type = ""; + if (fstinfo.ArcFilterType() == "epsilon") + arc_type = "epsilon "; + else if (fstinfo.ArcFilterType() == "iepsilon") + arc_type = "input-epsilon "; + else if (fstinfo.ArcFilterType() == "oepsilon") + arc_type = "output-epsilon "; + + string accessible_label = "# of " + arc_type + "accessible states"; + os.width(50); + os << accessible_label << fstinfo.NumAccessible() << endl; + string coaccessible_label = "# of " + arc_type + "coaccessible states"; + os.width(50); + os << coaccessible_label << fstinfo.NumCoAccessible() << endl; + string connected_label = "# of " + arc_type + "connected states"; + os.width(50); + os << connected_label << fstinfo.NumConnected() << endl; + string numcc_label = "# of " + arc_type + "connected components"; + os.width(50); + os << numcc_label << fstinfo.NumCc() << endl; + string numscc_label = "# of " + arc_type + "strongly conn components"; + os.width(50); + os << numscc_label << fstinfo.NumScc() << endl; + + os.width(50); + os << "input matcher" + << (fstinfo.InputMatchType() == MATCH_INPUT ? 'y' : + fstinfo.InputMatchType() == MATCH_NONE ? 'n' : '?') << endl; + os.width(50); + os << "output matcher" + << (fstinfo.OutputMatchType() == MATCH_OUTPUT ? 'y' : + fstinfo.OutputMatchType() == MATCH_NONE ? 'n' : '?') << endl; + os.width(50); + os << "input lookahead" + << (fstinfo.InputLookAhead() ? 'y' : 'n') << endl; + os.width(50); + os << "output lookahead" + << (fstinfo.OutputLookAhead() ? 'y' : 'n') << endl; + + uint64 prop = 1; + for (int i = 0; i < 64; ++i, prop <<= 1) { + if (prop & kBinaryProperties) { + char value = 'n'; + if (fstinfo.Properties() & prop) value = 'y'; + os.width(50); + os << PropertyNames[i] << value << endl; + } else if (prop & kPosTrinaryProperties) { + char value = '?'; + if (fstinfo.Properties() & prop) value = 'y'; + else if (fstinfo.Properties() & prop << 1) value = 'n'; + os.width(50); + os << PropertyNames[i] << value << endl; + } + } + os.setf(old); +} + +} // namespace fst + +#endif // FST_SCRIPT_INFO_IMPL_H_ diff --git a/src/include/fst/script/info.h b/src/include/fst/script/info.h new file mode 100644 index 0000000..f434bd5 --- /dev/null +++ b/src/include/fst/script/info.h @@ -0,0 +1,48 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_INFO_H_ +#define FST_SCRIPT_INFO_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/info-impl.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass&, bool, const string&, + const string&, bool, bool> InfoArgs; + +template<class Arc> +void PrintFstInfo(InfoArgs *args) { + const Fst<Arc> &fst = *(args->arg1.GetFst<Arc>()); + FstInfo<Arc> fstinfo(fst, args->arg2, args->arg3, + args->arg4, args->arg5); + PrintFstInfo(fstinfo, args->arg6); + + if (args->arg6) + fst.Write(""); +} + +void PrintFstInfo(const FstClass &f, bool test_properties, + const string &arc_filter, const string &info_type, + bool pipe, bool verify); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_INFO_H_ diff --git a/src/include/fst/script/intersect.h b/src/include/fst/script/intersect.h new file mode 100644 index 0000000..8011024 --- /dev/null +++ b/src/include/fst/script/intersect.h @@ -0,0 +1,65 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_INTERSECT_H_ +#define FST_SCRIPT_INTERSECT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/intersect.h> +#include <fst/script/compose.h> // for ComposeOptions, ComposeFilter + +namespace fst { +namespace script { + +typedef args::Package<const FstClass&, const FstClass&, + MutableFstClass*, ComposeFilter> IntersectArgs1; + +template<class Arc> +void Intersect(IntersectArgs1 *args) { + const Fst<Arc> &ifst1 = *(args->arg1.GetFst<Arc>()); + const Fst<Arc> &ifst2 = *(args->arg2.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg3->GetMutableFst<Arc>(); + + Intersect(ifst1, ifst2, ofst, args->arg4); +} + +typedef args::Package<const FstClass&, const FstClass&, + MutableFstClass*, const ComposeOptions &> IntersectArgs2; + +template<class Arc> +void Intersect(IntersectArgs2 *args) { + const Fst<Arc> &ifst1 = *(args->arg1.GetFst<Arc>()); + const Fst<Arc> &ifst2 = *(args->arg2.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg3->GetMutableFst<Arc>(); + + Intersect(ifst1, ifst2, ofst, args->arg4); +} + +void Intersect(const FstClass &ifst1, const FstClass &ifst2, + MutableFstClass *ofst, + ComposeFilter compose_filter); + +void Intersect(const FstClass &ifst, const FstClass &ifst2, + MutableFstClass *ofst, + const ComposeOptions &opts = fst::script::ComposeOptions()); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_INTERSECT_H_ diff --git a/src/include/fst/script/invert.h b/src/include/fst/script/invert.h new file mode 100644 index 0000000..1befd9f --- /dev/null +++ b/src/include/fst/script/invert.h @@ -0,0 +1,43 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_INVERT_H_ +#define FST_SCRIPT_INVERT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/invert.h> + +namespace fst { +namespace script { + +// The following confuses swig, because it has the same arguments +// as the non-templated version +#ifndef SWIG +template<class Arc> +void Invert(MutableFstClass *fst) { + MutableFst<Arc> *typed_fst = fst->GetMutableFst<Arc>(); + + Invert(typed_fst); +} +#endif + +void Invert(MutableFstClass *fst); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_INVERT_H_ diff --git a/src/include/fst/script/map.h b/src/include/fst/script/map.h new file mode 100644 index 0000000..2332074 --- /dev/null +++ b/src/include/fst/script/map.h @@ -0,0 +1,115 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_MAP_H_ +#define FST_SCRIPT_MAP_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> +#include <fst/arc-map.h> +#include <fst/state-map.h> + +namespace fst { +namespace script { + +template <class M> +Fst<typename M::ToArc> *ArcMap(const Fst<typename M::FromArc> &fst, + const M &mapper) { + typedef typename M::ToArc ToArc; + VectorFst<ToArc> *ofst = new VectorFst<ToArc>; + ArcMap(fst, ofst, mapper); + return ofst; +} + +template <class M> +Fst<typename M::ToArc> *StateMap(const Fst<typename M::FromArc> &fst, + const M &mapper) { + typedef typename M::ToArc ToArc; + VectorFst<ToArc> *ofst = new VectorFst<ToArc>; + StateMap(fst, ofst, mapper); + return ofst; +} + +enum MapType { ARC_SUM_MAPPER, IDENTITY_MAPPER, INVERT_MAPPER, PLUS_MAPPER, + QUANTIZE_MAPPER, RMWEIGHT_MAPPER, SUPERFINAL_MAPPER, + TIMES_MAPPER, TO_LOG_MAPPER, TO_LOG64_MAPPER, TO_STD_MAPPER }; + +typedef args::Package<const FstClass&, MapType, float, + const WeightClass &> MapInnerArgs; +typedef args::WithReturnValue<FstClass*, MapInnerArgs> MapArgs; + +template <class Arc> +void Map(MapArgs *args) { + const Fst<Arc> &ifst = *(args->args.arg1.GetFst<Arc>()); + MapType map_type = args->args.arg2; + float delta = args->args.arg3; + typename Arc::Weight w = *(args->args.arg4.GetWeight<typename Arc::Weight>()); + + if (map_type == ARC_SUM_MAPPER) { + args->retval = new FstClass( + script::StateMap(ifst, ArcSumMapper<Arc>(ifst))); + } else if (map_type == IDENTITY_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, IdentityArcMapper<Arc>())); + } else if (map_type == INVERT_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, InvertWeightMapper<Arc>())); + } else if (map_type == PLUS_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, PlusMapper<Arc>(w))); + } else if (map_type == QUANTIZE_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, QuantizeMapper<Arc>(delta))); + } else if (map_type == RMWEIGHT_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, RmWeightMapper<Arc>())); + } else if (map_type == SUPERFINAL_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, SuperFinalMapper<Arc>())); + } else if (map_type == TIMES_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, TimesMapper<Arc>(w))); + } else if (map_type == TO_LOG_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, WeightConvertMapper<Arc, LogArc>())); + } else if (map_type == TO_LOG64_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, WeightConvertMapper<Arc, Log64Arc>())); + } else if (map_type == TO_STD_MAPPER) { + args->retval = new FstClass( + script::ArcMap(ifst, WeightConvertMapper<Arc, StdArc>())); + } else { + FSTERROR() << "Error: unknown/unsupported mapper type: " + << map_type; + VectorFst<Arc> *ofst = new VectorFst<Arc>; + ofst->SetProperties(kError, kError); + args->retval = new FstClass(ofst); + } +} + + +#ifdef SWIG +%newobject Map; +#endif +FstClass *Map(const FstClass& f, MapType map_type, + float delta = fst::kDelta, + const WeightClass &w = fst::script::WeightClass::Zero()); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_MAP_H_ diff --git a/src/include/fst/script/minimize.h b/src/include/fst/script/minimize.h new file mode 100644 index 0000000..f250d03 --- /dev/null +++ b/src/include/fst/script/minimize.h @@ -0,0 +1,45 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_MINIMIZE_H_ +#define FST_SCRIPT_MINIMIZE_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/minimize.h> + +namespace fst { +namespace script { + +typedef args::Package<MutableFstClass*, MutableFstClass*, float> MinimizeArgs; + +template<class Arc> +void Minimize(MinimizeArgs *args) { + MutableFst<Arc> *ofst1 = args->arg1->GetMutableFst<Arc>(); + MutableFst<Arc> *ofst2 = args->arg2 ? args->arg2->GetMutableFst<Arc>() : 0; + + Minimize(ofst1, ofst2, args->arg3); +} + +void Minimize(MutableFstClass *ofst1, MutableFstClass *ofst2 = 0, + float delta = kDelta); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_MINIMIZE_H_ diff --git a/src/include/fst/script/print-impl.h b/src/include/fst/script/print-impl.h new file mode 100644 index 0000000..1433a29 --- /dev/null +++ b/src/include/fst/script/print-impl.h @@ -0,0 +1,149 @@ +// print.h + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: riley@google.com (Michael Riley) +// +// \file +// Stand-alone class to print out binary FSTs in the AT&T format, +// helper class for fstprint.cc + +#ifndef FST_SCRIPT_PRINT_IMPL_H_ +#define FST_SCRIPT_PRINT_IMPL_H_ + +#include <sstream> +#include <string> + +#include <fst/fst.h> +#include <fst/util.h> + +DECLARE_string(fst_field_separator); + +namespace fst { + +// Print a binary Fst in textual format, helper class for fstprint.cc +// WARNING: Stand-alone use of this class not recommended, most code should +// read/write using the binary format which is much more efficient. +template <class A> class FstPrinter { + public: + typedef A Arc; + typedef typename A::StateId StateId; + typedef typename A::Label Label; + typedef typename A::Weight Weight; + + FstPrinter(const Fst<A> &fst, + const SymbolTable *isyms, + const SymbolTable *osyms, + const SymbolTable *ssyms, + bool accep, + bool show_weight_one) + : fst_(fst), isyms_(isyms), osyms_(osyms), ssyms_(ssyms), + accep_(accep && fst.Properties(kAcceptor, true)), ostrm_(0), + show_weight_one_(show_weight_one) {} + + // Print Fst to an output stream + void Print(ostream *ostrm, const string &dest) { + ostrm_ = ostrm; + dest_ = dest; + StateId start = fst_.Start(); + if (start == kNoStateId) + return; + // initial state first + PrintState(start); + for (StateIterator< Fst<A> > siter(fst_); + !siter.Done(); + siter.Next()) { + StateId s = siter.Value(); + if (s != start) + PrintState(s); + } + } + + private: + // Maximum line length in text file. + static const int kLineLen = 8096; + + void PrintId(int64 id, const SymbolTable *syms, + const char *name) const { + if (syms) { + string symbol = syms->Find(id); + if (symbol == "") { + FSTERROR() << "FstPrinter: Integer " << id + << " is not mapped to any textual symbol" + << ", symbol table = " << syms->Name() + << ", destination = " << dest_; + symbol = "?"; + } + *ostrm_ << symbol; + } else { + *ostrm_ << id; + } + } + + void PrintStateId(StateId s) const { + PrintId(s, ssyms_, "state ID"); + } + + void PrintILabel(Label l) const { + PrintId(l, isyms_, "arc input label"); + } + + void PrintOLabel(Label l) const { + PrintId(l, osyms_, "arc output label"); + } + + void PrintState(StateId s) const { + bool output = false; + for (ArcIterator< Fst<A> > aiter(fst_, s); + !aiter.Done(); + aiter.Next()) { + Arc arc = aiter.Value(); + PrintStateId(s); + *ostrm_ << FLAGS_fst_field_separator[0]; + PrintStateId(arc.nextstate); + *ostrm_ << FLAGS_fst_field_separator[0]; + PrintILabel(arc.ilabel); + if (!accep_) { + *ostrm_ << FLAGS_fst_field_separator[0]; + PrintOLabel(arc.olabel); + } + if (show_weight_one_ || arc.weight != Weight::One()) + *ostrm_ << FLAGS_fst_field_separator[0] << arc.weight; + *ostrm_ << "\n"; + output = true; + } + Weight final = fst_.Final(s); + if (final != Weight::Zero() || !output) { + PrintStateId(s); + if (show_weight_one_ || final != Weight::One()) { + *ostrm_ << FLAGS_fst_field_separator[0] << final; + } + *ostrm_ << "\n"; + } + } + + const Fst<A> &fst_; + const SymbolTable *isyms_; // ilabel symbol table + const SymbolTable *osyms_; // olabel symbol table + const SymbolTable *ssyms_; // slabel symbol table + bool accep_; // print as acceptor when possible + ostream *ostrm_; // text FST destination + string dest_; // text FST destination name + bool show_weight_one_; // print weights equal to Weight::One() + DISALLOW_COPY_AND_ASSIGN(FstPrinter); +}; + +} // namespace fst + +#endif // FST_SCRIPT_PRINT_IMPL_H_ diff --git a/src/include/fst/script/print.h b/src/include/fst/script/print.h new file mode 100644 index 0000000..f82b19b --- /dev/null +++ b/src/include/fst/script/print.h @@ -0,0 +1,86 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_PRINT_H_ +#define FST_SCRIPT_PRINT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/print-impl.h> + +namespace fst { +namespace script { + +// Note: it is safe to pass these strings as references because +// this struct is only used to pass them deeper in the call graph. +// Be sure you understand why this is so before using this struct +// for anything else! +struct FstPrinterArgs { + const FstClass &fst; + const SymbolTable *isyms; + const SymbolTable *osyms; + const SymbolTable *ssyms; + const bool accept; + const bool show_weight_one; + ostream *ostrm; + const string &dest; + + FstPrinterArgs(const FstClass &fst, + const SymbolTable *isyms, + const SymbolTable *osyms, + const SymbolTable *ssyms, + bool accept, + bool show_weight_one, + ostream *ostrm, + const string &dest) : + fst(fst), isyms(isyms), osyms(osyms), ssyms(ssyms), accept(accept), + show_weight_one(show_weight_one), ostrm(ostrm), dest(dest) { } +}; + +template<class Arc> +void PrintFst(FstPrinterArgs *args) { + const Fst<Arc> &fst = *(args->fst.GetFst<Arc>()); + + fst::FstPrinter<Arc> fstprinter(fst, args->isyms, args->osyms, + args->ssyms, args->accept, + args->show_weight_one); + fstprinter.Print(args->ostrm, args->dest); +} + +void PrintFst(const FstClass &fst, ostream &ostrm, const string &dest, + const SymbolTable *isyms, + const SymbolTable *osyms, + const SymbolTable *ssyms, + bool accept, bool show_weight_one); + + +// Below are two printing methods with useful defaults for a few of +// the fst printer arguments. +template <class Arc> +void PrintFst(const Fst<Arc> &fst, ostream &os, const string dest = "", + const SymbolTable *isyms = NULL, + const SymbolTable *osyms = NULL, + const SymbolTable *ssyms = NULL) { + fst::FstPrinter<Arc> fstprinter(fst, isyms, osyms, ssyms, true, true); + fstprinter.Print(&os, dest); +} + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_PRINT_H_ diff --git a/src/include/fst/script/project.h b/src/include/fst/script/project.h new file mode 100644 index 0000000..12ee890 --- /dev/null +++ b/src/include/fst/script/project.h @@ -0,0 +1,43 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_PROJECT_H_ +#define FST_SCRIPT_PROJECT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/project.h> // for ProjectType + +namespace fst { +namespace script { + +typedef args::Package<MutableFstClass*, ProjectType> ProjectArgs; + +template<class Arc> +void Project(ProjectArgs *args) { + MutableFst<Arc> *ofst = args->arg1->GetMutableFst<Arc>(); + + Project(ofst, args->arg2); +} + +void Project(MutableFstClass *ofst, ProjectType project_type); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_PROJECT_H_ diff --git a/src/include/fst/script/prune.h b/src/include/fst/script/prune.h new file mode 100644 index 0000000..7118ff1 --- /dev/null +++ b/src/include/fst/script/prune.h @@ -0,0 +1,153 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_PRUNE_H_ +#define FST_SCRIPT_PRUNE_H_ + +#include <vector> +using std::vector; + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> +#include <fst/prune.h> +#include <fst/arcfilter.h> + +namespace fst { +namespace script { + +struct PruneOptions { + WeightClass weight_threshold; + int64 state_threshold; + const vector<WeightClass> *distance; + float delta; + + explicit PruneOptions(const WeightClass& w, int64 s, + vector<WeightClass> *d = 0, float e = kDelta) + : weight_threshold(w), + state_threshold(s), + distance(d), + delta(e) {} + private: + PruneOptions(); // disallow +}; + +// converts a script::PruneOptions into a fst::PruneOptions. +// Notes: +// If the original opts.distance is not NULL, a new distance will be +// created with new; it's the client's responsibility to delete this. + +template<class A> +fst::PruneOptions<A, AnyArcFilter<A> > ConvertPruneOptions( + const PruneOptions &opts) { + typedef typename A::Weight Weight; + typedef typename A::StateId StateId; + + Weight weight_threshold = *(opts.weight_threshold.GetWeight<Weight>()); + StateId state_threshold = opts.state_threshold; + vector<Weight> *distance = 0; + + if (opts.distance) { + distance = new vector<Weight>(opts.distance->size()); + for (unsigned i = 0; i < opts.distance->size(); ++i) { + (*distance)[i] = *((*opts.distance)[i].GetWeight<Weight>()); + } + } + + return fst::PruneOptions<A, AnyArcFilter<A> >( + weight_threshold, state_threshold, AnyArcFilter<A>(), distance, + opts.delta); +} + +// 1 +typedef args::Package<MutableFstClass *, const PruneOptions &> PruneArgs1; + +template<class Arc> +void Prune(PruneArgs1 *args) { + MutableFst<Arc> *ofst = args->arg1->GetMutableFst<Arc>(); + + typedef typename Arc::Weight Weight; + typedef typename Arc::StateId StateId; + + fst::PruneOptions<Arc, AnyArcFilter<Arc> > opts = + ConvertPruneOptions<Arc>(args->arg2); + Prune(ofst, opts); + delete opts.distance; +} + +// 2 +typedef args::Package<const FstClass &, MutableFstClass *, + const PruneOptions &> PruneArgs2; + +template<class Arc> +void Prune(PruneArgs2 *args) { + const Fst<Arc>& ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + + fst::PruneOptions<Arc, AnyArcFilter<Arc> > opts = + ConvertPruneOptions<Arc>(args->arg3); + Prune(ifst, ofst, opts); + delete opts.distance; +} + +// 3 +typedef args::Package<const FstClass &, + MutableFstClass *, + const WeightClass &, int64, float> PruneArgs3; + +template<class Arc> +void Prune(PruneArgs3 *args) { + const Fst<Arc>& ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + typename Arc::Weight w = *(args->arg3.GetWeight<typename Arc::Weight>()); + + Prune(ifst, ofst, w, args->arg4, args->arg5); +} + +// 4 +typedef args::Package<MutableFstClass *, const WeightClass&, + int64, float> PruneArgs4; +template<class Arc> +void Prune(PruneArgs4 *args) { + MutableFst<Arc> *fst = args->arg1->GetMutableFst<Arc>(); + typename Arc::Weight w = *(args->arg2.GetWeight<typename Arc::Weight>()); + Prune(fst, w, args->arg3, args->arg4); +} + + +// 1 +void Prune(MutableFstClass *fst, const PruneOptions &opts); + +// 2 +void Prune(const FstClass &ifst, MutableFstClass *fst, + const PruneOptions &opts); + +// 3 +void Prune(const FstClass &ifst, MutableFstClass *ofst, + const WeightClass &weight_threshold, + int64 state_threshold = kNoStateId, + float delta = kDelta); + +// 4 +void Prune(MutableFstClass *fst, const WeightClass& weight_threshold, + int64 state_threshold, float delta); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_PRUNE_H_ diff --git a/src/include/fst/script/push.h b/src/include/fst/script/push.h new file mode 100644 index 0000000..cebd655 --- /dev/null +++ b/src/include/fst/script/push.h @@ -0,0 +1,70 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_PUSH_H_ +#define FST_SCRIPT_PUSH_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/push.h> + +namespace fst { +namespace script { + +// 1 +typedef args::Package<MutableFstClass*, ReweightType, float, bool> PushArgs1; + +template<class Arc> +void Push(PushArgs1 *args) { + MutableFst<Arc> *ofst = args->arg1->GetMutableFst<Arc>(); + + if (args->arg2 == REWEIGHT_TO_FINAL) { + fst::Push(ofst, REWEIGHT_TO_FINAL, args->arg3, args->arg4); + } else { + fst::Push(ofst, REWEIGHT_TO_INITIAL, args->arg3, args->arg4); + } +} + +// 2 +typedef args::Package<const FstClass &, MutableFstClass *, uint32, + ReweightType, float> PushArgs2; + +template<class Arc> +void Push(PushArgs2 *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + + if (args->arg4 == REWEIGHT_TO_FINAL) { + fst::Push<Arc, REWEIGHT_TO_FINAL>(ifst, ofst, args->arg3, args->arg5); + } else { + fst::Push<Arc, REWEIGHT_TO_INITIAL>(ifst, ofst, args->arg3, args->arg5); + } +} + +// 1 +void Push(MutableFstClass *ofst, ReweightType type, float delta = kDelta, + bool remove_total_weight = false); + +// 2 +void Push(const FstClass &ifst, MutableFstClass *ofst, uint32 flags, + ReweightType dir, float delta); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_PUSH_H_ diff --git a/src/include/fst/script/randequivalent.h b/src/include/fst/script/randequivalent.h new file mode 100644 index 0000000..b929683 --- /dev/null +++ b/src/include/fst/script/randequivalent.h @@ -0,0 +1,105 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_RANDEQUIVALENT_H_ +#define FST_SCRIPT_RANDEQUIVALENT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/randgen.h> // for RandArcSelection +#include <fst/randequivalent.h> + +namespace fst { +namespace script { + +// 1 +typedef args::Package<const FstClass&, const FstClass&, + int32, float, int, int> RandEquivalentInnerArgs1; +typedef args::WithReturnValue<bool, + RandEquivalentInnerArgs1> RandEquivalentArgs1; + +template<class Arc> +void RandEquivalent(RandEquivalentArgs1 *args) { + const Fst<Arc> &fst1 = *(args->args.arg1.GetFst<Arc>()); + const Fst<Arc> &fst2 = *(args->args.arg2.GetFst<Arc>()); + + args->retval = RandEquivalent(fst1, fst2, args->args.arg3, args->args.arg4, + args->args.arg5, args->args.arg6); +} + +// 2 +typedef args::Package<const FstClass &, const FstClass &, int32, + ssize_t, float, + const RandGenOptions<RandArcSelection> &> + RandEquivalentInnerArgs2; + +typedef args::WithReturnValue<bool, + RandEquivalentInnerArgs2> RandEquivalentArgs2; + +template<class Arc> +void RandEquivalent(RandEquivalentArgs2 *args) { + const Fst<Arc> &fst1 = *(args->args.arg1.GetFst<Arc>()); + const Fst<Arc> &fst2 = *(args->args.arg2.GetFst<Arc>()); + const RandGenOptions<RandArcSelection> &opts = args->args.arg6; + int32 seed = args->args.arg3; + + if (opts.arc_selector == UNIFORM_ARC_SELECTOR) { + UniformArcSelector<Arc> arc_selector(seed); + RandGenOptions< UniformArcSelector<Arc> > + ropts(arc_selector, opts.max_length, opts.npath); + + args->retval = RandEquivalent(fst1, fst2, args->args.arg4, + args->args.arg5, ropts); + } else if (opts.arc_selector == FAST_LOG_PROB_ARC_SELECTOR) { + FastLogProbArcSelector<Arc> arc_selector(seed); + RandGenOptions< FastLogProbArcSelector<Arc> > + ropts(arc_selector, opts.max_length, opts.npath); + + args->retval = RandEquivalent(fst1, fst2, args->args.arg4, + args->args.arg5, ropts); + } else { + LogProbArcSelector<Arc> arc_selector(seed); + RandGenOptions< LogProbArcSelector<Arc> > + ropts(arc_selector, opts.max_length, opts.npath); + args->retval = RandEquivalent(fst1, fst2, args->args.arg4, + args->args.arg5, ropts); + } +} + + +// 1 +bool RandEquivalent(const FstClass &fst1, + const FstClass &fst2, + int32 seed = time(0), + ssize_t num_paths = 1, + float delta = fst::kDelta, + int path_length = INT_MAX); + +// 2 +bool RandEquivalent(const FstClass &fst1, + const FstClass &fst2, + int32 seed, + ssize_t num_paths, + float delta, + const fst::RandGenOptions< + fst::script::RandArcSelection> &opts); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_RANDEQUIVALENT_H_ diff --git a/src/include/fst/script/randgen.h b/src/include/fst/script/randgen.h new file mode 100644 index 0000000..817f9c1 --- /dev/null +++ b/src/include/fst/script/randgen.h @@ -0,0 +1,76 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_RANDGEN_H_ +#define FST_SCRIPT_RANDGEN_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/randgen.h> + +namespace fst { +namespace script { + +enum RandArcSelection { + UNIFORM_ARC_SELECTOR, + LOG_PROB_ARC_SELECTOR, + FAST_LOG_PROB_ARC_SELECTOR +}; + +typedef args::Package<const FstClass &, MutableFstClass*, int32, + const RandGenOptions<RandArcSelection> &> RandGenArgs; + +template<class Arc> +void RandGen(RandGenArgs *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + int32 seed = args->arg3; + const RandGenOptions<RandArcSelection> &opts = args->arg4; + + if (opts.arc_selector == UNIFORM_ARC_SELECTOR) { + UniformArcSelector<Arc> arc_selector(seed); + RandGenOptions< UniformArcSelector<Arc> > + ropts(arc_selector, opts.max_length, + opts.npath, opts.weighted); + RandGen(ifst, ofst, ropts); + } else if (opts.arc_selector == FAST_LOG_PROB_ARC_SELECTOR) { + FastLogProbArcSelector<Arc> arc_selector(seed); + RandGenOptions< FastLogProbArcSelector<Arc> > + ropts(arc_selector, opts.max_length, + opts.npath, opts.weighted); + RandGen(ifst, ofst, ropts); + } else { + LogProbArcSelector<Arc> arc_selector(seed); + RandGenOptions< LogProbArcSelector<Arc> > + ropts(arc_selector, opts.max_length, + opts.npath, opts.weighted); + RandGen(ifst, ofst, ropts); + } +} + + +// Client-facing prototype +void RandGen(const FstClass &ifst, MutableFstClass *ofst, int32 seed = time(0), + const RandGenOptions<RandArcSelection> &opts = + fst::RandGenOptions<fst::script::RandArcSelection>( + fst::script::UNIFORM_ARC_SELECTOR)); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_RANDGEN_H_ diff --git a/src/include/fst/script/register.h b/src/include/fst/script/register.h new file mode 100644 index 0000000..03e0e36 --- /dev/null +++ b/src/include/fst/script/register.h @@ -0,0 +1,120 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_REGISTER_H_ +#define FST_SCRIPT_REGISTER_H_ + +#include <string> + +#include <fst/generic-register.h> +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> + +// Holds methods and classes responsible for maintaining +// the register for FstClass arc types. + +namespace fst { +namespace script { + +// +// Registers for reading and converting various kinds of FST classes. +// + +// This class definition is to avoid a nested class definition inside +// the IORegistration struct. +template<class Reader, class Creator, class Converter> +struct FstClassRegEntry { + Reader reader; + Creator creator; + Converter converter; + + FstClassRegEntry(Reader r, Creator cr, Converter co) : + reader(r), creator(cr), converter(co) { } + FstClassRegEntry() : reader(0), creator(0), converter(0) { } +}; + +template<class Reader, class Creator, class Converter> +class FstClassIORegister + : public GenericRegister<string, + FstClassRegEntry<Reader, Creator, Converter>, + FstClassIORegister<Reader, Creator, + Converter> > { + public: + Reader GetReader(const string &arc_type) const { + return this->GetEntry(arc_type).reader; + } + + Creator GetCreator(const string &arc_type) const { + return this->GetEntry(arc_type).creator; + } + + Converter GetConverter(const string &arc_type) const { + return this->GetEntry(arc_type).converter; + } + + protected: + virtual string ConvertKeyToSoFilename( + const string& key) const { + string legal_type(key); + ConvertToLegalCSymbol(&legal_type); + + return legal_type + "-arc.so"; + } +}; + +// +// Struct containing everything needed to register a particular type +// of FST class (e.g. a plain FstClass, or a MutableFstClass, etc) +// +template<class FstClassType> +struct IORegistration { + typedef FstClassType *(*Reader)(istream &stream, + const FstReadOptions &opts); + + typedef FstClassImplBase *(*Creator)(); + typedef FstClassImplBase *(*Converter)(const FstClass &other); + + typedef FstClassRegEntry<Reader, Creator, Converter> Entry; + + // FST class Register + typedef FstClassIORegister<Reader, Creator, Converter> Register; + + // FST class Register-er + typedef GenericRegisterer<FstClassIORegister<Reader, Creator, Converter> > + Registerer; +}; + + +// +// REGISTRATION MACROS +// + +#define REGISTER_FST_CLASS(Class, Arc) \ + static IORegistration<Class>::Registerer Class ## _ ## Arc ## _registerer( \ + Arc::Type(), \ + IORegistration<Class>::Entry(Class::Read<Arc>, \ + Class::Create<Arc>, \ + Class::Convert<Arc>)) + +#define REGISTER_FST_CLASSES(Arc) \ + REGISTER_FST_CLASS(FstClass, Arc); \ + REGISTER_FST_CLASS(MutableFstClass, Arc); \ + REGISTER_FST_CLASS(VectorFstClass, Arc); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_REGISTER_H_ diff --git a/src/include/fst/script/relabel.h b/src/include/fst/script/relabel.h new file mode 100644 index 0000000..6bbb4c5 --- /dev/null +++ b/src/include/fst/script/relabel.h @@ -0,0 +1,102 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_RELABEL_H_ +#define FST_SCRIPT_RELABEL_H_ + +#include <utility> +using std::pair; using std::make_pair; +#include <algorithm> +#include <vector> +using std::vector; + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/relabel.h> + +namespace fst { +namespace script { + +// 1 +typedef args::Package<MutableFstClass *, + const SymbolTable *, const SymbolTable *, bool, + const SymbolTable *, const SymbolTable *, + bool> RelabelArgs1; + +template<class Arc> +void Relabel(RelabelArgs1 *args) { + MutableFst<Arc> *ofst = args->arg1->GetMutableFst<Arc>(); + + Relabel(ofst, args->arg2, args->arg3, args->arg4, + args->arg5, args->arg6, args->arg7); +} + +// 2 +typedef args::Package<MutableFstClass*, + const vector<pair<int64, int64> > &, + const vector<pair<int64, int64> > > RelabelArgs2; + +template<class Arc> +void Relabel(RelabelArgs2 *args) { + MutableFst<Arc> *ofst = args->arg1->GetMutableFst<Arc>(); + + // In case int64 is not the same as Arc::Label, + // copy the reassignments + typedef typename Arc::Label Label; + + vector<pair<Label, Label> > converted_ipairs(args->arg2.size()); + copy(args->arg2.begin(), args->arg2.end(), converted_ipairs.begin()); + + vector<pair<Label, Label> > converted_opairs(args->arg3.size()); + copy(args->arg3.begin(), args->arg3.end(), converted_opairs.begin()); + + Relabel(ofst, converted_ipairs, converted_opairs); +} + +// 3 +typedef args::Package<MutableFstClass*, const SymbolTable*, + const SymbolTable*> RelabelArgs3; +template<class Arc> +void Relabel(args::Package<MutableFstClass*, const SymbolTable*, + const SymbolTable*> *args) { + MutableFst<Arc> *fst = args->arg1->GetMutableFst<Arc>(); + Relabel(fst, args->arg2, args->arg3); +} + + +// 1 +void Relabel(MutableFstClass *ofst, + const SymbolTable *old_isyms, const SymbolTable *relabel_isyms, + bool attach_new_isyms, + const SymbolTable *old_osyms, const SymbolTable *relabel_osyms, + bool attch_new_osyms); + +// 2 +void Relabel(MutableFstClass *ofst, + const vector<pair<int64, int64> > &ipairs, + const vector<pair<int64, int64> > &opairs); + + +// 3 +void Relabel(MutableFstClass *fst, + const SymbolTable *new_isymbols, + const SymbolTable *new_osymbols); + + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_RELABEL_H_ diff --git a/src/include/fst/script/replace.h b/src/include/fst/script/replace.h new file mode 100644 index 0000000..5eaf5bf --- /dev/null +++ b/src/include/fst/script/replace.h @@ -0,0 +1,62 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_REPLACE_H_ +#define FST_SCRIPT_REPLACE_H_ + +#include <utility> +using std::pair; using std::make_pair; +#include <vector> +using std::vector; + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/replace.h> + +namespace fst { +namespace script { + +typedef args::Package<const vector<pair<int64, const FstClass *> > &, + MutableFstClass *, const int64, bool> ReplaceArgs; + +template<class Arc> +void Replace(ReplaceArgs *args) { + // Now that we know the arc type, we construct a vector of + // pair<real label, real fst> that the real Replace will use + const vector<pair<int64, const FstClass *> >& untyped_tuples = + args->arg1; + + vector<pair<typename Arc::Label, const Fst<Arc> *> > fst_tuples( + untyped_tuples.size()); + + for (unsigned i = 0; i < untyped_tuples.size(); ++i) { + fst_tuples[i].first = untyped_tuples[i].first; // convert label + fst_tuples[i].second = untyped_tuples[i].second->GetFst<Arc>(); + } + + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + + Replace(fst_tuples, ofst, args->arg3, args->arg4); +} + +void Replace(const vector<pair<int64, const FstClass *> > &tuples, + MutableFstClass *ofst, const int64 &root, + bool epsilon_on_replace = false); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_REPLACE_H_ diff --git a/src/include/fst/script/reverse.h b/src/include/fst/script/reverse.h new file mode 100644 index 0000000..3930875 --- /dev/null +++ b/src/include/fst/script/reverse.h @@ -0,0 +1,42 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_REVERSE_H_ +#define FST_SCRIPT_REVERSE_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/reverse.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass &, MutableFstClass *> ReverseArgs; + +template<class Arc> +void Reverse(ReverseArgs *args) { + const Fst<Arc> &fst1 = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *fst2 = args->arg2->GetMutableFst<Arc>(); + + Reverse(fst1, fst2); +} + +void Reverse(const FstClass &fst1, MutableFstClass *fst2); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_REVERSE_H_ diff --git a/src/include/fst/script/reweight.h b/src/include/fst/script/reweight.h new file mode 100644 index 0000000..7bce839 --- /dev/null +++ b/src/include/fst/script/reweight.h @@ -0,0 +1,53 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_REWEIGHT_H_ +#define FST_SCRIPT_REWEIGHT_H_ + +#include <vector> +using std::vector; + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> +#include <fst/reweight.h> + +namespace fst { +namespace script { + +typedef args::Package<MutableFstClass *, const vector<WeightClass> &, + ReweightType> ReweightArgs; + +template<class Arc> +void Reweight(ReweightArgs *args) { + MutableFst<Arc> *fst = args->arg1->GetMutableFst<Arc>(); + typedef typename Arc::Weight Weight; + vector<Weight> potentials(args->arg2.size()); + + for (unsigned i = 0; i < args->arg2.size(); ++i) { + potentials[i] = *(args->arg2[i].GetWeight<Weight>()); + } + + Reweight(fst, potentials, args->arg3); +} + +void Reweight(MutableFstClass *fst, const vector<WeightClass> &potential, + ReweightType reweight_type); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_REWEIGHT_H_ diff --git a/src/include/fst/script/rmepsilon.h b/src/include/fst/script/rmepsilon.h new file mode 100644 index 0000000..62fed03 --- /dev/null +++ b/src/include/fst/script/rmepsilon.h @@ -0,0 +1,211 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_RMEPSILON_H_ +#define FST_SCRIPT_RMEPSILON_H_ + +#include <vector> +using std::vector; + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> +#include <fst/script/shortest-distance.h> // for ShortestDistanceOptions +#include <fst/rmepsilon.h> +#include <fst/queue.h> + +// the following is necessary, or SWIG complains mightily about +// shortestdistanceoptions not being defined before being used as a base. +#ifdef SWIG +%include "nlp/fst/script/shortest-distance.h" +#endif + + +namespace fst { +namespace script { + +// +// OPTIONS +// + +struct RmEpsilonOptions : public fst::script::ShortestDistanceOptions { + bool connect; + WeightClass weight_threshold; + int64 state_threshold; + + RmEpsilonOptions(QueueType qt = AUTO_QUEUE, float d = kDelta, bool c = true, + WeightClass w = fst::script::WeightClass::Zero(), + int64 n = kNoStateId) + : ShortestDistanceOptions(qt, EPSILON_ARC_FILTER, + kNoStateId, d), + connect(c), weight_threshold(w), state_threshold(n) { } +}; + + +// +// TEMPLATES +// + +// this function takes care of transforming a script-land RmEpsilonOptions +// into a lib-land RmEpsilonOptions +template<class Arc> +void RmEpsilonHelper(MutableFst<Arc> *fst, + vector<typename Arc::Weight> *distance, + const RmEpsilonOptions &opts) { + typedef typename Arc::StateId StateId; + typedef typename Arc::Weight Weight; + + typename Arc::Weight weight_thresh = + *(opts.weight_threshold.GetWeight<Weight>()); + + switch (opts.queue_type) { + case AUTO_QUEUE: { + AutoQueue<StateId> queue(*fst, distance, EpsilonArcFilter<Arc>()); + fst::RmEpsilonOptions<Arc, AutoQueue<StateId> > ropts( + &queue, opts.delta, opts.connect, weight_thresh, + opts.state_threshold); + RmEpsilon(fst, distance, ropts); + break; + } + case FIFO_QUEUE: { + FifoQueue<StateId> queue; + fst::RmEpsilonOptions<Arc, FifoQueue<StateId> > ropts( + &queue, opts.delta, opts.connect, weight_thresh, + opts.state_threshold); + RmEpsilon(fst, distance, ropts); + break; + } + case LIFO_QUEUE: { + LifoQueue<StateId> queue; + fst::RmEpsilonOptions<Arc, LifoQueue<StateId> > ropts( + &queue, opts.delta, opts.connect, weight_thresh, + opts.state_threshold); + RmEpsilon(fst, distance, ropts); + break; + } + case SHORTEST_FIRST_QUEUE: { + NaturalShortestFirstQueue<StateId, Weight> queue(*distance); + fst::RmEpsilonOptions<Arc, NaturalShortestFirstQueue<StateId, + Weight> > ropts( + &queue, opts.delta, opts.connect, weight_thresh, + opts.state_threshold); + RmEpsilon(fst, distance, ropts); + break; + } + case STATE_ORDER_QUEUE: { + StateOrderQueue<StateId> queue; + fst::RmEpsilonOptions<Arc, StateOrderQueue<StateId> > ropts( + &queue, opts.delta, opts.connect, weight_thresh, + opts.state_threshold); + RmEpsilon(fst, distance, ropts); + break; + } + case TOP_ORDER_QUEUE: { + TopOrderQueue<StateId> queue(*fst, EpsilonArcFilter<Arc>()); + fst::RmEpsilonOptions<Arc, TopOrderQueue<StateId> > ropts( + &queue, opts.delta, opts.connect, weight_thresh, + opts.state_threshold); + RmEpsilon(fst, distance, ropts); + break; + } + default: + FSTERROR() << "Unknown or unsupported queue type: " << opts.queue_type; + fst->SetProperties(kError, kError); + } +} + +// 1 +typedef args::Package<const FstClass &, MutableFstClass *, + bool, const RmEpsilonOptions &> RmEpsilonArgs1; + +template<class Arc> +void RmEpsilon(RmEpsilonArgs1 *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + vector<typename Arc::Weight> distance; + bool reverse = args->arg3; + + if (reverse) { + VectorFst<Arc> rfst; + Reverse(ifst, &rfst); + RmEpsilonHelper(&rfst, &distance, args->arg4); + Reverse(rfst, ofst); + } else { + *ofst = ifst; + } + RmEpsilonHelper(ofst, &distance, args->arg4); +} + +// 2 +typedef args::Package<MutableFstClass *, bool, + const WeightClass, int64, + float> RmEpsilonArgs2; + +template<class Arc> +void RmEpsilon(RmEpsilonArgs2 *args) { + MutableFst<Arc> *fst = args->arg1->GetMutableFst<Arc>(); + typename Arc::Weight w = *(args->arg3.GetWeight<typename Arc::Weight>()); + + RmEpsilon(fst, args->arg2, w, args->arg4, args->arg5); +} + +// 3 +typedef args::Package<MutableFstClass *, vector<WeightClass> *, + const RmEpsilonOptions &> RmEpsilonArgs3; + +template<class Arc> +void RmEpsilon(RmEpsilonArgs3 *args) { + MutableFst<Arc> *fst = args->arg1->GetMutableFst<Arc>(); + const RmEpsilonOptions &opts = args->arg3; + + vector<typename Arc::Weight> weights; + + RmEpsilonHelper(fst, &weights, opts); + + // Copy the weights back + args->arg2->resize(weights.size()); + for (unsigned i = 0; i < weights.size(); ++i) { + (*args->arg2)[i] = WeightClass(weights[i]); + } +} + +// +// PROTOTYPES +// + +// 1 +void RmEpsilon(const FstClass &ifst, MutableFstClass *ofst, + bool reverse = false, + const RmEpsilonOptions& opts = + fst::script::RmEpsilonOptions()); + +// 2 +void RmEpsilon(MutableFstClass *arc, bool connect = true, + const WeightClass &weight_threshold = + fst::script::WeightClass::Zero(), + int64 state_threshold = fst::kNoStateId, + float delta = fst::kDelta); + +// 3 +void RmEpsilon(MutableFstClass *fst, vector<WeightClass> *distance, + const RmEpsilonOptions &opts); + + +} // namespace script +} // namespace fst + + +#endif // FST_SCRIPT_RMEPSILON_H_ diff --git a/src/include/fst/script/script-impl.h b/src/include/fst/script/script-impl.h new file mode 100644 index 0000000..452c7c5 --- /dev/null +++ b/src/include/fst/script/script-impl.h @@ -0,0 +1,206 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +// This file defines the registration mechanism for new operations. +// These operations are designed to enable scripts to work with FST classes +// at a high level. + +// If you have a new arc type and want these operations to work with FSTs +// with that arc type, see below for the registration steps +// you must take. + +// These methods are only recommended for use in high-level scripting +// applications. Most users should use the lower-level templated versions +// corresponding to these. + +// If you have a new arc type you'd like these operations to work with, +// use the REGISTER_FST_OPERATIONS macro defined in fstcsript.h + +// If you have a custom operation you'd like to define, you need four +// components. In the following, assume you want to create a new operation +// with the signature +// +// void Foo(const FstClass &ifst, MutableFstClass *ofst); +// +// You need: +// +// 1) A way to bundle the args that your new Foo operation will take, as +// a single struct. The template structs in arg-packs.h provide a handy +// way to do this. In Foo's case, that might look like this: +// +// typedef args::Package<const FstClass &, +// MutableFstClass *> FooArgs; +// +// Note: this package of args is going to be passed by non-const pointer. +// +// 2) A function template that is able to perform Foo, given the args and +// arc type. Yours might look like this: +// +// template<class Arc> +// void Foo(FooArgs *args) { +// // Pull out the actual, arc-templated FSTs +// const Fst<Arc> &ifst = args->arg1.GetFst<Arc>(); +// MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); +// +// // actually perform foo on ifst and ofst... +// } +// +// 3) a client-facing function for your operation. This would look like +// the following: +// +// void Foo(const FstClass &ifst, MutableFstClass *ofst) { +// // Check that the arc types of the FSTs match +// if (!ArcTypesMatch(ifst, *ofst, "Foo")) return; +// // package the args +// FooArgs args(ifst, ofst); +// // Finally, call the operation +// Apply<Operation<FooArgs> >("Foo", ifst->ArcType(), &args); +// } +// +// The Apply<> function template takes care of the link between 2 and 3, +// provided you also have: +// +// 4) A registration for your new operation, on the arc types you care about. +// This can be provided easily by the REGISTER_FST_OPERATION macro in +// operations.h: +// +// REGISTER_FST_OPERATION(Foo, StdArc, FooArgs); +// REGISTER_FST_OPERATION(Foo, MyArc, FooArgs); +// // .. etc +// +// +// That's it! Now when you call Foo(const FstClass &, MutableFstClass *), +// it dispatches (in #3) via the Apply<> function to the correct +// instantiation of the template function in #2. +// + + +#ifndef FST_SCRIPT_SCRIPT_IMPL_H_ +#define FST_SCRIPT_SCRIPT_IMPL_H_ + +// +// This file contains general-purpose templates which are used in the +// implementation of the operations. +// + +#include <utility> +using std::pair; using std::make_pair; +#include <string> + +#include <fst/script/fst-class.h> +#include <fst/generic-register.h> +#include <fst/script/arg-packs.h> + +#include <fst/types.h> + +namespace fst { +namespace script { + +// +// A generic register for operations with various kinds of signatures. +// Needed since every function signature requires a new registration class. +// The pair<string, string> is understood to be the operation name and arc +// type; subclasses (or typedefs) need only provide the operation signature. +// + +template<class OperationSignature> +class GenericOperationRegister + : public GenericRegister<pair<string, string>, + OperationSignature, + GenericOperationRegister<OperationSignature> > { + public: + void RegisterOperation(const string &operation_name, + const string &arc_type, + OperationSignature op) { + this->SetEntry(make_pair(operation_name, arc_type), op); + } + + OperationSignature GetOperation( + const string &operation_name, const string &arc_type) { + return this->GetEntry(make_pair(operation_name, arc_type)); + } + + protected: + virtual string ConvertKeyToSoFilename( + const pair<string, string>& key) const { + // Just use the old-style FST for now. + string legal_type(key.second); // the arc type + ConvertToLegalCSymbol(&legal_type); + + return legal_type + "-arc.so"; + } +}; + + +// Operation package - everything you need to register a new type of operation + +// The ArgPack should be the type that's passed into each wrapped function - +// for instance, it might be a struct containing all the args. +// It's always passed by pointer, so const members should be used to enforce +// constness where it's needed. Return values should be implemented as a +// member of ArgPack as well. + +template<class ArgPack> +struct Operation { + typedef ArgPack Args; + typedef void (*OpType)(ArgPack *args); + + // The register (hash) type + typedef GenericOperationRegister<OpType> Register; + + // The register-er type + typedef GenericRegisterer<Register> Registerer; +}; + + +// Macro for registering new types of operations. + +#define REGISTER_FST_OPERATION(Op, Arc, ArgPack) \ + static fst::script::Operation<ArgPack>::Registerer \ + arc_dispatched_operation_ ## ArgPack ## Op ## Arc ## _registerer( \ + make_pair(#Op, Arc::Type()), Op<Arc>) + + +// +// Template function to apply an operation by name +// + +template<class OpReg> +void Apply(const string &op_name, const string &arc_type, + typename OpReg::Args *args) { + typename OpReg::Register *reg = OpReg::Register::GetRegister(); + + typename OpReg::OpType op = reg->GetOperation(op_name, arc_type); + + if (op == 0) { + FSTERROR() << "No operation found for \"" << op_name << "\" on " + << "arc type " << arc_type; + return; + } + + op(args); +} + + +// Helper that logs to ERROR if the arc types of a and b don't match. +// The op_name is also printed. +bool ArcTypesMatch(const FstClass &a, const FstClass &b, + const string &op_name); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_SCRIPT_IMPL_H_ diff --git a/src/include/fst/script/shortest-distance.h b/src/include/fst/script/shortest-distance.h new file mode 100644 index 0000000..5fc2976 --- /dev/null +++ b/src/include/fst/script/shortest-distance.h @@ -0,0 +1,250 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_SHORTEST_DISTANCE_H_ +#define FST_SCRIPT_SHORTEST_DISTANCE_H_ + +#include <vector> +using std::vector; + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> +#include <fst/script/prune.h> // for ArcFilterType +#include <fst/queue.h> // for QueueType +#include <fst/shortest-distance.h> + +namespace fst { +namespace script { + +enum ArcFilterType { ANY_ARC_FILTER, EPSILON_ARC_FILTER, + INPUT_EPSILON_ARC_FILTER, OUTPUT_EPSILON_ARC_FILTER }; + +// See nlp/fst/lib/shortest-distance.h for the template options class +// that this one shadows +struct ShortestDistanceOptions { + const QueueType queue_type; + const ArcFilterType arc_filter_type; + const int64 source; + const float delta; + const bool first_path; + + ShortestDistanceOptions(QueueType qt, ArcFilterType aft, int64 s, + float d) + : queue_type(qt), arc_filter_type(aft), source(s), delta(d), + first_path(false) { } +}; + + + +// 1 +typedef args::Package<const FstClass &, vector<WeightClass> *, + const ShortestDistanceOptions &> ShortestDistanceArgs1; + +template<class Queue, class Arc, class ArcFilter> +struct QueueConstructor { + // template<class Arc, class ArcFilter> + static Queue *Construct(const Fst<Arc> &, + const vector<typename Arc::Weight> *) { + return new Queue(); + } +}; + +// Specializations to deal with AutoQueue, NaturalShortestFirstQueue, +// and TopOrderQueue's different constructors +template<class Arc, class ArcFilter> +struct QueueConstructor<AutoQueue<typename Arc::StateId>, Arc, ArcFilter> { + // template<class Arc, class ArcFilter> + static AutoQueue<typename Arc::StateId> *Construct( + const Fst<Arc> &fst, + const vector<typename Arc::Weight> *distance) { + return new AutoQueue<typename Arc::StateId>(fst, distance, ArcFilter()); + } +}; + +template<class Arc, class ArcFilter> +struct QueueConstructor<NaturalShortestFirstQueue<typename Arc::StateId, + typename Arc::Weight>, + Arc, ArcFilter> { + // template<class Arc, class ArcFilter> + static NaturalShortestFirstQueue<typename Arc::StateId, typename Arc::Weight> + *Construct(const Fst<Arc> &fst, + const vector<typename Arc::Weight> *distance) { + return new NaturalShortestFirstQueue<typename Arc::StateId, + typename Arc::Weight>(*distance); + } +}; + +template<class Arc, class ArcFilter> +struct QueueConstructor<TopOrderQueue<typename Arc::StateId>, Arc, ArcFilter> { + // template<class Arc, class ArcFilter> + static TopOrderQueue<typename Arc::StateId> *Construct( + const Fst<Arc> &fst, const vector<typename Arc::Weight> *weights) { + return new TopOrderQueue<typename Arc::StateId>(fst, ArcFilter()); + } +}; + + +template<class Arc, class Queue> +void ShortestDistanceHelper(ShortestDistanceArgs1 *args) { + const Fst<Arc> &fst = *(args->arg1.GetFst<Arc>()); + const ShortestDistanceOptions &opts = args->arg3; + + vector<typename Arc::Weight> weights; + + switch (opts.arc_filter_type) { + case ANY_ARC_FILTER: { + Queue *queue = + QueueConstructor<Queue, Arc, AnyArcFilter<Arc> >::Construct( + fst, &weights); + fst::ShortestDistanceOptions<Arc, Queue, AnyArcFilter<Arc> > sdopts( + queue, AnyArcFilter<Arc>(), opts.source, opts.delta); + ShortestDistance(fst, &weights, sdopts); + delete queue; + break; + } + case EPSILON_ARC_FILTER: { + Queue *queue = + QueueConstructor<Queue, Arc, AnyArcFilter<Arc> >::Construct( + fst, &weights); + fst::ShortestDistanceOptions<Arc, Queue, + EpsilonArcFilter<Arc> > sdopts( + queue, EpsilonArcFilter<Arc>(), opts.source, opts.delta); + ShortestDistance(fst, &weights, sdopts); + delete queue; + break; + } + case INPUT_EPSILON_ARC_FILTER: { + Queue *queue = + QueueConstructor<Queue, Arc, InputEpsilonArcFilter<Arc> >::Construct( + fst, &weights); + fst::ShortestDistanceOptions<Arc, Queue, + InputEpsilonArcFilter<Arc> > sdopts( + queue, InputEpsilonArcFilter<Arc>(), opts.source, opts.delta); + ShortestDistance(fst, &weights, sdopts); + delete queue; + break; + } + case OUTPUT_EPSILON_ARC_FILTER: { + Queue *queue = + QueueConstructor<Queue, Arc, + OutputEpsilonArcFilter<Arc> >::Construct( + fst, &weights); + fst::ShortestDistanceOptions<Arc, Queue, + OutputEpsilonArcFilter<Arc> > sdopts( + queue, OutputEpsilonArcFilter<Arc>(), opts.source, opts.delta); + ShortestDistance(fst, &weights, sdopts); + delete queue; + break; + } + } + + // Copy the weights back + args->arg2->resize(weights.size()); + for (unsigned i = 0; i < weights.size(); ++i) { + (*args->arg2)[i] = WeightClass(weights[i]); + } +} + +template<class Arc> +void ShortestDistance(ShortestDistanceArgs1 *args) { + const ShortestDistanceOptions &opts = args->arg3; + typedef typename Arc::StateId StateId; + typedef typename Arc::Weight Weight; + + // Must consider (opts.queue_type x opts.filter_type) options + switch (opts.queue_type) { + default: + FSTERROR() << "Unknown queue type." << opts.queue_type; + + case AUTO_QUEUE: + ShortestDistanceHelper<Arc, AutoQueue<StateId> >(args); + return; + + case FIFO_QUEUE: + ShortestDistanceHelper<Arc, FifoQueue<StateId> >(args); + return; + + case LIFO_QUEUE: + ShortestDistanceHelper<Arc, LifoQueue<StateId> >(args); + return; + + case SHORTEST_FIRST_QUEUE: + ShortestDistanceHelper<Arc, + NaturalShortestFirstQueue<StateId, Weight> >(args); + return; + + case STATE_ORDER_QUEUE: + ShortestDistanceHelper<Arc, StateOrderQueue<StateId> >(args); + return; + + case TOP_ORDER_QUEUE: + ShortestDistanceHelper<Arc, TopOrderQueue<StateId> >(args); + return; + } +} + +// 2 +typedef args::Package<const FstClass&, vector<WeightClass>*, + bool, double> ShortestDistanceArgs2; + +template<class Arc> +void ShortestDistance(ShortestDistanceArgs2 *args) { + const Fst<Arc> &fst = *(args->arg1.GetFst<Arc>()); + vector<typename Arc::Weight> distance; + + ShortestDistance(fst, &distance, args->arg3, args->arg4); + + // convert the typed weights back into weightclass + vector<WeightClass> *retval = args->arg2; + retval->resize(distance.size()); + + for (unsigned i = 0; i < distance.size(); ++i) { + (*retval)[i] = WeightClass(distance[i]); + } +} + +// 3 +typedef args::WithReturnValue<WeightClass, + const FstClass &> ShortestDistanceArgs3; + +template<class Arc> +void ShortestDistance(ShortestDistanceArgs3 *args) { + const Fst<Arc> &fst = *(args->args.GetFst<Arc>()); + + args->retval = WeightClass(ShortestDistance(fst)); +} + + +// 1 +void ShortestDistance(const FstClass &fst, vector<WeightClass> *distance, + const ShortestDistanceOptions &opts); + +// 2 +void ShortestDistance(const FstClass &ifst, vector<WeightClass> *distance, + bool reverse = false, double delta = fst::kDelta); + +#ifndef SWIG +// 3 +WeightClass ShortestDistance(const FstClass &ifst); +#endif + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_SHORTEST_DISTANCE_H_ diff --git a/src/include/fst/script/shortest-path.h b/src/include/fst/script/shortest-path.h new file mode 100644 index 0000000..b3a3eb9 --- /dev/null +++ b/src/include/fst/script/shortest-path.h @@ -0,0 +1,190 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_SHORTEST_PATH_H_ +#define FST_SCRIPT_SHORTEST_PATH_H_ + +#include <vector> +using std::vector; + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/script/weight-class.h> +#include <fst/shortest-path.h> +#include <fst/script/shortest-distance.h> // for ShortestDistanceOptions + +namespace fst { +namespace script { + +struct ShortestPathOptions + : public fst::script::ShortestDistanceOptions { + const size_t nshortest; + const bool unique; + const bool has_distance; + const bool first_path; + const WeightClass weight_threshold; + const int64 state_threshold; + + ShortestPathOptions(QueueType qt, size_t n = 1, + bool u = false, bool hasdist = false, + float d = fst::kDelta, bool fp = false, + WeightClass w = fst::script::WeightClass::Zero(), + int64 s = fst::kNoStateId) + : ShortestDistanceOptions(qt, ANY_ARC_FILTER, kNoStateId, d), + nshortest(n), unique(u), has_distance(hasdist), first_path(fp), + weight_threshold(w), state_threshold(s) { } +}; + +typedef args::Package<const FstClass &, MutableFstClass *, + vector<WeightClass> *, const ShortestPathOptions &> + ShortestPathArgs1; + + +template<class Arc> +void ShortestPath(ShortestPathArgs1 *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + const ShortestPathOptions &opts = args->arg4; + typedef typename Arc::StateId StateId; + typedef typename Arc::Weight Weight; + typedef AnyArcFilter<Arc> ArcFilter; + + vector<typename Arc::Weight> weights; + typename Arc::Weight weight_threshold = + *(opts.weight_threshold.GetWeight<Weight>()); + + switch (opts.queue_type) { + case AUTO_QUEUE: { + typedef AutoQueue<StateId> Queue; + Queue *queue = QueueConstructor<Queue, Arc, + ArcFilter>::Construct(ifst, &weights); + fst::ShortestPathOptions<Arc, Queue, ArcFilter> spopts( + queue, ArcFilter(), opts.nshortest, opts.unique, + opts.has_distance, opts.delta, opts.first_path, + weight_threshold, opts.state_threshold); + ShortestPath(ifst, ofst, &weights, spopts); + delete queue; + return; + } + case FIFO_QUEUE: { + typedef FifoQueue<StateId> Queue; + Queue *queue = QueueConstructor<Queue, Arc, + ArcFilter>::Construct(ifst, &weights); + fst::ShortestPathOptions<Arc, Queue, ArcFilter> spopts( + queue, ArcFilter(), opts.nshortest, opts.unique, + opts.has_distance, opts.delta, opts.first_path, + weight_threshold, opts.state_threshold); + ShortestPath(ifst, ofst, &weights, spopts); + delete queue; + return; + } + case LIFO_QUEUE: { + typedef LifoQueue<StateId> Queue; + Queue *queue = QueueConstructor<Queue, Arc, + ArcFilter >::Construct(ifst, &weights); + fst::ShortestPathOptions<Arc, Queue, ArcFilter> spopts( + queue, ArcFilter(), opts.nshortest, opts.unique, + opts.has_distance, opts.delta, opts.first_path, + weight_threshold, opts.state_threshold); + ShortestPath(ifst, ofst, &weights, spopts); + delete queue; + return; + } + case SHORTEST_FIRST_QUEUE: { + typedef NaturalShortestFirstQueue<StateId, Weight> Queue; + Queue *queue = QueueConstructor<Queue, Arc, + ArcFilter>::Construct(ifst, &weights); + fst::ShortestPathOptions<Arc, Queue, ArcFilter> spopts( + queue, ArcFilter(), opts.nshortest, opts.unique, + opts.has_distance, opts.delta, opts.first_path, + weight_threshold, opts.state_threshold); + ShortestPath(ifst, ofst, &weights, spopts); + delete queue; + return; + } + case STATE_ORDER_QUEUE: { + typedef StateOrderQueue<StateId> Queue; + Queue *queue = QueueConstructor<Queue, Arc, + ArcFilter>::Construct(ifst, &weights); + fst::ShortestPathOptions<Arc, Queue, ArcFilter> spopts( + queue, ArcFilter(), opts.nshortest, opts.unique, + opts.has_distance, opts.delta, opts.first_path, + weight_threshold, opts.state_threshold); + ShortestPath(ifst, ofst, &weights, spopts); + delete queue; + return; + } + case TOP_ORDER_QUEUE: { + typedef TopOrderQueue<StateId> Queue; + Queue *queue = QueueConstructor<Queue, Arc, + ArcFilter>::Construct(ifst, &weights); + fst::ShortestPathOptions<Arc, Queue, ArcFilter> spopts( + queue, ArcFilter(), opts.nshortest, opts.unique, + opts.has_distance, opts.delta, opts.first_path, + weight_threshold, opts.state_threshold); + ShortestPath(ifst, ofst, &weights, spopts); + delete queue; + return; + } + default: + FSTERROR() << "Unknown queue type: " << opts.queue_type; + ofst->SetProperties(kError, kError); + } + + // Copy the weights back + args->arg3->resize(weights.size()); + for (unsigned i = 0; i < weights.size(); ++i) { + (*args->arg3)[i] = WeightClass(weights[i]); + } +} + +// 2 +typedef args::Package<const FstClass &, MutableFstClass *, + size_t, bool, bool, WeightClass, + int64> ShortestPathArgs2; + +template<class Arc> +void ShortestPath(ShortestPathArgs2 *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + typename Arc::Weight weight_threshold = + *(args->arg6.GetWeight<typename Arc::Weight>()); + + ShortestPath(ifst, ofst, args->arg3, args->arg4, args->arg5, + weight_threshold, args->arg7); +} + + +// 1 +void ShortestPath(const FstClass &ifst, MutableFstClass *ofst, + vector<WeightClass> *distance, + const ShortestPathOptions &opts); + + +// 2 +void ShortestPath(const FstClass &ifst, MutableFstClass *ofst, + size_t n = 1, bool unique = false, + bool first_path = false, + WeightClass weight_threshold = + fst::script::WeightClass::Zero(), + int64 state_threshold = fst::kNoStateId); + +} // namespace script +} // namespace fst + + + +#endif // FST_SCRIPT_SHORTEST_PATH_H_ diff --git a/src/include/fst/script/symbols.h b/src/include/fst/script/symbols.h new file mode 100644 index 0000000..927600a --- /dev/null +++ b/src/include/fst/script/symbols.h @@ -0,0 +1,20 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_SYMBOLS_H_ +#define FST_SCRIPT_SYMBOLS_H_ + +#endif // FST_SCRIPT_SYMBOLS_H_ diff --git a/src/include/fst/script/synchronize.h b/src/include/fst/script/synchronize.h new file mode 100644 index 0000000..3c0c905 --- /dev/null +++ b/src/include/fst/script/synchronize.h @@ -0,0 +1,42 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_SYNCHRONIZE_H_ +#define FST_SCRIPT_SYNCHRONIZE_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/synchronize.h> + +namespace fst { +namespace script { + +typedef args::Package<const FstClass &, MutableFstClass *> SynchronizeArgs; + +template<class Arc> +void Synchronize(SynchronizeArgs *args) { + const Fst<Arc> &ifst = *(args->arg1.GetFst<Arc>()); + MutableFst<Arc> *ofst = args->arg2->GetMutableFst<Arc>(); + + Synchronize(ifst, ofst); +} + +void Synchronize(const FstClass &ifst, MutableFstClass *ofst); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_SYNCHRONIZE_H_ diff --git a/src/include/fst/script/text-io.h b/src/include/fst/script/text-io.h new file mode 100644 index 0000000..95cc182 --- /dev/null +++ b/src/include/fst/script/text-io.h @@ -0,0 +1,50 @@ +// text-io.h + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: riley@google.com (Michael Riley) +// Modified: jpr@google.com (Jake Ratkiewicz) to work with generic WeightClass +// +// \file +// Utilities for reading and writing textual strings representing +// states, labels, and weights and files specifying label-label pairs +// and potentials (state-weight pairs). +// + +#ifndef FST_SCRIPT_TEXT_IO_H__ +#define FST_SCRIPT_TEXT_IO_H__ + +#include <string> +#include <vector> +using std::vector; + + +#include <iostream> +#include <fstream> +#include <fst/script/weight-class.h> + +namespace fst { +namespace script { + +bool ReadPotentials(const string &weight_type, + const string& filename, + vector<WeightClass>* potential); + +bool WritePotentials(const string& filename, + const vector<WeightClass>& potential); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_TEXT_IO_H__ diff --git a/src/include/fst/script/topsort.h b/src/include/fst/script/topsort.h new file mode 100644 index 0000000..4e27e48 --- /dev/null +++ b/src/include/fst/script/topsort.h @@ -0,0 +1,40 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_TOPSORT_H_ +#define FST_SCRIPT_TOPSORT_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/topsort.h> + +namespace fst { +namespace script { + +typedef args::WithReturnValue<bool, MutableFstClass*> TopSortArgs; + +template<class Arc> +void TopSort(TopSortArgs *args) { + MutableFst<Arc> *fst = args->args->GetMutableFst<Arc>(); + args->retval = TopSort(fst); +} + +bool TopSort(MutableFstClass *fst); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_TOPSORT_H_ diff --git a/src/include/fst/script/union.h b/src/include/fst/script/union.h new file mode 100644 index 0000000..780e484 --- /dev/null +++ b/src/include/fst/script/union.h @@ -0,0 +1,42 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +#ifndef FST_SCRIPT_UNION_H_ +#define FST_SCRIPT_UNION_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/union.h> + +namespace fst { +namespace script { + +typedef args::Package<MutableFstClass *, const FstClass &> UnionArgs; + +template<class Arc> +void Union(UnionArgs *args) { + MutableFst<Arc> *fst1 = args->arg1->GetMutableFst<Arc>(); + const Fst<Arc> &fst2 = *(args->arg2.GetFst<Arc>()); + + Union(fst1, fst2); +} + +void Union(MutableFstClass *fst1, const FstClass &fst2); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_UNION_H_ diff --git a/src/include/fst/script/verify.h b/src/include/fst/script/verify.h new file mode 100644 index 0000000..6904003 --- /dev/null +++ b/src/include/fst/script/verify.h @@ -0,0 +1,40 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: sorenj@google.com (Jeffrey Sorensen) + +#ifndef FST_SCRIPT_VERIFY_H_ +#define FST_SCRIPT_VERIFY_H_ + +#include <fst/script/arg-packs.h> +#include <fst/script/fst-class.h> +#include <fst/verify.h> + +namespace fst { +namespace script { + +typedef args::WithReturnValue<bool, const FstClass *> VerifyArgs; + +template<class Arc> +void Verify(VerifyArgs *args) { + const Fst<Arc> *fst = args->args->GetFst<Arc>(); + args->retval = Verify(*fst); +} + +bool Verify(const FstClass &fst1); + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_VERIFY_H_ diff --git a/src/include/fst/script/weight-class.h b/src/include/fst/script/weight-class.h new file mode 100644 index 0000000..5a4890f --- /dev/null +++ b/src/include/fst/script/weight-class.h @@ -0,0 +1,216 @@ + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright 2005-2010 Google, Inc. +// Author: jpr@google.com (Jake Ratkiewicz) + +// Represents a generic weight in an FST -- that is, represents a specific +// type of weight underneath while hiding that type from a client. + + +#ifndef FST_SCRIPT_WEIGHT_CLASS_H_ +#define FST_SCRIPT_WEIGHT_CLASS_H_ + +#include <string> + +#include <fst/generic-register.h> +#include <fst/util.h> + +namespace fst { +namespace script { + +class WeightImplBase { + public: + virtual WeightImplBase *Copy() const = 0; + virtual void Print(ostream *o) const = 0; + virtual const string &Type() const = 0; + virtual string to_string() const = 0; + virtual bool operator == (const WeightImplBase &other) const = 0; + virtual ~WeightImplBase() { } +}; + +template<class W> +struct WeightClassImpl : public WeightImplBase { + W weight; + + explicit WeightClassImpl(const W& weight) : weight(weight) { } + + virtual WeightClassImpl<W> *Copy() const { + return new WeightClassImpl<W>(weight); + } + + virtual const string &Type() const { return W::Type(); } + + virtual void Print(ostream *o) const { + *o << weight; + } + + virtual string to_string() const { + ostringstream s; + s << weight; + return s.str(); + } + + virtual bool operator == (const WeightImplBase &other) const { + if (Type() != other.Type()) { + return false; + } else { + const WeightClassImpl<W> *typed_other = + static_cast<const WeightClassImpl<W> *>(&other); + + return typed_other->weight == weight; + } + } +}; + + +class WeightClass { + public: + WeightClass() : element_type_(ZERO), impl_(0) { } + + template<class W> + explicit WeightClass(const W& weight) + : element_type_(OTHER), impl_(new WeightClassImpl<W>(weight)) { } + + WeightClass(const string &weight_type, const string &weight_str); + + WeightClass(const WeightClass &other) : + element_type_(other.element_type_), + impl_(other.impl_ ? other.impl_->Copy() : 0) { } + + WeightClass &operator = (const WeightClass &other) { + if (impl_) delete impl_; + impl_ = other.impl_ ? other.impl_->Copy() : 0; + element_type_ = other.element_type_; + return *this; + } + + template<class W> + const W* GetWeight() const; + + string to_string() const { + switch (element_type_) { + case ZERO: + return "ZERO"; + case ONE: + return "ONE"; + default: + case OTHER: + return impl_->to_string(); + } + } + + bool operator == (const WeightClass &other) const { + return element_type_ == other.element_type_ && + ((impl_ && other.impl_ && (*impl_ == *other.impl_)) || + (impl_ == 0 && other.impl_ == 0)); + } + + static const WeightClass &Zero() { + static WeightClass w(ZERO); + + return w; + } + + static const WeightClass &One() { + static WeightClass w(ONE); + + return w; + } + + ~WeightClass() { if (impl_) delete impl_; } + private: + enum ElementType { ZERO, ONE, OTHER }; + ElementType element_type_; + + WeightImplBase *impl_; + + explicit WeightClass(ElementType et) : element_type_(et), impl_(0) { } + + friend ostream &operator << (ostream &o, const WeightClass &c); +}; + +template<class W> +const W* WeightClass::GetWeight() const { + // We need to store zero and one as statics, because the weight type + // W might return them as temporaries. We're returning a pointer, + // and it won't do to get the address of a temporary. + static const W zero = W::Zero(); + static const W one = W::One(); + + if (element_type_ == ZERO) { + return &zero; + } else if (element_type_ == ONE) { + return &one; + } else { + if (W::Type() != impl_->Type()) { + return NULL; + } else { + WeightClassImpl<W> *typed_impl = + static_cast<WeightClassImpl<W> *>(impl_); + return &typed_impl->weight; + } + } +} + +// +// Registration for generic weight types. +// + +typedef WeightImplBase* (*StrToWeightImplBaseT)(const string &str, + const string &src, + size_t nline); + +template<class W> +WeightImplBase* StrToWeightImplBase(const string &str, + const string &src, size_t nline) { + return new WeightClassImpl<W>(StrToWeight<W>(str, src, nline)); +} + +// The following confuses swig, and doesn't need to be wrapped anyway. +#ifndef SWIG +ostream& operator << (ostream &o, const WeightClass &c); + +class WeightClassRegister : public GenericRegister<string, + StrToWeightImplBaseT, + WeightClassRegister> { + protected: + virtual string ConvertKeyToSoFilename(const string &key) const { + return key + ".so"; + } +}; + +typedef GenericRegisterer<WeightClassRegister> WeightClassRegisterer; +#endif + +// internal version, needs to be called by wrapper in order for +// macro args to expand +#define REGISTER_FST_WEIGHT__(Weight, line) \ + static WeightClassRegisterer weight_registerer ## _ ## line( \ + Weight::Type(), \ + StrToWeightImplBase<Weight>) + +// This layer is where __FILE__ and __LINE__ are expanded +#define REGISTER_FST_WEIGHT_EXPANDER(Weight, line) \ + REGISTER_FST_WEIGHT__(Weight, line) + +// +// Macro for registering new weight types. Clients call this. +// +#define REGISTER_FST_WEIGHT(Weight) \ + REGISTER_FST_WEIGHT_EXPANDER(Weight, __LINE__) + +} // namespace script +} // namespace fst + +#endif // FST_SCRIPT_WEIGHT_CLASS_H_ |