From 5b6dc79427b8f7eeb6a7ff68034ab8548ce670ea Mon Sep 17 00:00:00 2001 From: Alexander Gutkin Date: Thu, 28 Feb 2013 00:24:20 +0000 Subject: Bumped OpenFST implementation to openfst-1.3.3-CL41851770. Updated OpenFST implementation to the most recent version used by Greco3 (corresponds to nlp::fst exported at Perforce CL 41851770). In particular this version has an improved PDT support. Change-Id: I5aadfc962297eef73922c67e7d57866f11ee7d81 --- src/include/fst/const-fst.h | 64 +++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 26 deletions(-) (limited to 'src/include/fst/const-fst.h') diff --git a/src/include/fst/const-fst.h b/src/include/fst/const-fst.h index 80efc8d..e6e85af 100644 --- a/src/include/fst/const-fst.h +++ b/src/include/fst/const-fst.h @@ -28,6 +28,7 @@ using std::vector; #include #include // For optional argument declarations +#include #include #include @@ -55,7 +56,8 @@ class ConstFstImpl : public FstImpl { typedef U Unsigned; ConstFstImpl() - : states_(0), arcs_(0), nstates_(0), narcs_(0), start_(kNoStateId) { + : states_region_(0), arcs_region_(0), states_(0), arcs_(0), nstates_(0), + narcs_(0), start_(kNoStateId) { string type = "const"; if (sizeof(U) != sizeof(uint32)) { string size; @@ -69,8 +71,8 @@ class ConstFstImpl : public FstImpl { explicit ConstFstImpl(const Fst &fst); ~ConstFstImpl() { - delete[] states_; - delete[] arcs_; + delete arcs_region_; + delete states_region_; } StateId Start() const { return start_; } @@ -125,9 +127,9 @@ class ConstFstImpl : public FstImpl { static const int kAlignedFileVersion = 1; // Minimum file format version supported static const int kMinFileVersion = 1; - // Byte alignment for states and arcs in file format (version 1 only) - static const int kFileAlign = 16; + MappedFile *states_region_; // Mapped file for states + MappedFile *arcs_region_; // Mapped file for arcs State *states_; // States represenation A *arcs_; // Arcs representation StateId nstates_; // Number of states @@ -145,8 +147,6 @@ template const int ConstFstImpl::kAlignedFileVersion; template const int ConstFstImpl::kMinFileVersion; -template -const int ConstFstImpl::kFileAlign; template @@ -173,8 +173,10 @@ ConstFstImpl::ConstFstImpl(const Fst &fst) : nstates_(0), narcs_(0) { aiter.Next()) ++narcs_; } - states_ = new State[nstates_]; - arcs_ = new A[narcs_]; + states_region_ = MappedFile::Allocate(nstates_ * sizeof(*states_)); + arcs_region_ = MappedFile::Allocate(narcs_ * sizeof(*arcs_)); + states_ = reinterpret_cast(states_region_->mutable_data()); + arcs_ = reinterpret_cast(arcs_region_->mutable_data()); size_t pos = 0; for (StateId s = 0; s < nstates_; ++s) { states_[s].final = fst.Final(s); @@ -210,39 +212,40 @@ ConstFstImpl *ConstFstImpl::Read(istream &strm, impl->start_ = hdr.Start(); impl->nstates_ = hdr.NumStates(); impl->narcs_ = hdr.NumArcs(); - impl->states_ = new State[impl->nstates_]; - impl->arcs_ = new A[impl->narcs_]; // Ensures compatibility if (hdr.Version() == kAlignedFileVersion) hdr.SetFlags(hdr.GetFlags() | FstHeader::IS_ALIGNED); - if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && - !AlignInput(strm, kFileAlign)) { + if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && !AlignInput(strm)) { LOG(ERROR) << "ConstFst::Read: Alignment failed: " << opts.source; delete impl; return 0; } + size_t b = impl->nstates_ * sizeof(typename ConstFstImpl::State); - strm.read(reinterpret_cast(impl->states_), b); - if (!strm) { + impl->states_region_ = MappedFile::Map(&strm, opts, b); + if (!strm || impl->states_region_ == NULL) { LOG(ERROR) << "ConstFst::Read: Read failed: " << opts.source; delete impl; return 0; } - if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && - !AlignInput(strm, kFileAlign)) { + impl->states_ = reinterpret_cast( + impl->states_region_->mutable_data()); + if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && !AlignInput(strm)) { LOG(ERROR) << "ConstFst::Read: Alignment failed: " << opts.source; delete impl; return 0; } + b = impl->narcs_ * sizeof(A); - strm.read(reinterpret_cast(impl->arcs_), b); - if (!strm) { + impl->arcs_region_ = MappedFile::Map(&strm, opts, b); + if (!strm || impl->arcs_region_ == NULL) { LOG(ERROR) << "ConstFst::Read: Read failed: " << opts.source; delete impl; return 0; } + impl->arcs_ = reinterpret_cast(impl->arcs_region_->mutable_data()); return impl; } @@ -318,6 +321,17 @@ class ConstFst : public ImplToExpandedFst< ConstFstImpl > { ImplToFst< Impl, ExpandedFst >::SetImpl(impl, own_impl); } + // Use overloading to extract the type of the argument. + static Impl* GetImplIfConstFst(const ConstFst &const_fst) { + return const_fst.GetImpl(); + } + + // Note that this does not give privileged treatment to subtypes of ConstFst. + template + static Impl* GetImplIfConstFst(const NonConstFst& fst) { + return NULL; + } + void operator=(const ConstFst &fst); // disallow }; @@ -333,11 +347,9 @@ bool ConstFst::WriteFst(const F &fst, ostream &strm, size_t num_arcs = -1, num_states = -1; size_t start_offset = 0; bool update_header = true; - if (fst.Type() == ConstFst().Type()) { - const ConstFst *const_fst = - reinterpret_cast *>(&fst); - num_arcs = const_fst->GetImpl()->narcs_; - num_states = const_fst->GetImpl()->nstates_; + if (Impl* impl = GetImplIfConstFst(fst)) { + num_arcs = impl->narcs_; + num_states = impl->nstates_; update_header = false; } else if ((start_offset = strm.tellp()) == -1) { // precompute values needed for header when we cannot seek to rewrite it. @@ -363,7 +375,7 @@ bool ConstFst::WriteFst(const F &fst, ostream &strm, ConstFstImpl::kStaticProperties; FstImpl::WriteFstHeader(fst, strm, opts, file_version, type, properties, &hdr); - if (opts.align && !AlignOutput(strm, ConstFstImpl::kFileAlign)) { + if (opts.align && !AlignOutput(strm)) { LOG(ERROR) << "Could not align file during write after header"; return false; } @@ -381,7 +393,7 @@ bool ConstFst::WriteFst(const F &fst, ostream &strm, } hdr.SetNumStates(states); hdr.SetNumArcs(pos); - if (opts.align && !AlignOutput(strm, ConstFstImpl::kFileAlign)) { + if (opts.align && !AlignOutput(strm)) { LOG(ERROR) << "Could not align file during write after writing states"; } for (StateIterator siter(fst); !siter.Done(); siter.Next()) { -- cgit v1.2.3