From 5b6dc79427b8f7eeb6a7ff68034ab8548ce670ea Mon Sep 17 00:00:00 2001 From: Alexander Gutkin Date: Thu, 28 Feb 2013 00:24:20 +0000 Subject: Bumped OpenFST implementation to openfst-1.3.3-CL41851770. Updated OpenFST implementation to the most recent version used by Greco3 (corresponds to nlp::fst exported at Perforce CL 41851770). In particular this version has an improved PDT support. Change-Id: I5aadfc962297eef73922c67e7d57866f11ee7d81 --- src/include/fst/compact-fst.h | 89 +++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 38 deletions(-) (limited to 'src/include/fst/compact-fst.h') diff --git a/src/include/fst/compact-fst.h b/src/include/fst/compact-fst.h index 57c927e..6db3317 100644 --- a/src/include/fst/compact-fst.h +++ b/src/include/fst/compact-fst.h @@ -32,6 +32,7 @@ using std::vector; #include #include #include // For optional argument declarations +#include #include #include #include @@ -134,7 +135,9 @@ class CompactFstData { typedef U Unsigned; CompactFstData() - : states_(0), + : states_region_(0), + compacts_region_(0), + states_(0), compacts_(0), nstates_(0), ncompacts_(0), @@ -150,8 +153,14 @@ class CompactFstData { const Compactor &compactor); ~CompactFstData() { - delete[] states_; - delete[] compacts_; + if (states_region_ == NULL) { + delete [] states_; + } + delete states_region_; + if (compacts_region_ == NULL) { + delete [] compacts_; + } + delete compacts_region_; } template @@ -175,10 +184,9 @@ class CompactFstData { bool Error() const { return error_; } - // Byte alignment for states and arcs in file format (version 1 only) - static const int kFileAlign = 16; - private: + MappedFile *states_region_; + MappedFile *compacts_region_; Unsigned *states_; CompactElement *compacts_; size_t nstates_; @@ -189,14 +197,12 @@ class CompactFstData { bool error_; }; -template -const int CompactFstData::kFileAlign; - - template template CompactFstData::CompactFstData(const Fst &fst, const C &compactor) - : states_(0), + : states_region_(0), + compacts_region_(0), + states_(0), compacts_(0), nstates_(0), ncompacts_(0), @@ -265,7 +271,9 @@ template CompactFstData::CompactFstData(const Iterator &begin, const Iterator &end, const C &compactor) - : states_(0), + : states_region_(0), + compacts_region_(0), + states_(0), compacts_(0), nstates_(0), ncompacts_(0), @@ -361,42 +369,40 @@ CompactFstData *CompactFstData::Read( data->narcs_ = hdr.NumArcs(); if (compactor.Size() == -1) { - data->states_ = new Unsigned[data->nstates_ + 1]; - if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && - !AlignInput(strm, kFileAlign)) { + if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && !AlignInput(strm)) { LOG(ERROR) << "CompactFst::Read: Alignment failed: " << opts.source; delete data; return 0; } - // TODO: memory map this size_t b = (data->nstates_ + 1) * sizeof(Unsigned); - strm.read(reinterpret_cast(data->states_), b); - if (!strm) { + data->states_region_ = MappedFile::Map(&strm, opts, b); + if (!strm || data->states_region_ == NULL) { LOG(ERROR) << "CompactFst::Read: Read failed: " << opts.source; delete data; return 0; } + data->states_ = static_cast( + data->states_region_->mutable_data()); } else { data->states_ = 0; } data->ncompacts_ = compactor.Size() == -1 ? data->states_[data->nstates_] : data->nstates_ * compactor.Size(); - data->compacts_ = new CompactElement[data->ncompacts_]; - // TODO: memory map this - size_t b = data->ncompacts_ * sizeof(CompactElement); - if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && - !AlignInput(strm, kFileAlign)) { + if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && !AlignInput(strm)) { LOG(ERROR) << "CompactFst::Read: Alignment failed: " << opts.source; delete data; return 0; } - strm.read(reinterpret_cast(data->compacts_), b); - if (!strm) { + size_t b = data->ncompacts_ * sizeof(CompactElement); + data->compacts_region_ = MappedFile::Map(&strm, opts, b); + if (!strm || data->compacts_region_ == NULL) { LOG(ERROR) << "CompactFst::Read: Read failed: " << opts.source; delete data; return 0; } + data->compacts_ = static_cast( + data->compacts_region_->mutable_data()); return data; } @@ -404,14 +410,14 @@ template bool CompactFstData::Write(ostream &strm, const FstWriteOptions &opts) const { if (states_) { - if (opts.align && !AlignOutput(strm, kFileAlign)) { + if (opts.align && !AlignOutput(strm)) { LOG(ERROR) << "CompactFst::Write: Alignment failed: " << opts.source; return false; } strm.write(reinterpret_cast(states_), (nstates_ + 1) * sizeof(Unsigned)); } - if (opts.align && !AlignOutput(strm, kFileAlign)) { + if (opts.align && !AlignOutput(strm)) { LOG(ERROR) << "CompactFst::Write: Alignment failed: " << opts.source; return false; } @@ -899,6 +905,17 @@ class CompactFst : public ImplToExpandedFst< CompactFstImpl > { ImplToFst< Impl, ExpandedFst >::SetImpl(impl, own_impl); } + // Use overloading to extract the type of the argument. + static Impl* GetImplIfCompactFst(const CompactFst &compact_fst) { + return compact_fst.GetImpl(); + } + + // This does not give privileged treatment to subclasses of CompactFst. + template + static Impl* GetImplIfCompactFst(const NonCompactFst& fst) { + return NULL; + } + void operator=(const CompactFst &fst); // disallow }; @@ -914,20 +931,16 @@ bool CompactFst::WriteFst(const F &fst, typedef U Unsigned; typedef typename C::Element CompactElement; typedef typename A::Weight Weight; - static const int kFileAlign = - CompactFstData::kFileAlign; int file_version = opts.align ? CompactFstImpl::kAlignedFileVersion : CompactFstImpl::kFileVersion; size_t num_arcs = -1, num_states = -1, num_compacts = -1; C first_pass_compactor = compactor; - if (fst.Type() == CompactFst().Type()) { - const CompactFst *compact_fst = - reinterpret_cast *>(&fst); - num_arcs = compact_fst->GetImpl()->Data()->NumArcs(); - num_states = compact_fst->GetImpl()->Data()->NumStates(); - num_compacts = compact_fst->GetImpl()->Data()->NumCompacts(); - first_pass_compactor = *compact_fst->GetImpl()->GetCompactor(); + if (Impl* impl = GetImplIfCompactFst(fst)) { + num_arcs = impl->Data()->NumArcs(); + num_states = impl->Data()->NumStates(); + num_compacts = impl->Data()->NumCompacts(); + first_pass_compactor = *impl->GetCompactor(); } else { // A first pass is needed to compute the state of the compactor, which // is saved ahead of the rest of the data structures. This unfortunately @@ -971,7 +984,7 @@ bool CompactFst::WriteFst(const F &fst, &hdr); first_pass_compactor.Write(strm); if (first_pass_compactor.Size() == -1) { - if (opts.align && !AlignOutput(strm, kFileAlign)) { + if (opts.align && !AlignOutput(strm)) { LOG(ERROR) << "CompactFst::Write: Alignment failed: " << opts.source; return false; } @@ -986,7 +999,7 @@ bool CompactFst::WriteFst(const F &fst, } strm.write(reinterpret_cast(&compacts), sizeof(compacts)); } - if (opts.align && !AlignOutput(strm, kFileAlign)) { + if (opts.align && !AlignOutput(strm)) { LOG(ERROR) << "Could not align file during write after writing states"; } C second_pass_compactor = compactor; -- cgit v1.2.3