From b887460afe53fbb00b82b936836455913c7a74e5 Mon Sep 17 00:00:00 2001 From: Karel Vesely Date: Tue, 25 Jan 2022 20:01:35 +0100 Subject: [PATCH 1/4] Extending 'lattice-compose.cc' to compose with ark of fsts, - This is a follow-up of https://github.com/kaldi-asr/kaldi/pull/4571 - Refactoring 'lattice-compose.cc' to support composition with ark of fsts, so that it is done as Dan suggested before: I am thinking this can be done with a string arg called e.g. "--compose-with-fst", defaulting to "auto" which is the old behavior, meaning: rspecifier=lats, rxfilename=FST; and true/True or false/False is FST or lattice respectively. - I added there possibility of rho-composition, which is useful for biasing lattices with word-sequences. Thanks to rho-composition, the biasing graph does not need to contain all words from lexicon. - Would you be interested in an example how to use this? (i.e. create graphs from text file with python script using openfst as library, but that would need to change build of openfst to enable python extensions) - Also which 'egs' recipe would be convenient to use it with? --- src/fstext/fstext-utils.h | 2 +- src/latbin/lattice-compose.cc | 179 ++++++++++++++++++++++++++++++---- 2 files changed, 160 insertions(+), 21 deletions(-) diff --git a/src/fstext/fstext-utils.h b/src/fstext/fstext-utils.h index 2810ef0de13..5789dbe7cc3 100644 --- a/src/fstext/fstext-utils.h +++ b/src/fstext/fstext-utils.h @@ -381,7 +381,7 @@ template void PropagateFinal(typename Arc::Label phi_label, MutableFst *fst); -// PhiCompose is a version of composition where +// RhoCompose is a version of composition where // the right hand FST (fst2) has speciall "rho transitions" // which are taken whenever no normal transition matches; these // transitions will be rewritten with whatever symbol was on diff --git a/src/latbin/lattice-compose.cc b/src/latbin/lattice-compose.cc index df70229bfd8..09a2fd500fc 100644 --- a/src/latbin/lattice-compose.cc +++ b/src/latbin/lattice-compose.cc @@ -1,6 +1,7 @@ // latbin/lattice-compose.cc // Copyright 2009-2011 Microsoft Corporation; Saarland University +// 2022 Brno University of Technology // See ../../COPYING for clarification regarding multiple authors // @@ -17,7 +18,6 @@ // See the Apache 2 License for the specific language governing permissions and // limitations under the License. - #include "base/kaldi-common.h" #include "util/common-utils.h" #include "fstext/fstext-lib.h" @@ -39,22 +39,34 @@ int main(int argc, char *argv[]) { "or lattices with FSTs (rspecifiers are assumed to be lattices, and\n" "rxfilenames are assumed to be FSTs, which have their weights interpreted\n" "as \"graph weights\" when converted into the Lattice format.\n" + "Or, rspecifier can be ark of biasing FSTs, see --compose-with-fst=true.\n" "\n" "Usage: lattice-compose [options] lattice-rspecifier1 " "(lattice-rspecifier2|fst-rxfilename2) lattice-wspecifier\n" " e.g.: lattice-compose ark:1.lats ark:2.lats ark:composed.lats\n" - " or: lattice-compose ark:1.lats G.fst ark:composed.lats\n"; + " or: lattice-compose ark:1.lats G.fst ark:composed.lats\n" + " or: lattice-compose --compose-with-fst=true ark:1.lats\n" + " ark:biasing.fsts ark:composed.lats\n"; ParseOptions po(usage); bool write_compact = true; int32 num_states_cache = 50000; int32 phi_label = fst::kNoLabel; // == -1 + int32 rho_label = fst::kNoLabel; // == -1 + std::string compose_with_fst = "auto"; + po.Register("write-compact", &write_compact, "If true, write in normal (compact) form."); po.Register("phi-label", &phi_label, "If >0, the label on backoff arcs of the LM"); + po.Register("rho-label", &rho_label, + "If >0, the label to forward fst1 paths not present biasing graph fst2. " + "(rho is input and output symbol on special arc in biasing graph)"); po.Register("num-states-cache", &num_states_cache, "Number of states we cache when mapping LM FST to lattice type. " "More -> more memory but faster."); + po.Register("compose-with-fst", &compose_with_fst, + "(true|false|auto) For auto arg2 is: rspecifier=lats, rxfilename=fst " + "(old behavior), for true/false rspecifier is fst/lattice."); po.Read(argc, argv); if (po.NumArgs() != 3) { @@ -63,6 +75,22 @@ int main(int argc, char *argv[]) { } KALDI_ASSERT(phi_label > 0 || phi_label == fst::kNoLabel); // e.g. 0 not allowed. + KALDI_ASSERT(rho_label > 0 || rho_label == fst::kNoLabel); // e.g. 0 not allowed. + if (phi_label > 0 && rho_label > 0) { + KALDI_ERR << "You cannot set both 'phi_label' and 'rho_label' at the same time."; + } + + { // convert 'compose_with_fst' to lowercase to support: true, True, TRUE + std::string tmp_lc(compose_with_fst); + std::transform(compose_with_fst.begin(), compose_with_fst.end(), + tmp_lc.begin(), ::tolower); // lc + compose_with_fst.swap(tmp_lc); + } + if (compose_with_fst != "auto" && compose_with_fst != "true" && + compose_with_fst != "false") { + KALDI_ERR << "Unkown 'compose_with_fst' value : " << compose_with_fst + << " , values are (auto|true|false)"; + } std::string lats_rspecifier1 = po.GetArg(1), arg2 = po.GetArg(2), @@ -70,7 +98,7 @@ int main(int argc, char *argv[]) { int32 n_done = 0, n_fail = 0; SequentialLatticeReader lattice_reader1(lats_rspecifier1); - + CompactLatticeWriter compact_lattice_writer; LatticeWriter lattice_writer; @@ -79,33 +107,48 @@ int main(int argc, char *argv[]) { else lattice_writer.Open(lats_wspecifier); - if (ClassifyRspecifier(arg2, NULL, NULL) == kNoRspecifier) { + bool arg2_is_rxfilename = (ClassifyRspecifier(arg2, NULL, NULL) == kNoRspecifier); + + /** + * arg2 is rxfilename that contains a single fst + * - compose arg1 lattices with single fst in arg2 + */ + if (arg2_is_rxfilename && (compose_with_fst == "auto" || compose_with_fst == "true")) { std::string fst_rxfilename = arg2; - VectorFst *fst2 = fst::ReadFstKaldi(fst_rxfilename); - // mapped_fst2 is fst2 interpreted using the LatticeWeight semiring, - // with all the cost on the first member of the pair (since we're - // assuming it's a graph weight). + VectorFst* fst2 = fst::ReadFstKaldi(fst_rxfilename); + + // Make sure fst2 is sorted on ilabel if (fst2->Properties(fst::kILabelSorted, true) == 0) { - // Make sure fst2 is sorted on ilabel. fst::ILabelCompare ilabel_comp; ArcSort(fst2, ilabel_comp); } + if (phi_label > 0) PropagateFinal(phi_label, fst2); + // mapped_fst2 is fst2 interpreted using the LatticeWeight semiring, + // with all the cost on the first member of the pair (since we're + // assuming it's a graph weight). fst::CacheOptions cache_opts(true, num_states_cache); fst::MapFstOptions mapfst_opts(cache_opts); fst::StdToLatticeMapper mapper; fst::MapFst > mapped_fst2(*fst2, mapper, mapfst_opts); + for (; !lattice_reader1.Done(); lattice_reader1.Next()) { std::string key = lattice_reader1.Key(); KALDI_VLOG(1) << "Processing lattice for key " << key; Lattice lat1 = lattice_reader1.Value(); ArcSort(&lat1, fst::OLabelCompare()); + Lattice composed_lat; - if (phi_label > 0) PhiCompose(lat1, mapped_fst2, phi_label, &composed_lat); - else Compose(lat1, mapped_fst2, &composed_lat); + if (phi_label > 0) { + PhiCompose(lat1, mapped_fst2, phi_label, &composed_lat); + } else if (rho_label > 0) { + RhoCompose(lat1, mapped_fst2, rho_label, &composed_lat); + } else { + Compose(lat1, mapped_fst2, &composed_lat); + } if (composed_lat.Start() == fst::kNoStateId) { KALDI_WARN << "Empty lattice for utterance " << key << " (incompatible LM?)"; n_fail++; @@ -121,7 +164,23 @@ int main(int argc, char *argv[]) { } } delete fst2; - } else { + } + + /** + * arg2 is rxfilename that contains a single lattice + */ + else if (arg2_is_rxfilename && compose_with_fst == "false") { + // Would it make sense to do this? Not implementing... + KALDI_ERR << "Unimplemented..."; + } + + /** + * arg2 is rspecifier that contains a table of lattices + * - composing arg1 lattices with arg2 lattices + */ + else if (not arg2_is_rxfilename && + (compose_with_fst == "auto" || compose_with_fst == "false")) { + // std::string lats_rspecifier2 = arg2; // This is the case similar to lattice-interp.cc, where we // read in another set of lattices and compose them. But in this @@ -139,6 +198,7 @@ int main(int argc, char *argv[]) { n_fail++; continue; } + Lattice lat2 = lattice_reader2.Value(key); // Make sure that either lat2 is ilabel sorted // or lat1 is olabel sorted, to ensure that @@ -150,29 +210,108 @@ int main(int argc, char *argv[]) { fst::ArcSort(&lat2, ilabel_comp); } - Lattice lat_out; + Lattice composed_lat; + // Btw, can the lat2 lattice contin phi/rho symbols ? if (phi_label > 0) { PropagateFinal(phi_label, &lat2); - PhiCompose(lat1, lat2, phi_label, &lat_out); + PhiCompose(lat1, lat2, phi_label, &composed_lat); + } else if (rho_label > 0) { + RhoCompose(lat1, lat2, rho_label, &composed_lat); + } else { + Compose(lat1, lat2, &composed_lat); + } + if (composed_lat.Start() == fst::kNoStateId) { + KALDI_WARN << "Empty lattice for utterance " << key << " (incompatible LM?)"; + n_fail++; + } else { + if (write_compact) { + CompactLattice clat; + ConvertLattice(composed_lat, &clat); + compact_lattice_writer.Write(key, clat); + } else { + lattice_writer.Write(key, composed_lat); + } + n_done++; + } + } + } + + /** + * arg2 is rspecifier that contains a table of fsts + * - composing arg1 lattices with arg2 fsts + */ + else if (not arg2_is_rxfilename && compose_with_fst == "true") { + std::string fst_rspecifier2 = arg2; + RandomAccessTableReader fst_reader2(fst_rspecifier2); + + for (; !lattice_reader1.Done(); lattice_reader1.Next()) { + std::string key = lattice_reader1.Key(); + KALDI_VLOG(1) << "Processing lattice for key " << key; + Lattice lat1 = lattice_reader1.Value(); + lattice_reader1.FreeCurrent(); + + if (!fst_reader2.HasKey(key)) { + KALDI_WARN << "Not producing output for utterance " << key + << " because not present in second table."; + n_fail++; + continue; + } + + VectorFst fst2 = fst_reader2.Value(key); + // Make sure fst2 is sorted on ilabel + if (fst2.Properties(fst::kILabelSorted, true) == 0) { + fst::ILabelCompare ilabel_comp; + fst::ArcSort(&fst2, ilabel_comp); + } + + // for composing with LM-fsts, it makes all fst2 states final + if (phi_label > 0) + PropagateFinal(phi_label, &fst2); + + // mapped_fst2 is fst2 interpreted using the LatticeWeight semiring, + // with all the cost on the first member of the pair (since we're + // assuming it's a graph weight). + fst::CacheOptions cache_opts(true, num_states_cache); + fst::MapFstOptions mapfst_opts(cache_opts); + fst::StdToLatticeMapper mapper; + fst::MapFst > + mapped_fst2(fst2, mapper, mapfst_opts); + + // sort lat1 on olabel. + ArcSort(&lat1, fst::OLabelCompare()); + + Lattice composed_lat; + if (phi_label > 0) { + PhiCompose(lat1, mapped_fst2, phi_label, &composed_lat); + } else if (rho_label > 0) { + RhoCompose(lat1, mapped_fst2, rho_label, &composed_lat); } else { - Compose(lat1, lat2, &lat_out); + Compose(lat1, mapped_fst2, &composed_lat); } - if (lat_out.Start() == fst::kNoStateId) { + + if (composed_lat.Start() == fst::kNoStateId) { KALDI_WARN << "Empty lattice for utterance " << key << " (incompatible LM?)"; n_fail++; } else { if (write_compact) { - CompactLattice clat_out; - ConvertLattice(lat_out, &clat_out); - compact_lattice_writer.Write(key, clat_out); + CompactLattice clat; + ConvertLattice(composed_lat, &clat); + compact_lattice_writer.Write(key, clat); } else { - lattice_writer.Write(key, lat_out); + lattice_writer.Write(key, composed_lat); } n_done++; } } } + /** + * none of the 'if-else-if' applied... + */ + else { + KALDI_ERR << "You should never reach here..."; + } + KALDI_LOG << "Done " << n_done << " lattices; failed for " << n_fail; return (n_done != 0 ? 0 : 1); From f185670e8ea876e27775348a8d3cb5e84cab15ea Mon Sep 17 00:00:00 2001 From: Karel Vesely Date: Thu, 27 Jan 2022 18:41:47 +0100 Subject: [PATCH 2/4] lattice-compose.cc, resolving remarks from PR #4692 --- src/latbin/lattice-compose.cc | 81 +++++++++++++++-------------------- 1 file changed, 35 insertions(+), 46 deletions(-) diff --git a/src/latbin/lattice-compose.cc b/src/latbin/lattice-compose.cc index 09a2fd500fc..74786f3b647 100644 --- a/src/latbin/lattice-compose.cc +++ b/src/latbin/lattice-compose.cc @@ -34,19 +34,15 @@ int main(int argc, char *argv[]) { using fst::StdArc; const char *usage = - "Composes lattices (in transducer form, as type Lattice). Depending\n" - "on the command-line arguments, either composes lattices with lattices,\n" - "or lattices with FSTs (rspecifiers are assumed to be lattices, and\n" - "rxfilenames are assumed to be FSTs, which have their weights interpreted\n" - "as \"graph weights\" when converted into the Lattice format.\n" - "Or, rspecifier can be ark of biasing FSTs, see --compose-with-fst=true.\n" + "Composes lattices (in transducer form, as type Lattice).\n" + "Depending on the command-line arguments, either composes\n" + "lattices with lattices, or lattices with a single FST or\n" + " multiple FSTs (whose weights are interpreted as \"graph weights\").\n" "\n" - "Usage: lattice-compose [options] lattice-rspecifier1 " - "(lattice-rspecifier2|fst-rxfilename2) lattice-wspecifier\n" - " e.g.: lattice-compose ark:1.lats ark:2.lats ark:composed.lats\n" - " or: lattice-compose ark:1.lats G.fst ark:composed.lats\n" - " or: lattice-compose --compose-with-fst=true ark:1.lats\n" - " ark:biasing.fsts ark:composed.lats\n"; + "Usage: lattice-compose [options] " + " \n" + "If the 2nd arg is an rspecifier, it is interpreted by default as a table of\n" + "lattices, or as a table of FSTs if you specify --compose-with-fst=true.\n"; ParseOptions po(usage); @@ -59,7 +55,7 @@ int main(int argc, char *argv[]) { po.Register("write-compact", &write_compact, "If true, write in normal (compact) form."); po.Register("phi-label", &phi_label, "If >0, the label on backoff arcs of the LM"); po.Register("rho-label", &rho_label, - "If >0, the label to forward fst1 paths not present biasing graph fst2. " + "If >0, the label to forward fst1 paths not present in biasing graph fst2 " "(rho is input and output symbol on special arc in biasing graph)"); po.Register("num-states-cache", &num_states_cache, "Number of states we cache when mapping LM FST to lattice type. " @@ -81,10 +77,8 @@ int main(int argc, char *argv[]) { } { // convert 'compose_with_fst' to lowercase to support: true, True, TRUE - std::string tmp_lc(compose_with_fst); - std::transform(compose_with_fst.begin(), compose_with_fst.end(), - tmp_lc.begin(), ::tolower); // lc - compose_with_fst.swap(tmp_lc); + std::string& str(compose_with_fst); + std::transform(str.begin(), str.end(), str.begin(), ::tolower); // lc } if (compose_with_fst != "auto" && compose_with_fst != "true" && compose_with_fst != "false") { @@ -109,11 +103,11 @@ int main(int argc, char *argv[]) { bool arg2_is_rxfilename = (ClassifyRspecifier(arg2, NULL, NULL) == kNoRspecifier); - /** - * arg2 is rxfilename that contains a single fst - * - compose arg1 lattices with single fst in arg2 - */ if (arg2_is_rxfilename && (compose_with_fst == "auto" || compose_with_fst == "true")) { + /** + * arg2 is rxfilename that contains a single fst + * - compose arg1 lattices with single fst in arg2 + */ std::string fst_rxfilename = arg2; VectorFst* fst2 = fst::ReadFstKaldi(fst_rxfilename); @@ -164,29 +158,27 @@ int main(int argc, char *argv[]) { } } delete fst2; - } - /** - * arg2 is rxfilename that contains a single lattice - */ - else if (arg2_is_rxfilename && compose_with_fst == "false") { - // Would it make sense to do this? Not implementing... + } else if (arg2_is_rxfilename && compose_with_fst == "false") { + /** + * arg2 is rxfilename that contains a single lattice + * - would it make sense to do this? Not implementing... + */ KALDI_ERR << "Unimplemented..."; - } - /** - * arg2 is rspecifier that contains a table of lattices - * - composing arg1 lattices with arg2 lattices - */ - else if (not arg2_is_rxfilename && + } else if (!arg2_is_rxfilename && (compose_with_fst == "auto" || compose_with_fst == "false")) { - // + /** + * arg2 is rspecifier that contains a table of lattices + * - composing arg1 lattices with arg2 lattices + */ std::string lats_rspecifier2 = arg2; // This is the case similar to lattice-interp.cc, where we // read in another set of lattices and compose them. But in this // case we don't do any projection; we assume that the user has already // done this (e.g. with lattice-project). RandomAccessLatticeReader lattice_reader2(lats_rspecifier2); + for (; !lattice_reader1.Done(); lattice_reader1.Next()) { std::string key = lattice_reader1.Key(); KALDI_VLOG(1) << "Processing lattice for key " << key; @@ -234,13 +226,12 @@ int main(int argc, char *argv[]) { n_done++; } } - } - /** - * arg2 is rspecifier that contains a table of fsts - * - composing arg1 lattices with arg2 fsts - */ - else if (not arg2_is_rxfilename && compose_with_fst == "true") { + } else if (!arg2_is_rxfilename && compose_with_fst == "true") { + /** + * arg2 is rspecifier that contains a table of fsts + * - composing arg1 lattices with arg2 fsts + */ std::string fst_rspecifier2 = arg2; RandomAccessTableReader fst_reader2(fst_rspecifier2); @@ -303,12 +294,10 @@ int main(int argc, char *argv[]) { n_done++; } } - } - - /** - * none of the 'if-else-if' applied... - */ - else { + } else { + /** + * none of the 'if-else-if' applied... + */ KALDI_ERR << "You should never reach here..."; } From 7dc37235dcc0d972bd8c7c161b5999b455531d4a Mon Sep 17 00:00:00 2001 From: Karel Vesely Date: Thu, 27 Jan 2022 19:15:29 +0100 Subject: [PATCH 3/4] fixing issue in std::transform with std::tolower, suggesting variant of overloaded function --- src/latbin/lattice-compose.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/latbin/lattice-compose.cc b/src/latbin/lattice-compose.cc index 74786f3b647..301b0f38eb2 100644 --- a/src/latbin/lattice-compose.cc +++ b/src/latbin/lattice-compose.cc @@ -78,7 +78,7 @@ int main(int argc, char *argv[]) { { // convert 'compose_with_fst' to lowercase to support: true, True, TRUE std::string& str(compose_with_fst); - std::transform(str.begin(), str.end(), str.begin(), ::tolower); // lc + std::transform(str.begin(), str.end(), str.begin(), (int(*)(int))std::tolower); // lc } if (compose_with_fst != "auto" && compose_with_fst != "true" && compose_with_fst != "false") { From cb6f272b285584f3dc14a1c96912f74560220547 Mon Sep 17 00:00:00 2001 From: Karel Vesely Date: Mon, 31 Jan 2022 12:32:33 +0100 Subject: [PATCH 4/4] lattice-compose, extending the rho explanation --- src/latbin/lattice-compose.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/latbin/lattice-compose.cc b/src/latbin/lattice-compose.cc index 301b0f38eb2..d191b076a96 100644 --- a/src/latbin/lattice-compose.cc +++ b/src/latbin/lattice-compose.cc @@ -55,8 +55,10 @@ int main(int argc, char *argv[]) { po.Register("write-compact", &write_compact, "If true, write in normal (compact) form."); po.Register("phi-label", &phi_label, "If >0, the label on backoff arcs of the LM"); po.Register("rho-label", &rho_label, - "If >0, the label to forward fst1 paths not present in biasing graph fst2 " - "(rho is input and output symbol on special arc in biasing graph)"); + "If >0, the label to forward lat1 paths not present in biasing graph fst2 " + "(rho is input and output symbol on special arc in biasing graph fst2;" + " rho is like phi (matches rest), but rho label is rewritten to the" + " specific symbol from lat1)"); po.Register("num-states-cache", &num_states_cache, "Number of states we cache when mapping LM FST to lattice type. " "More -> more memory but faster.");