Add bucket to explicit conversion script.

Author: sebpuetz

.github/workflows/python.yml

@@ -42,6 +42,4 @@ jobs:
         pytest --doctest-modules
     - name: Conversion tests
       if: ${{ matrix.python-version == '3.7' }}
-      run: |
-        bash ./tests/integration/conversion.sh
-
+      run: bash ./tests/integration/all.sh

README.md

@@ -122,6 +122,13 @@ between the supported formats.
 $ ffp-convert -f fasttext fasttext.bin -t finalfusion embeddings.fifu
 ~~~
 
+`ffp-bucket-to-explicit` can be used to convert bucket embeddings to embeddings
+with an explicit ngram lookup.
+~~~shell
+# convert finalfusion bucket embeddings to explicit
+$ ffp-bucket-to-explicit -f finalfusion embeddings.fifu explicit.fifu
+~~~ 
+
 ## Where to go from here
 
   * [finalfrontier](https://finalfusion.github.io/finalfrontier)

setup.py

@@ -103,6 +103,7 @@ def run(self):
       },
       entry_points=dict(console_scripts=[
           'ffp-convert=finalfusion.scripts.convert:main',
+          'ffp-bucket-to-explicit=finalfusion.scripts.bucket_to_explicit:main',
       ]),
       version="0.7.0-pre"
       )

src/finalfusion/scripts/bucket_to_explicit.py

@@ -0,0 +1,36 @@
+"""
+Conversion from bucket embeddings to explicit.
+"""
+import argparse
+
+from finalfusion.scripts.util import Format
+
+
+def main() -> None:  # pylint: disable=missing-function-docstring
+    parser = argparse.ArgumentParser(
+        prog="ffp-bucket-to-explicit",
+        description="Convert bucket embeddings to explicit lookups.")
+    parser.add_argument("input",
+                        help="Input bucket embeddings",
+                        type=str,
+                        metavar="INPUT")
+    parser.add_argument("output",
+                        help="Output path",
+                        type=str,
+                        metavar="OUTPUT")
+    parser.add_argument(
+        "-f",
+        "--from",
+        type=str,
+        choices=['finalfusion', 'fasttext'],
+        default="finalfusion",
+        help=
+        "Valid choices: ['finalfusion', 'fasttext'] Default: 'finalfusion'",
+        metavar="INPUT_FORMAT")
+    args = parser.parse_args()
+    embeds = Format(getattr(args, 'from')).load(args.input)
+    embeds.bucket_to_explicit().write(args.output)
+
+
+if __name__ == '__main__':
+    main()

tests/integration/all.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+TESTDIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+echo conversions >&2
+"${TESTDIR}"/conversion.sh
+
+echo bucket-to-explicit >&2
+"${TESTDIR}"/bucket_to_explicit.sh

tests/integration/bucket_to_explicit.py

@@ -0,0 +1,39 @@
+import numpy as np
+from finalfusion.scripts.util import Format
+from finalfusion.subword import ngrams
+from finalfusion.vocab.subword import FastTextVocab, ExplicitVocab, FinalfusionBucketVocab
+
+
+def test(inp, input_format, output):
+    e1 = Format(input_format).load(inp)
+    e2 = Format("finalfusion").load(output)
+
+    v1 = e1.vocab
+    v2 = e2.vocab
+    assert isinstance(v1, (FinalfusionBucketVocab, FastTextVocab))
+    assert isinstance(v2, ExplicitVocab)
+    assert v1.words == v2.words
+    assert v1.word_index == v2.word_index
+    assert v1.subword_indexer.min_n == v2.subword_indexer.min_n, \
+        f"{v1.subword_indexer.min_n} == {v2.subword_indexer.min_n}"
+    assert v1.subword_indexer.max_n == v2.subword_indexer.max_n, \
+        f"{v1.subword_indexer.max_n} == {v2.subword_indexer.max_n}"
+    v1_ngrams = set([ngram for word in v1.words for ngram in ngrams(word)])
+    v1_unique_indices = set((v1.subword_indexer(ngram) for ngram in v1_ngrams))
+    assert v1_ngrams == set(v2.subword_indexer.ngrams)
+    assert len(v1_unique_indices) == v2.subword_indexer.upper_bound, \
+        f"{len(v1_unique_indices)} == {v2.subword_indexer.upper_bound}"
+    assert len(v1_unique_indices) + len(v1) == v2.upper_bound, \
+        f"{len(v1_unique_indices)} + {len(v1)} == {v2.upper_bound}"
+    assert e2.storage.shape[0] == v2.upper_bound, \
+        f"{e2.storage.shape[0]} == {v2.upper_bound}"
+    assert np.allclose(e1.storage[:len(v1)], e2.storage[:len(v2)])
+    for ngram in v1_ngrams:
+        e1_ngram_embed = e1.storage[v1.subword_indexer(ngram) + len(v1)]
+        e2_ngram_embed = e2.storage[v2.subword_indexer(ngram) + len(v1)]
+        assert np.allclose(e1_ngram_embed, e2_ngram_embed)
+
+
+if __name__ == '__main__':
+    import sys
+    test(*sys.argv[1:])

tests/integration/bucket_to_explicit.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -eu
+
+tmp_dir=$(mktemp -d /tmp/bucket_to_explicit.XXXXXX)
+
+function finish() {
+  rm -rf "$tmp_dir"
+}
+
+trap finish EXIT
+
+TESTDIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+function convert_and_verify() {
+  echo ffp-bucket-to-explicit "${1}" -f "${2}" "${3}" >&2
+  ffp-bucket-to-explicit "${1}" -f "${2}" "${3}"
+  python "${TESTDIR}"/bucket_to_explicit.py "${1}" "${2}" "${3}"
+}
+
+convert_and_verify "${TESTDIR}/../data/ff_buckets.fifu" finalfusion fifu_bucket_to_expl.fifu
+
+convert_and_verify "${TESTDIR}/../data/fasttext.bin" fasttext fasttext_to_expl.fifu