Add conversion script.

Author: sebpuetz

.github/workflows/python.yml

@@ -40,3 +40,7 @@ jobs:
       run: |
         pip install pytest
         pytest --doctest-modules
+    - name: Conversion tests
+      if: ${{ matrix.python-version == '3.7' }}
+      run: bash ./tests/conversion_integration.sh
+

setup.py

@@ -88,4 +88,6 @@ def run(self):
       include_package_data=True,
       package_dir={'': 'src'},
       package_data={'finalfusion': ['py.typed', '*.pyi', '*.h', '*.c']},
-      url="https://github.com/finalfusion/ffp")
+      url="https://github.com/finalfusion/ffp",
+      entry_points=dict(console_scripts=['ffp-convert=finalfusion.scripts.convert:main']),
+      )

src/finalfusion/embeddings.py

@@ -363,15 +363,15 @@ def chunks(self) -> List[Chunk]:
             chunks.append(self.norms)
         return chunks
 
-    def write(self, file: str):
+    def write(self, file: Union[str, bytes, int, PathLike]):
         """
         Write the Embeddings to the given file.
 
         Writes the Embeddings to a finalfusion file at the given file.
 
         Parameters
         ----------
-        file : str
+        file : str, bytes, int, PathLike
             Path of the output file.
         """
         with open(file, 'wb') as outf:

src/finalfusion/scripts/__init__.py

@@ -0,0 +1,52 @@
+"""
+Conversion for Word Embeddings.
+
+Offers conversion from and to any combination of:
+    * finalfusion
+    * fastText
+    * word2vec
+    * textdims
+    * text
+
+Conversion of finalfusion files with FinalfusionBucketVocab or ExplicitVocab to fastText
+fails.
+"""
+import argparse
+
+from finalfusion.scripts.util import Format
+
+
+def main():  # pylint: disable=missing-function-docstring
+    formats = ["word2vec", "finalfusion", "fasttext", "text", "textdims"]
+    parser = argparse.ArgumentParser(prog="ffp-convert",
+                                     description="Convert embeddings.")
+    parser.add_argument("input",
+                        type=str,
+                        help="Input embeddings",
+                        metavar="INPUT")
+    parser.add_argument("output",
+                        type=str,
+                        help="Output path",
+                        metavar="OUTPUT")
+    parser.add_argument("-f",
+                        "--from",
+                        type=str,
+                        choices=formats,
+                        default="word2vec",
+                        help=f"Valid choices: {formats} Default: 'word2vec'",
+                        metavar="INPUT_FORMAT")
+    parser.add_argument(
+        "-t",
+        "--to",
+        type=str,
+        choices=formats,
+        default="finalfusion",
+        help=f"Valid choices: {formats} Default: 'finalfusion'",
+        metavar="OUTPUT_FORMAT")
+    args = parser.parse_args()
+    embeds = Format(getattr(args, 'from')).load(args.input)
+    Format(args.to).write(args.output, embeds)
+
+
+if __name__ == '__main__':
+    main()

src/finalfusion/scripts/convert.py

@@ -0,0 +1,61 @@
+# pylint: disable=missing-docstring
+from enum import Enum
+from functools import partial
+from os import PathLike
+from typing import Union, Callable
+
+from finalfusion import Embeddings, load_finalfusion
+from finalfusion.compat import write_word2vec, write_text, write_text_dims, write_fasttext, \
+    load_word2vec, load_text, load_text_dims, load_fasttext
+
+
+class Format(Enum):
+    """
+    Supported embedding formats.
+    """
+    finalfusion = "finalfusion"
+    fasttext = "fasttext"
+    word2vec = "word2vec"
+    textdims = "textdims"
+    text = "text"
+
+    @property
+    def write(
+            self
+    ) -> Callable[[Union[str, bytes, int, PathLike], Embeddings], None]:
+        """
+        Helper to get the write method for different Formats
+        """
+        if self == Format.finalfusion:
+
+            def write_fifu(path: Union[str, bytes, int, PathLike],
+                           embeddings: Embeddings):
+                embeddings.write(path)
+
+            return write_fifu
+        if self == Format.word2vec:
+            return write_word2vec
+        if self == Format.text:
+            return write_text
+        if self == Format.textdims:
+            return write_text_dims
+        if self == Format.fasttext:
+            return write_fasttext
+        raise ValueError(f"Unknown format {str(self)}")
+
+    @property
+    def load(self) -> Callable[[Union[str, bytes, int, PathLike]], Embeddings]:
+        """
+        Helper to get the load method for different Formats
+        """
+        if self == Format.finalfusion:
+            return partial(load_finalfusion, mmap=True)
+        if self == Format.word2vec:
+            return load_word2vec
+        if self == Format.text:
+            return load_text
+        if self == Format.textdims:
+            return load_text_dims
+        if self == Format.fasttext:
+            return load_fasttext
+        raise ValueError(f"Unknown format {str(self)}")

src/finalfusion/scripts/util.py

@@ -0,0 +1,54 @@
+import sys
+
+import numpy as np
+
+from finalfusion.scripts.util import Format
+from finalfusion.vocab.subword import FastTextVocab, SubwordVocab
+
+
+def test(inp, input_format, output, output_format):
+    e1 = Format(input_format).load(inp)
+    e2 = Format(output_format).load(output)
+    if isinstance(e1.vocab, FastTextVocab) and \
+            output_format in ["finalfusion", "fasttext"]:
+        exit(cmp_subword_embeds(e1, e2))
+
+    if isinstance(e1.vocab, SubwordVocab) and \
+            input_format in ["fasttext", "finalfusion"] and \
+            output_format in ["word2vec", "text", "textdims"]:
+        exit(cmp_subword_embeds_to_simple(e1, e2))
+
+    if input_format in ["finalfusion", "word2vec", "text", "textdims"]:
+        exit(cmp_simple_embeds(e1, e2))
+    print(
+        f"missing testcase for {input_format} to {output_format} ({inp} to {output})",
+        file=sys.stderr)
+    exit(1)
+
+
+def cmp_simple_embeds(e1, e2):
+    assert e1.vocab == e2.vocab
+    assert np.allclose(e1.storage, e2.storage, atol=1e-5)
+    assert np.allclose(e1.norms, e2.norms, atol=1e-5)
+    return 0
+
+
+def cmp_subword_embeds_to_simple(e1, e2):
+    assert e1.vocab.words == e2.vocab.words
+    assert e1.vocab.word_index == e2.vocab.word_index
+    assert np.allclose(e1.storage[:len(e1.vocab)], e2.storage, atol=1e-5)
+    if e1.norms is not None:
+        assert np.allclose(e1.norms, e2.norms, atol=1e-5)
+    return 0
+
+
+def cmp_subword_embeds(e1, e2):
+    assert e1.vocab == e2.vocab
+    assert np.allclose(e1.storage, e2.storage, atol=1e-5)
+    if e1.norms is not None:
+        assert np.allclose(e1.norms, e2.norms, atol=1e-5)
+    return 0
+
+
+if __name__ == '__main__':
+    test(*sys.argv[1:])

tests/conversion_integration.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -eu
+
+tmp_dir=$(mktemp -d /tmp/run_conversion.XXXXXX)
+
+function finish() {
+  rm -rf "$tmp_dir"
+}
+
+trap finish EXIT
+
+TESTDIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+function convert_and_verify() {
+  echo ffp-convert "${1}" -f "${2}" "${3}" -t "${4}" >&2
+  ffp-convert "${1}" -f "${2}" "${3}" -t "${4}"
+  python "${TESTDIR}"/conversion_integration.py "${1}" "${2}" "${3}" "${4}"
+}
+
+function verify_all_conversions() {
+  local input=${1}
+  local in_format=${2}
+  local out_path_prefix=${tmp_dir}/${in_format}
+  convert_and_verify "${input}" "${in_format}" "${out_path_prefix}_to_fifu.fifu" finalfusion
+  convert_and_verify "${input}" "${in_format}" "${out_path_prefix}_to_w2v.w2v" word2vec
+  convert_and_verify "${input}" "${in_format}" "${out_path_prefix}_to_ft.bin" fasttext
+  convert_and_verify "${input}" "${in_format}" "${out_path_prefix}_to_text.txt" text
+  convert_and_verify "${input}" "${in_format}" "${out_path_prefix}_to_text.dims.txt" textdims
+}
+
+# txt dims
+input="${TESTDIR}/data/embeddings.dims.txt"
+verify_all_conversions "${input}" textdims
+
+# txt
+input="${TESTDIR}/data/embeddings.txt"
+verify_all_conversions  "${input}" text
+
+# w2v
+input="${TESTDIR}/data/embeddings.w2v"
+verify_all_conversions  "${input}" word2vec
+
+# fifu
+input="${TESTDIR}/data/embeddings.fifu"
+verify_all_conversions  "${input}" finalfusion
+
+# fasttext doesn't support fifu bucket indexers
+# so we're making explicit calls for the other formats
+input="${TESTDIR}/data/ff_buckets.fifu"
+convert_and_verify "${input}" finalfusion \
+  "${tmp_dir}/fifu_buckets_to_fifu.fifu" finalfusion
+convert_and_verify "${input}" finalfusion \
+  "${tmp_dir}/fifu_buckets_to_w2v.w2v" word2vec
+convert_and_verify "${input}" finalfusion \
+  "${tmp_dir}/fifu_buckets_to_text.txt" text
+convert_and_verify "${input}" finalfusion \
+  "${tmp_dir}/fifu_buckets_to_text.dims.txt" textdims
+
+input="${TESTDIR}/data/fasttext.bin"
+verify_all_conversions  "${input}" fasttext