Merge pull request #18 from bricksdont/fix_dgs_types

Fix dgs types

Author: AmitMY

examples/load.ipynb

@@ -300,6 +300,36 @@
    "execution_count": null,
    "outputs": []
   },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# DGS Types"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, include_pose=None, process_video=False)\n",
+    "dgs_types = tfds.load('dgs_types', builder_kwargs=dict(config=config))\n",
+    "\n",
+    "for datum in itertools.islice(dgs_types[\"train\"], 0, 10):\n",
+    "  print(datum)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
   {
    "cell_type": "markdown",
    "source": [

setup.py

@@ -11,7 +11,7 @@
 setup(
     name="sign-language-datasets",
     packages=packages,
-    version="0.1.0",
+    version="0.1.1",
     description="TFDS Datasets for sign language",
     author="Amit Moryossef",
     author_email="amitmoryossef@gmail.com",

sign_language_datasets/datasets/dgs_types/dgs_types.py

@@ -3,6 +3,7 @@
 import re
 from collections import defaultdict
 
+import tensorflow as tf
 import tensorflow_datasets as tfds
 
 from os import path
@@ -79,6 +80,7 @@ def _info(self) -> tfds.core.DatasetInfo:
         features = {
             "id": tfds.features.Text(),
             "glosses": tfds.features.Sequence(tfds.features.Text()),
+            "frequencies": tfds.features.Sequence(tf.int32),
             "hamnosys": tfds.features.Text(),
             "views": tfds.features.Sequence(video_feature)
         }
@@ -112,6 +114,7 @@ def get_galex_data(self, dl_manager: tfds.download.DownloadManager):
                 datum = {
                     "id": "galex_" + gloss,
                     "glosses": [gloss],
+                    "frequencies": [],
                     "hamnosys": re.findall(r'a class=\"ham\".*?>(.*?)<', content)[0],
                     "views": [{
                         "name": "front",
@@ -131,11 +134,16 @@ def get_galex_data(self, dl_manager: tfds.download.DownloadManager):
     def get_dgs_data(self, dl_manager: tfds.download.DownloadManager):
         MEINE_DGS = "https://www.sign-lang.uni-hamburg.de/meinedgs/"
         dgs_index = dl_manager.download(MEINE_DGS + "ling/types_de.html")
+
         gloss_map = defaultdict(list)
+        gloss_frequencies = defaultdict(list)
+
         with open(dgs_index, "r", encoding="utf-8") as f:
-            for match in re.finditer(r'<p>(.*?) \(\d* Tokens\)( → )?(.*?)</p>', f.read()):
+            for match in re.finditer(r'<p>(.*?) \((\d+) Tokens?\)( → )?(.*?)</p>', f.read()):
                 gloss_id = re.findall(r'\.\.\/types\/(.*?)\.html', match.group(0))[0]
-                gloss_text = match.group(1) if match.group(3) != "" else re.findall(r'>(.*?)<', match.group(1))[0]
+                gloss_frequency = int(match.group(2))
+                gloss_frequencies[gloss_id].append(gloss_frequency)
+                gloss_text = match.group(1) if match.group(3) is not None else re.findall(r'>(.*?)<', match.group(1))[0]
                 gloss_map[gloss_id].append(gloss_text)
 
         gloss_ids = list(gloss_map.keys())
@@ -162,11 +170,14 @@ def get_dgs_data(self, dl_manager: tfds.download.DownloadManager):
                     })
                     video_urls[view_video_url] = view_video_url
 
+            frequencies = gloss_frequencies[gloss_id]
+
             hamnosys_search = re.findall(r'class=\"hamnosys\".*?>(.*?)<', content)
             hamnosys = hamnosys_search[0] if len(hamnosys_search) > 0 else ""
 
             data.append({
                 "id": gloss_id,
+                "frequencies": frequencies,
                 "glosses": glosses,
                 "hamnosys": hamnosys,
                 "views": views