OpusCleaner fixes for Japanese and Korean (#1041)

* Disable num-mismatch for Japanes (same reason as Chinese)

* Normalize Japanese punctuation

Only for en->ja

* Disable num_mismatch for Korean

Author: ZJaume

pipeline/clean/opuscleaner/configs/en-ja/default.filters.json

@@ -30,6 +30,69 @@
       },
       "language": null
     },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#([\\x{3040}-\\x{309F}\\x{30A0}-\\x{30FF}\\x{FF00}-\\x{FFEF}\\x{4E00}-\\x{9FAF}\\x{3000}-\\x{303F}\\x{3400}-\\x{4DBF}\\?])\\!#\\1\\x{ff01}#g"
+      },
+      "language": "ja"
+    },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#([\\x{3040}-\\x{309F}\\x{30A0}-\\x{30FF}\\x{FF00}-\\x{FFEF}\\x{4E00}-\\x{9FAF}\\x{3000}-\\x{303F}\\x{3400}-\\x{4DBF}\\?])\\?#\\1\\x{ff1f}#g"
+      },
+      "language": "ja"
+    },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#([\\x{3040}-\\x{309F}\\x{30A0}-\\x{30FF}\\x{FF00}-\\x{FFEF}\\x{4E00}-\\x{9FAF}\\x{3000}-\\x{303F}\\x{3400}-\\x{4DBF}\\?])\\:#\\1\\x{ff1a}#g"
+      },
+      "language": "ja"
+    },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#\\.\\.\\.\\x{3002}#...#g"
+      },
+      "language": "ja"
+    },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#\\x{30fb}\\x{30fb}\\x{30fb}#\\x{2026}#g"
+      },
+      "language": "ja"
+    },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#\\. ?\\. ?\\. ?#\\x{2026}#g"
+      },
+      "language": "ja"
+    },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#\\x{ff0c}#\\x{3001}#g"
+      },
+      "language": "ja"
+    },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#([\\x{3040}-\\x{309F}\\x{30A0}-\\x{30FF}\\x{FF00}-\\x{FFEF}\\x{4E00}-\\x{9FAF}\\x{3000}-\\x{303F}\\x{3400}-\\x{4DBF}]),#\\x{3001}#g"
+      },
+      "language": "ja"
+    },
+    {
+      "filter": "regexp",
+      "parameters": {
+        "PATTERN": "s#([\\x{3040}-\\x{309F}\\x{30A0}-\\x{30FF}\\x{FF00}-\\x{FFEF}\\x{4E00}-\\x{9FAF}\\x{3000}-\\x{303F}\\x{3400}-\\x{4DBF}])\\.\\b#\\x{3002}#g"
+      },
+      "language": "ja"
+    },
     {
       "filter": "fix_wiki",
       "parameters": {
@@ -56,14 +119,6 @@
       },
       "language": null
     },
-    {
-      "filter": "num_mismatch",
-      "parameters": {
-        "RATIO": 1,
-        "DEBUG": false
-      },
-      "language": null
-    },
     {
       "filter": "fasttext_filter",
       "parameters": {

pipeline/clean/opuscleaner/configs/en-ko/default.filters.json

@@ -56,14 +56,6 @@
       },
       "language": null
     },
-    {
-      "filter": "num_mismatch",
-      "parameters": {
-        "RATIO": 1,
-        "DEBUG": false
-      },
-      "language": null
-    },
     {
       "filter": "fasttext_filter",
       "parameters": {

pipeline/clean/opuscleaner/configs/ja-en/default.filters.json

@@ -56,14 +56,6 @@
       },
       "language": null
     },
-    {
-      "filter": "num_mismatch",
-      "parameters": {
-        "RATIO": 1,
-        "DEBUG": false
-      },
-      "language": null
-    },
     {
       "filter": "fasttext_filter",
       "parameters": {

pipeline/clean/opuscleaner/configs/ko-en/default.filters.json

@@ -56,14 +56,6 @@
       },
       "language": null
     },
-    {
-      "filter": "num_mismatch",
-      "parameters": {
-        "RATIO": 1,
-        "DEBUG": false
-      },
-      "language": null
-    },
     {
       "filter": "fasttext_filter",
       "parameters": {