redact identity

Author: jararap

eval_tokenizer_example.ipynb

@@ -514,7 +514,7 @@
     "tokens, _, __ = read_cpp_res('wiki')\n",
     "gb = greedy_builder.build(tokens)\n",
     "\n",
-    "orig = process_wiki_xml(open(\"/data/jiapeng/wiki/cleaned/AA/wiki_00\"))\n",
+    "orig = process_wiki_xml(open(\"/data/wiki/cleaned/AA/wiki_00\"))\n",
     "print(\"Number of texts:\", len(orig))\n",
     "pat_str=r\"\"\"'s|'t|'re|'ve|'m|'ll|'d| ?[\\p{L}]+| ?[\\p{N}]+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+\"\"\"\n",
     "tokenized = gb.batch_tokenize([[re.sub(' ','Ġ',x) for x in regex.findall(pat_str, doc)] for doc in orig])\n",