Merge pull request #137 from vmenger/tab-spaces-as-token

Recognize multiple spaces as token

Author: vmenger

.gitignore

@@ -124,4 +124,6 @@ ENV/
 # mypy
 .mypy_cache/
 
-.idea
+# ide
+.idea
+.vscode

CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 3.0.2 (2023-02-15)
+
+### Changed
+- recognize 4+ spaces as a token, blocking annotations
+
 ## 3.0.1 (2023-12-20)
 
 ### Fixed

deduce/tokenizer.py

@@ -3,7 +3,7 @@
 import docdeid as dd
 import regex
 
-_TOKENIZER_PATTERN = regex.compile(r"\w+|[\n\r\t]|.(?<! )", flags=regex.I | regex.M)
+_TOKENIZER_PATTERN = regex.compile(r"\w+|[\n\r\t]| {4,}|[^ ]", flags=regex.I | regex.M)
 
 
 class DeduceTokenizer(dd.tokenizer.Tokenizer):  # pylint: disable=R0903

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deduce"
-version = "3.0.1"
+version = "3.0.2"
 description = "Deduce: de-identification method for Dutch medical text"
 authors = ["Vincent Menger <vmenger@protonmail.com>"]
 maintainers = ["Vincent Menger <vmenger@protonmail.com>"]

tests/unit/test_tokenizer.py

@@ -47,6 +47,22 @@ def test_split_nonalpha(self):
 
         assert tokenizer._split_text(text=text) == expected_tokens
 
+    def test_split_multiple_spaces(self):
+        tokenizer = DeduceTokenizer()
+        text = "Pieter van der Zee     Bergen Op  Zoom"
+        expected_tokens = [
+            dd.Token(text="Pieter", start_char=0, end_char=6),
+            dd.Token(text="van", start_char=7, end_char=10),
+            dd.Token(text="der", start_char=11, end_char=14),
+            dd.Token(text="Zee", start_char=15, end_char=18),
+            dd.Token(text="     ", start_char=18, end_char=23),
+            dd.Token(text="Bergen", start_char=23, end_char=29),
+            dd.Token(text="Op", start_char=30, end_char=32),
+            dd.Token(text="Zoom", start_char=34, end_char=38),
+        ]
+
+        assert tokenizer._split_text(text=text) == expected_tokens
+
     def test_split_newline(self):
         tokenizer = DeduceTokenizer()
         text = "regel 1 \n gevolgd door regel 2"