Allow caller to forcibly set locale -- parse all dates as one locale.

Author: mubaldino

src/main/python/.gitignore

@@ -1,3 +1,4 @@
+MANIFEST
 *.pyc
 chardet
 pysolr.py

src/main/python/opensextant/extractors/xtemporal.py

@@ -22,7 +22,7 @@
 NO_DAY = -5
 
 log = logger_config("INFO", pkg=__name__)
-
+_default_locale = None
 
 def format_date(d):
     if isinstance(d, arrow.Arrow):
@@ -92,26 +92,37 @@ def normalize_month_num(slots: dict):
     return INVALID_DATE
 
 
-def test_european_locale(slots: dict):
+def test_european_locale(slots: dict, locale=None):
     """
 
     :param slots:
     :return:  day, month
     """
-    if "DM1" in slots and "DM2" in slots:
-        # Matched as MDY
-        # But we test if DMY is valid based on values.
-        try:
-            day = int(slots["DM1"])
-            mon = int(slots["DM2"])
+    if not ("DM1" in slots and "DM2" in slots):
+        return None, None
+
+    # Matched as MDY
+    # But we test if DMY is valid based on values.
+    try:
+        day = int(slots["DM1"])
+        mon = int(slots["DM2"])
+        # First pass -- if LOCALE == "euro", then assume pattern matches DAY/MON/YYYY
+        if locale and locale == "euro":
+            if mon <= 12 and  day <= 31:
+                return day, mon
+            else:
+                return -1, -1
+        else:
+            # Otherwise -- this is a test and we're guessing. Only return
+            # a date if date pattern appears to be unambiguous.  03/05   is Mar-5th or May-3rd, for example
             if day > 12 and mon <= 12:
                 # Valid match  31/12/...  new year's eve.
                 return day, mon
             if day > 12 and mon > 12:
                 # Invalid date match for this pattern, e.g., 13/13/, or 30/13/...
                 return -1, -1
-        except:
-            pass
+    except:
+        pass
     return None, None
 
 
@@ -217,11 +228,15 @@ def normalize_time(slots):
 
 
 class XTemporal(PatternExtractor):
-    def __init__(self, cfg="datetime_patterns_py.cfg", debug=False):
+    def __init__(self, cfg="datetime_patterns_py.cfg", debug=False, locale=None):
         """
         :param cfg: patterns config file.
         """
         PatternExtractor.__init__(self, RegexPatternManager(cfg, debug=debug, testing=debug))
+        if locale:
+            global _default_locale
+            _default_locale = locale.lower()
+
         if debug:
             log.setLevel("DEBUG")
 
@@ -237,6 +252,20 @@ class Resolution:
 
 
 class DateTimeMatch(PatternMatch):
+    """
+    DateTimeMatch puts out a matched date with attributes:
+
+        datenorm -- ISO yyyy-mm-dd date
+        epoch    -- seconds from 1970-01-01
+        resolution - D, M, h, m, s
+        locale   -- "north-am" or "euro".
+
+        If locale is set using XTemporal(locale='euro')
+        matching Euro-style dates will be forced as such through out document.
+        When locale is not set, the default is to only use euro locale for dates
+        that are not ambiguous, e.g., 30/05/1977.
+        Ambiguous dates (with no default locale used) are parsed as "north-am".
+    """
     def __init__(self, *args, **kwargs):
         PatternMatch.__init__(self, *args, **kwargs)
         self.case = PatternMatch.LOWER_CASE
@@ -273,7 +302,7 @@ def normalize(self):
         # resolution = Resolution.YEAR
         day, month = None, None
         if self.pattern_id in {"MDY-01", "MDY-02"}:
-            day, month = test_european_locale(slots) # Uses DM slots only
+            day, month = test_european_locale(slots, _default_locale) # Uses DM slots only
             if day and day < 0:
                 return False
             if day and month:

src/main/python/setup.py

@@ -14,7 +14,7 @@
 
 setup(
     name='opensextant',
-    version='1.6.7',
+    version='1.6.8',
 
     description='OpenSextant APIs and Utilities',
     long_description=long_description,

src/main/python/tests/test_xtemporal.py

@@ -1,40 +1,59 @@
 import copy
 import os
+import unittest
 
 from opensextant.extractors.xtemporal import XTemporal
 from opensextant.utility import get_csv_writer, ensure_dirs
 
 
-def run_test():
-    print("Run Default Tests")
-    datex = XTemporal(debug=True)
-    test_results = datex.default_tests()
-
-    print("Save Test Results")
-    libdir = os.path.dirname(os.path.abspath(__file__))
-    output = os.path.abspath(os.path.join(libdir, "..", "..", "results", "xtemporal-tests.csv"))
-    ensure_dirs(output)
-    print("... output file at ", output)
-
-    with open(output, "w", encoding="UTF-8") as fh:
-        header = ["TEST", "TEXT", "RESULT", "MATCH_TEXT", "MATCH_ATTRS"]
-        csvout = get_csv_writer(fh, header)
-        csvout.writeheader()
-        for result in test_results:
-
-            baserow = {
-                "TEST": result["TEST"],
-                "TEXT": result["TEXT"],
-                "RESULT": result["PASS"],
-                "MATCH_TEXT": "",
-                "MATCH_ATTRS": ""
-            }
-            for m in result["MATCHES"]:
-                row = copy.copy(baserow)
-                row["MATCH_TEXT"] = m.text
-                row["MATCH_ATTRS"] = repr(m.attrs)
-                csvout.writerow(row)
+class TestXTemporal(unittest.TestCase):
+    def test_xtemp_euro(self):
+        datex = XTemporal(debug=True, locale="Euro")
+
+        euro_tests = [
+            ("text text 04/05/2025", "2025-05-04"),
+            ("text text 30/05/2025", "2025-05-30"),
+            ("text text 12/05/2025", "2025-05-30"),
+            ("text text 12/12/2025", "2025-12-12"),
+            ("text text 05/12/2025", "2025-12-05")
+        ]
+        for tst, expected in euro_tests:
+            for date_match in datex.extract(tst):
+                result = date_match.attrs.get("datenorm")
+                print(date_match.text, result, "Expected", expected)
+                self.assertEqual(expected, result)
+
+    def run_test(self):
+        print("Run Default Tests")
+
+        datex = XTemporal(debug=True)
+        test_results = datex.default_tests()
+
+        print("Save Test Results")
+        libdir = os.path.dirname(os.path.abspath(__file__))
+        output = os.path.abspath(os.path.join(libdir, "..", "..", "results", "xtemporal-tests.csv"))
+        ensure_dirs(output)
+        print("... output file at ", output)
+
+        with open(output, "w", encoding="UTF-8") as fh:
+            header = ["TEST", "TEXT", "RESULT", "MATCH_TEXT", "MATCH_ATTRS"]
+            csvout = get_csv_writer(fh, header)
+            csvout.writeheader()
+            for result in test_results:
+
+                baserow = {
+                    "TEST": result["TEST"],
+                    "TEXT": result["TEXT"],
+                    "RESULT": result["PASS"],
+                    "MATCH_TEXT": "",
+                    "MATCH_ATTRS": ""
+                }
+                for m in result["MATCHES"]:
+                    row = copy.copy(baserow)
+                    row["MATCH_TEXT"] = m.text
+                    row["MATCH_ATTRS"] = repr(m.attrs)
+                    csvout.writerow(row)
 
 
 if __name__ == "__main__":
-    run_test()
+    unittest.main()