Track changes in API, etc. (#29)

* Track changes in API, etc.
* Add script to check version
* Need dev mathics scanner
* Use PatternLite instead of defunct pattern, but we get a loss in quality

Author: rocky

.github/workflows/consistency-checks.yml

@@ -23,9 +23,9 @@ jobs:
         # python -m pip install --upgrade pip
         python -m pip install pytest
         # Can comment out when next Mathics3 core and Mathics-scanner are released
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        (cd src/mathics3 && bash ./admin-tools/make-op-tables.sh)
+        (cd src/mathics3 && bash ./admin-tools/make-JSON-tables.sh)
         # python -m pip install Mathics3[full]
     - name: Install Pymathics.natlang
       run: |

.github/workflows/ubuntu.yml

@@ -24,11 +24,11 @@ jobs:
         # python -m pip install --upgrade pip
         python -m pip install pytest
         # Can comment out when next Mathics3 core and Mathics-scanner are released
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
         git clone https://github.com/Mathics3/mathics-core
-        # (cd mathics-core && pip3 install -e .[full])
-        # (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
+        (cd mathics-core && pip3 install -e .[full])
+        (cd mathics-core && bash ./admin-tools/make-JSON-tables.sh)
         python -m pip install Mathics3[full]
     - name: install pymathics natlang
       run: |

admin-tools/check-versions.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+function finish {
+  cd $mathics_natlang_owd
+}
+
+# FIXME put some of the below in a common routine
+mathics_natlang_owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-versions ; then
+    exit $?
+fi
+
+cd ..
+for version in $PYVERSIONS; do
+    echo --- $version ---
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && pip install -e .
+    if ! make check; then
+	exit $?
+    fi
+    echo === $version ===
+done
+finish

pymathics/natlang/nltk.py

@@ -7,6 +7,8 @@
 from itertools import chain
 
 import nltk
+from pattern.en import lexeme, pluralize
+
 from mathics.builtin.codetables import iso639_3
 from mathics.core.atoms import String
 from mathics.core.builtin import Builtin, MessageException
@@ -304,12 +306,9 @@ def inflected_forms(self, syn, desc):
         try:
             word, pos, _ = desc
             if pos == "Verb":
-                from pattern.en import lexeme
 
                 return [w for w in reversed(lexeme(word)) if w != word]
             elif pos == "Noun":
-                from pattern.en import pluralize
-
                 return [pluralize(word)]
             elif pos == "Adjective":
                 from pattern.en import comparative, superlative

pymathics/natlang/normalization.py

@@ -101,7 +101,7 @@ class TextCases(_SpacyBuiltin):
     ## >> TextCases[Import["ExampleData/EinsteinSzilLetter.txt"], "Person", 3][[2;;3]]
     ##  = {L. Szilard, Joliot}
 
-    >> TextCases["Anne, Peter and Mr Johnes say hello.", "Person", 3][[2;;3]]
+    >> TextCases["Saul, Peter and Mr Johnes say hello.", "Person", 3][[2;;3]]
      = {Peter, Johnes}
 
     """

pymathics/natlang/textual_analysis.py

@@ -217,13 +217,13 @@ class WordSimilarity(_SpacyBuiltin):
     </dl>
 
     >> NumberForm[WordSimilarity["car", "train"], 3]
-     = 0.439
+     = 0.169
 
     >> NumberForm[WordSimilarity["car", "hedgehog"], 3]
-     = 0.195
+     = 0.0173
 
     >> NumberForm[WordSimilarity[{"An ocean full of water.", {2, 2}}, { "A desert full of sand.", {2, 5}}], 3]
-     = {0.505, 0.481}
+     = {0.127, 0.256}
     """
 
     messages = merge_dictionaries(

setup.py

@@ -59,7 +59,7 @@ def read(*rnames):
         "langid",  # replace with a supported newer package, e.g. via spacy
         "llvmlite>=0.36",
         "nltk>=3.8.0",
-        "pattern>=3.6.0",
+        "PatternLite",
         "pyenchant>=3.2.0",
         "pycountry>=3.2.0",
         "spacy>=3.4",
@@ -76,11 +76,11 @@ def read(*rnames):
         "Intended Audience :: Science/Research",
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Topic :: Scientific/Engineering",

test/consistency-and-style/test_summary_text.py

@@ -6,6 +6,9 @@
 
 import pytest
 
+from types import ModuleType
+from typing import Dict
+
 from mathics.core.load_builtin import name_is_builtin_symbol
 from mathics.core.builtin import Builtin
 from mathics.doc.gather import skip_doc
@@ -60,7 +63,7 @@
 language_tool = None
 if CHECK_GRAMMAR:
     try:
-        import language_tool_python
+        import language_tool_python  # type: ignore[import-not-found]
 
         language_tool = language_tool_python.LanguageToolPublicAPI("en-US")
         # , config={ 'cacheSize': 1000, 'pipelineCaching': True })
@@ -95,14 +98,15 @@ def import_module(module_name: str):
         module_names.append(f"{subdir}.{modname}")
 
 
-modules = dict()
+modules: Dict[str, ModuleType] = dict()
 for module_name in module_names:
     import_module(module_name)
 
 # modules = {"compilation": modules["compilation"],}
 
 
 def check_grammar(text: str):
+    assert language_tool is not None
     matches = language_tool.check(text)
     filtered_matches = []
     if matches:
@@ -138,10 +142,10 @@ def check_well_formatted_docstring(docstr: str, instance: Builtin, module_name:
     ), f"missing <dd> field {instance.get_name()} from {module_name}"
     assert (
         docstr.count("</dt>") == 0
-    ), f"unnecesary </dt> {instance.get_name()} from {module_name}"
+    ), f"unnecessary </dt> {instance.get_name()} from {module_name}"
     assert (
         docstr.count("</dd>") == 0
-    ), f"unnecesary </dd> field {instance.get_name()} from {module_name}"
+    ), f"unnecessary </dd> field {instance.get_name()} from {module_name}"
 
     assert (
         docstr.count("<url>") > 0