Revise doc data for TeX

We now keep both a version of the internal docs for LaTeX formatting
in both package space and user space.

User space is writable and can be updated with additional modules as
might be found in PyMathics modules one day

The system space is a fallback and is supplied by the distribution.
It has document data at the time the distribution was packaged.

Note that data destined for HTML needs to be separate.

The main difference is that the output results format are different:
LaTeX/asymptote versus HTML/MathJax/ThreeJS.

Simple, huh?

Author: rocky

MANIFEST.in

@@ -7,4 +7,5 @@ include Makefile
 include requirements-dev.txt
 include requirements-full.txt
 recursive-include mathics *.py
+recursive-include mathics/data *
 recursive-include test *.py *.m

admin-tools/make-dist.sh

@@ -14,25 +14,19 @@ if ! source ./pyenv-versions ; then
     exit $?
 fi
 
-
 cd ..
 source mathics/version.py
+cp -v ${HOME}/.local/var/mathics/doc_tex_data.pcl mathics/data/
+
 echo $__version__
 
 for pyversion in $PYVERSIONS; do
     if ! pyenv local $pyversion ; then
 	exit $?
     fi
-    # pip bdist_egg create too-general wheels. So
-    # we narrow that by moving the generated wheel.
-
-    # Pick out first two number of version, e.g. 3.7.9 -> 37
-    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
     rm -fr build
     python setup.py bdist_egg
     python setup.py bdist_wheel
-    python setup.py bdist_wheel --universal
-    mv -v dist/${PACKAGE}-$VERSION-{py2.py3,py$first_two}-none-any.whl
 done
 
 python ./setup.py sdist

mathics/doc/common_doc.py

@@ -27,6 +27,7 @@
 """
 
 import importlib
+import os.path as osp
 import pkgutil
 import re
 
@@ -719,7 +720,7 @@ def __init__(self):
         self.parts = []
         self.parts_by_slug = {}
         self.pymathics_doc_loaded = False
-        self.doc_data_file = settings.DOC_DATA_PATH
+        self.doc_data_file = settings.get_doc_tex_data_path(should_be_readable=True)
         self.title = "Overview"
         files = listdir(self.doc_dir)
         files.sort()
@@ -730,7 +731,7 @@ def __init__(self):
             if part_title.endswith(".mdoc"):
                 part_title = part_title[: -len(".mdoc")]
                 part = DocPart(self, part_title)
-                text = open(self.doc_dir + file, "rb").read().decode("utf8")
+                text = open(osp.join(self.doc_dir, file), "rb").read().decode("utf8")
                 text = filter_comments(text)
                 chapters = CHAPTER_RE.findall(text)
                 for title, text in chapters:

mathics/doc/tex/Makefile

@@ -6,16 +6,17 @@ LATEXMK ?= latexmk
 BASH ?= /bin/bash
 #-quiet
 
-DOC_DATA_PCL ?= $(HOME)/.local/var/mathics/doc_data.pcl
+DOC_TEX_DATA_PCL ?= $(HOME)/.local/var/mathics/doc_tex_data.pcl
 
 #: Default target: Make everything
 all doc texdoc: mathics.pdf
 
-doc-data $(DOC_DATA_PCL):
+#: Create internal Document Data from .mdoc and Python builtin module docstrings
+doc-data $(DOC_TEX_DATA_PCL):
 	(cd ../.. && $(PYTHON) docpipeline.py --output --keep-going)
 
 #: Build mathics PDF
-mathics.pdf: mathics.tex documentation.tex logo-text-nodrop.pdf logo-heptatom.pdf $(DOC_DATA_PCL)
+mathics.pdf: mathics.tex documentation.tex logo-text-nodrop.pdf logo-heptatom.pdf $(DOC_TEX_DATA_PCL)
 	$(LATEXMK) --verbose -f -pdf -pdflatex="$(XETEX) -halt-on-error" mathics
 
 #: Build test PDF
@@ -28,7 +29,7 @@ logo-heptatom.pdf logo-text-nodrop.pdf:
 	(cd .. && $(BASH) ./images.sh)
 
 #: The build of the documentation which is derived from docstrings in the Python code
-documentation.tex: $(DOC_DATA_PCL)
+documentation.tex: $(DOC_TEX_DATA_PCL)
 	$(PYTHON) ./doc2latex.py
 
 #: Same as mathics.pdf
@@ -40,6 +41,6 @@ clean:
 	rm -f test-mathics.aux test-mathics.idx test-mathics.log test-mathics.mtc test-mathics.mtc* test-mathics.out test-mathics.toc || true
 	rm -f mathics.fdb_latexmk mathics.ilg mathics.ind mathics.maf mathics.pre || true
 	rm -f mathics_*.* || true
-	rm -f mathics-*.* documentation.tex doc_data.pcl || true
-	rm -f mathics.pdf mathics.dvi data_tex_data.pcl test-mathics.pdf test-mathics.dvi || true
+	rm -f mathics-*.* documentation.tex $(DOC_TEX_DATA_PCL) || true
+	rm -f mathics.pdf mathics.dvi test-mathics.pdf test-mathics.dvi || true
 	rm -f mathics-test.pdf mathics-test.dvi || true

mathics/doc/tex/doc2latex.py

@@ -28,7 +28,7 @@ def extract_doc_from_source(quiet=False):
     if not quiet:
         print(f"Extracting internal doc data for {version_string}")
     try:
-        return load_doc_data(settings.DOC_DATA_PATH)
+        return load_doc_data(settings.get_doc_tex_data_path(should_be_readable=True))
     except KeyboardInterrupt:
         print("\nAborted.\n")
         return

mathics/docpipeline.py

@@ -10,6 +10,7 @@
 """
 
 import os
+import os.path as osp
 import pickle
 import re
 import sys
@@ -302,8 +303,8 @@ def open_ensure_dir(f, *args, **kwargs):
     try:
         return open(f, *args, **kwargs)
     except (IOError, OSError):
-        d = os.path.dirname(f)
-        if d and not os.path.exists(d):
+        d = osp.dirname(f)
+        if d and not osp.exists(d):
             os.makedirs(d)
         return open(f, *args, **kwargs)
 
@@ -323,7 +324,11 @@ def test_all(
 
     if generate_output:
         if texdatafolder is None:
-            texdatafolder = settings.DOC_DATA_PATH
+            texdatafolder = osp.dirname(
+                settings.get_doc_tex_data_path(
+                    should_be_readable=False, create_parent=True
+                )
+            )
     try:
         index = 0
         total = failed = skipped = 0
@@ -385,14 +390,18 @@ def test_all(
 
 
 def load_doc_data():
-    print(f"Loading internal document data from {settings.DOC_DATA_PATH}")
-    with open_ensure_dir(settings.DOC_DATA_PATH, "rb") as doc_data_file:
+    doc_tex_data_path = settings.get_doc_tex_data_path(should_be_readable=True)
+    print(f"Loading internal document data from {doc_tex_data_path}")
+    with open_ensure_dir(doc_tex_data_path, "rb") as doc_data_file:
         return pickle.load(doc_data_file)
 
 
 def save_doc_data(output_data):
-    print(f"Writing internal document data to {settings.DOC_DATA_PATH}")
-    with open_ensure_dir(settings.DOC_DATA_PATH, "wb") as output_file:
+    doc_tex_data_path = settings.get_doc_tex_data_path(
+        should_be_readable=False, create_parent=True
+    )
+    print(f"Writing internal document data to {doc_tex_data_path}")
+    with open(settings.DOC_USER_TEX_DATA_PATH, "wb") as output_file:
         pickle.dump(output_data, output_file, 4)
 
 

mathics/settings.py

@@ -4,7 +4,8 @@
 import pkg_resources
 import sys
 import os
-from os import path
+import os.path as osp
+from pathlib import Path
 
 
 DEBUG = True
@@ -27,27 +28,53 @@
 if sys.platform.startswith("win"):
     DATA_DIR = os.environ["APPDATA"].replace(os.sep, "/") + "/Python/Mathics/"
 else:
-    DATA_DIR = path.expanduser("~/.local/var/mathics/")
-# if not path.exists(DATA_DIR):
-#    os.makedirs(DATA_DIR)
+    DATA_DIR = osp.expanduser("~/.local/var/mathics/")
 
-# Location of internal document data.
-# NOTE: Storing this in JSON if possible would be preferable and faster
-DOC_DATA_PATH = os.path.join(DATA_DIR, "doc_data.pcl")
+# Location of internal document data. Current this is in Python
+# Pickle form, but storing this in JSON if possible would be preferable and faster
 
-DOC_DIR = os.path.join(ROOT_DIR, "doc/documentation/")
-DOC_LATEX_FILE = os.path.join(ROOT_DIR, "doc/tex/documentation.tex")
+# We need two versions, one in the user space which is updated with
+# local packages installed and is user writable.
+DOC_USER_TEX_DATA_PATH = osp.join(DATA_DIR, "doc_tex_data.pcl")
 
+# We need another version as a fallback, and that is distributed with the
+# package. It is note user writable and not in the user space.
+DOC_SYSTEM_TEX_DATA_PATH = osp.join(ROOT_DIR, "data", "doc_tex_data.pcl")
+
+DOC_DIR = osp.join(ROOT_DIR, "doc", "documentation")
+DOC_LATEX_FILE = osp.join(ROOT_DIR, "doc", "tex", "documentation.tex")
 
 # Set this True if you prefer 12 hour time to be the default
 TIME_12HOUR = False
 
 # Leave this True unless you have specific reason for not permitting
-# users to access local files
+# users to access local files.
 ENABLE_FILES_MODULE = True
 
 # Rocky: this is probably a hack. LoadModule[] needs to handle
 # whatever it is that setting this thing did.
 default_pymathics_modules = []
 
 SYSTEM_CHARACTER_ENCODING = "UTF-8" if sys.getdefaultencoding() == "utf-8" else "ASCII"
+
+
+def get_doc_tex_data_path(should_be_readable=False, create_parent=False) -> str:
+    """Returns a string path where we can find Python Pickle data for LaTeX
+    processing.
+
+    If `should_be_readable` is True, the we will check to see whether this file is
+    readable (which also means it exists). If not, we'll return the `DOC_SYSTEM_DATA_PATH`.
+    """
+    doc_user_tex_data_path = Path(DOC_USER_TEX_DATA_PATH)
+    base_config_dir = doc_user_tex_data_path.parent
+    if not base_config_dir.is_dir() and create_parent:
+        Path("base_config_dir").mkdir(parents=True, exist_ok=True)
+
+    if should_be_readable:
+        return (
+            DOC_USER_TEX_DATA_PATH
+            if doc_user_tex_data_path.is_file
+            else DOC_SYSTEM_TEX_DATA_PATH
+        )
+    else:
+        return DOC_USER_TEX_DATA_PATH

mathics/version.py

@@ -5,4 +5,4 @@
 # well as importing into Python. That's why there is no
 # space around "=" below.
 # fmt: off
-__version__="3.1.1.dev0"  # noqa
+__version__="4.0.0.dev0"  # noqa

setup.py

@@ -158,8 +158,10 @@ def subdirs(root, file="*.*", depth=10):
     package_data={
         "mathics": [
             "data/*.csv",
+            "data/*.json",
             "data/*.yml",
             "data/*.yaml",
+            "data/*.pcl",
             "data/ExampleData/*",
             "doc/xml/data",
             "doc/tex/data",