Merge pull request #1509 from mathics/dup-settings

Revise doc data for TeX

Author: rocky

MANIFEST.in

@@ -7,4 +7,5 @@ include Makefile
 include requirements-dev.txt
 include requirements-full.txt
 recursive-include mathics *.py
+recursive-include mathics/data *
 recursive-include test *.py *.m

admin-tools/make-dist.sh

@@ -14,25 +14,19 @@ if ! source ./pyenv-versions ; then
     exit $?
 fi
 
-
 cd ..
 source mathics/version.py
+cp -v ${HOME}/.local/var/mathics/doc_tex_data.pcl mathics/data/
+
 echo $__version__
 
 for pyversion in $PYVERSIONS; do
     if ! pyenv local $pyversion ; then
 	exit $?
     fi
-    # pip bdist_egg create too-general wheels. So
-    # we narrow that by moving the generated wheel.
-
-    # Pick out first two number of version, e.g. 3.7.9 -> 37
-    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
     rm -fr build
     python setup.py bdist_egg
     python setup.py bdist_wheel
-    python setup.py bdist_wheel --universal
-    mv -v dist/${PACKAGE}-$VERSION-{py2.py3,py$first_two}-none-any.whl
 done
 
 python ./setup.py sdist

mathics/doc/common_doc.py

@@ -27,6 +27,7 @@
 """
 
 import importlib
+import os.path as osp
 import pkgutil
 import re
 
@@ -719,7 +720,7 @@ def __init__(self):
         self.parts = []
         self.parts_by_slug = {}
         self.pymathics_doc_loaded = False
-        self.doc_data_file = settings.DOC_DATA_PATH
+        self.doc_data_file = settings.get_doc_tex_data_path(should_be_readable=True)
         self.title = "Overview"
         files = listdir(self.doc_dir)
         files.sort()
@@ -730,7 +731,7 @@ def __init__(self):
             if part_title.endswith(".mdoc"):
                 part_title = part_title[: -len(".mdoc")]
                 part = DocPart(self, part_title)
-                text = open(self.doc_dir + file, "rb").read().decode("utf8")
+                text = open(osp.join(self.doc_dir, file), "rb").read().decode("utf8")
                 text = filter_comments(text)
                 chapters = CHAPTER_RE.findall(text)
                 for title, text in chapters:

mathics/doc/tex/Makefile

@@ -6,16 +6,17 @@ LATEXMK ?= latexmk
 BASH ?= /bin/bash
 #-quiet
 
-DOC_DATA_PCL ?= $(HOME)/.local/var/mathics/doc_data.pcl
+DOC_TEX_DATA_PCL ?= $(HOME)/.local/var/mathics/doc_tex_data.pcl
 
 #: Default target: Make everything
 all doc texdoc: mathics.pdf
 
-doc-data $(DOC_DATA_PCL):
+#: Create internal Document Data from .mdoc and Python builtin module docstrings
+doc-data $(DOC_TEX_DATA_PCL):
 	(cd ../.. && $(PYTHON) docpipeline.py --output --keep-going)
 
 #: Build mathics PDF
-mathics.pdf: mathics.tex documentation.tex logo-text-nodrop.pdf logo-heptatom.pdf $(DOC_DATA_PCL)
+mathics.pdf: mathics.tex documentation.tex logo-text-nodrop.pdf logo-heptatom.pdf $(DOC_TEX_DATA_PCL)
 	$(LATEXMK) --verbose -f -pdf -pdflatex="$(XETEX) -halt-on-error" mathics
 
 #: Build test PDF
@@ -28,7 +29,7 @@ logo-heptatom.pdf logo-text-nodrop.pdf:
 	(cd .. && $(BASH) ./images.sh)
 
 #: The build of the documentation which is derived from docstrings in the Python code
-documentation.tex: $(DOC_DATA_PCL)
+documentation.tex: $(DOC_TEX_DATA_PCL)
 	$(PYTHON) ./doc2latex.py
 
 #: Same as mathics.pdf
@@ -40,6 +41,6 @@ clean:
 	rm -f test-mathics.aux test-mathics.idx test-mathics.log test-mathics.mtc test-mathics.mtc* test-mathics.out test-mathics.toc || true
 	rm -f mathics.fdb_latexmk mathics.ilg mathics.ind mathics.maf mathics.pre || true
 	rm -f mathics_*.* || true
-	rm -f mathics-*.* documentation.tex doc_data.pcl || true
-	rm -f mathics.pdf mathics.dvi data_tex_data.pcl test-mathics.pdf test-mathics.dvi || true
+	rm -f mathics-*.* documentation.tex $(DOC_TEX_DATA_PCL) || true
+	rm -f mathics.pdf mathics.dvi test-mathics.pdf test-mathics.dvi || true
 	rm -f mathics-test.pdf mathics-test.dvi || true

mathics/doc/tex/doc2latex.py

@@ -28,7 +28,7 @@ def extract_doc_from_source(quiet=False):
     if not quiet:
         print(f"Extracting internal doc data for {version_string}")
     try:
-        return load_doc_data(settings.DOC_DATA_PATH)
+        return load_doc_data(settings.get_doc_tex_data_path(should_be_readable=True))
     except KeyboardInterrupt:
         print("\nAborted.\n")
         return

mathics/docpipeline.py

@@ -10,6 +10,7 @@
 """
 
 import os
+import os.path as osp
 import pickle
 import re
 import sys
@@ -302,8 +303,8 @@ def open_ensure_dir(f, *args, **kwargs):
     try:
         return open(f, *args, **kwargs)
     except (IOError, OSError):
-        d = os.path.dirname(f)
-        if d and not os.path.exists(d):
+        d = osp.dirname(f)
+        if d and not osp.exists(d):
             os.makedirs(d)
         return open(f, *args, **kwargs)
 
@@ -323,7 +324,11 @@ def test_all(
 
     if generate_output:
         if texdatafolder is None:
-            texdatafolder = settings.DOC_DATA_PATH
+            texdatafolder = osp.dirname(
+                settings.get_doc_tex_data_path(
+                    should_be_readable=False, create_parent=True
+                )
+            )
     try:
         index = 0
         total = failed = skipped = 0
@@ -385,14 +390,18 @@ def test_all(
 
 
 def load_doc_data():
-    print(f"Loading internal document data from {settings.DOC_DATA_PATH}")
-    with open_ensure_dir(settings.DOC_DATA_PATH, "rb") as doc_data_file:
+    doc_tex_data_path = settings.get_doc_tex_data_path(should_be_readable=True)
+    print(f"Loading internal document data from {doc_tex_data_path}")
+    with open_ensure_dir(doc_tex_data_path, "rb") as doc_data_file:
         return pickle.load(doc_data_file)
 
 
 def save_doc_data(output_data):
-    print(f"Writing internal document data to {settings.DOC_DATA_PATH}")
-    with open_ensure_dir(settings.DOC_DATA_PATH, "wb") as output_file:
+    doc_tex_data_path = settings.get_doc_tex_data_path(
+        should_be_readable=False, create_parent=True
+    )
+    print(f"Writing internal document data to {doc_tex_data_path}")
+    with open(settings.DOC_USER_TEX_DATA_PATH, "wb") as output_file:
         pickle.dump(output_data, output_file, 4)
 
 

mathics/settings.py

@@ -4,7 +4,8 @@
 import pkg_resources
 import sys
 import os
-from os import path
+import os.path as osp
+from pathlib import Path
 
 
 DEBUG = True
@@ -27,27 +28,53 @@
 if sys.platform.startswith("win"):
     DATA_DIR = os.environ["APPDATA"].replace(os.sep, "/") + "/Python/Mathics/"
 else:
-    DATA_DIR = path.expanduser("~/.local/var/mathics/")
-# if not path.exists(DATA_DIR):
-#    os.makedirs(DATA_DIR)
+    DATA_DIR = osp.expanduser("~/.local/var/mathics/")
 
-# Location of internal document data.
-# NOTE: Storing this in JSON if possible would be preferable and faster
-DOC_DATA_PATH = os.path.join(DATA_DIR, "doc_data.pcl")
+# Location of internal document data. Currently this is in Python
+# Pickle form, but storing this in JSON if possible would be preferable and faster
 
-DOC_DIR = os.path.join(ROOT_DIR, "doc/documentation/")
-DOC_LATEX_FILE = os.path.join(ROOT_DIR, "doc/tex/documentation.tex")
+# We need two versions, one in the user space which is updated with
+# local packages installed and is user writable.
+DOC_USER_TEX_DATA_PATH = osp.join(DATA_DIR, "doc_tex_data.pcl")
 
+# We need another version as a fallback, and that is distributed with the
+# package. It is note user writable and not in the user space.
+DOC_SYSTEM_TEX_DATA_PATH = osp.join(ROOT_DIR, "data", "doc_tex_data.pcl")
+
+DOC_DIR = osp.join(ROOT_DIR, "doc", "documentation")
+DOC_LATEX_FILE = osp.join(ROOT_DIR, "doc", "tex", "documentation.tex")
 
 # Set this True if you prefer 12 hour time to be the default
 TIME_12HOUR = False
 
 # Leave this True unless you have specific reason for not permitting
-# users to access local files
+# users to access local files.
 ENABLE_FILES_MODULE = True
 
 # Rocky: this is probably a hack. LoadModule[] needs to handle
 # whatever it is that setting this thing did.
 default_pymathics_modules = []
 
 SYSTEM_CHARACTER_ENCODING = "UTF-8" if sys.getdefaultencoding() == "utf-8" else "ASCII"
+
+
+def get_doc_tex_data_path(should_be_readable=False, create_parent=False) -> str:
+    """Returns a string path where we can find Python Pickle data for LaTeX
+    processing.
+
+    If `should_be_readable` is True, the we will check to see whether this file is
+    readable (which also means it exists). If not, we'll return the `DOC_SYSTEM_DATA_PATH`.
+    """
+    doc_user_tex_data_path = Path(DOC_USER_TEX_DATA_PATH)
+    base_config_dir = doc_user_tex_data_path.parent
+    if not base_config_dir.is_dir() and create_parent:
+        Path("base_config_dir").mkdir(parents=True, exist_ok=True)
+
+    if should_be_readable:
+        return (
+            DOC_USER_TEX_DATA_PATH
+            if doc_user_tex_data_path.is_file
+            else DOC_SYSTEM_TEX_DATA_PATH
+        )
+    else:
+        return DOC_USER_TEX_DATA_PATH

mathics/version.py

@@ -5,4 +5,4 @@
 # well as importing into Python. That's why there is no
 # space around "=" below.
 # fmt: off
-__version__="3.1.1.dev0"  # noqa
+__version__="4.0.0.dev0"  # noqa

setup.py

@@ -158,8 +158,10 @@ def subdirs(root, file="*.*", depth=10):
     package_data={
         "mathics": [
             "data/*.csv",
+            "data/*.json",
             "data/*.yml",
             "data/*.yaml",
+            "data/*.pcl",
             "data/ExampleData/*",
             "doc/xml/data",
             "doc/tex/data",