Add optional packages into setup.py extra_requires

In particular ujson.
`pip install mathicsscript[full]` will include everything

Author: rocky

.github/workflows/osx.yaml

@@ -22,11 +22,13 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        pip install -e .
     - name: Install Mathics Scanner
       run: |
         make
     - name: Test Mathics Scanner
       run: |
-        pip install -r requirement-dev.txt
+        pip install -r requirements-dev.txt
+        pip install -r requirements-extra.txt
         python -m mathics_scanner.generate.build_tables
         make check

.github/workflows/ubuntu.yaml

@@ -21,11 +21,13 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        pip install -e .
     - name: Install Mathics_Scanner
       run: |
         make
     - name: Test Mathics Scanner
       run: |
-        pip install -r requirement-dev.txt
+        pip install -r requirements-dev.txt
+        pip install -r requirements-extra.txt
         python -m mathics_scanner.generate.build_tables
         make check

.github/workflows/windows.yml

@@ -22,11 +22,13 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        pip install -e .
     - name: Install Mathics_Scanner
       run: |
         python setup.py install
     - name: Test Mathics
       run: |
         pip install -r requirements-dev.txt
+        pip install -r requirements-extra.txt
         python mathics_scanner/generate/build_tables.py
         py.test test

requirements-dev.txt

@@ -1,3 +1 @@
 pytest
-# ujson is optional
-ujson

requirements-extra.txt

@@ -0,0 +1,2 @@
+# Optional packages which add functionality or speed things up
+ujson

setup.py

@@ -25,11 +25,11 @@
 [email protected] and ask for help.
 """
 
+import re
 import sys
 import os.path as osp
 import platform
-import subprocess
-from setuptools import setup, Command, Extension
+from setuptools import setup
 
 # Ensure user has the correct Python version
 if sys.version_info < (3, 6):
@@ -62,11 +62,22 @@ def read(*rnames):
 # General Requirements
 INSTALL_REQUIRES = [
     "chardet",  # Used in mathics_scanner.feed
-    # "ujson",  # Used in mathics_scanner.characters
+    "PyYAML",   # Used in mathics-generate-json-table
+    # "ujson",  # Optional Used in mathics_scanner.characters
     "click",  # Usin in CLI: mathics-generate-json-table
 ]
 
 
+extra_requires = []
+for line in open("requirements-extra.txt").read().split("\n"):
+    if line and not line.startswith("#"):
+        requires = re.sub(r"([^#]+)(\s*#.*$)?", r"\1", line)
+        extra_requires.append(requires)
+
+EXTRA_REQUIRES = {
+    "full": extra_requires
+}
+
 def subdirs(root, file="*.*", depth=10):
     for k in range(depth):
         yield root + "*/" * k + file
@@ -80,6 +91,7 @@ def subdirs(root, file="*.*", depth=10):
         "mathics_scanner.generate",
     ],
     install_requires=INSTALL_REQUIRES,
+    extra_requires=EXTRA_REQUIRES,
     entry_points={
         "console_scripts": [
             "mathics-generate-json-table=mathics_scanner.generate.build_tables:main"