Add Settings`$PygmentsShowTokens

Author: rocky

mathicsscript/__main__.py

@@ -247,7 +247,13 @@ def main(
     # Set a default value for $ShowFullFormInput to False.
     # Then, it can be changed by the settings file (in WL)
     # and overwritten by the command line parameter.
-    definitions.set_ownvalue("Settings`$ShowFullFormInput", from_python(False))
+    definitions.set_ownvalue(
+        "Settings`$ShowFullFormInput", from_python(1 if full_form else 0)
+    )
+    definitions.set_ownvalue(
+        "Settings`$PygmentsShowTokens", from_python(1 if pygments_tokens else 0)
+    )
+
     shell = TerminalShell(definitions, style, readline, completion)
     load_settings(shell)
     if initfile:
@@ -277,7 +283,7 @@ def main(
                 shell.definitions, output=TerminalOutput(shell), format="text"
             )
             result = evaluation.parse_evaluate(expr, timeout=settings.TIMEOUT)
-            shell.print_result(result, debug_pygments=pygments_tokens)
+            shell.print_result(result)
 
             # After the next release, we can remove the hasattr test.
             if hasattr(evaluation, "exc_result"):
@@ -326,6 +332,9 @@ def main(
     definitions.set_ownvalue(
         "Settings`$PygmentsStyle", from_python(shell.pygments_style)
     )
+    definitions.set_ownvalue(
+        "Settings`$PygmentsShowTokens", from_python(pygments_tokens)
+    )
 
     TeXForm = Symbol("System`TeXForm")
 
@@ -369,7 +378,7 @@ def main(
                 print(fmt(query))
             result = evaluation.evaluate(query, timeout=settings.TIMEOUT)
             if result is not None:
-                shell.print_result(result, output_style, debug_pygments=pygments_tokens)
+                shell.print_result(result, output_style)
         except (KeyboardInterrupt):
             print("\nKeyboardInterrupt")
         except EOFError:

mathicsscript/settings.m

@@ -12,14 +12,9 @@
 Note this is for input entered, not the output of the evaluated result.
 "
 
-
 Settings`$ShowFullFormInput = 0
-
-
 Settings`$PygmentsStyle::usage = "This sets the Pygments style used to colorize output. The value should be a string.
 
 The default value changes background depending on whether the terminal has a light or dark background. You can also set the color style used on the command with the ``--style`` option, or look at the variable ```Settings`PygmentsStylesAvailable```. Or it can be set in the settings.m file."
 
-
-
-
+Settings`$PygmentsShowTokens::usage = "Setting this 1 will show Pygments tokenization of the output."

mathicsscript/termshell.py

@@ -14,8 +14,8 @@
 from mathics.core.rules import Rule
 from mathics.core.characters import named_characters
 
-from pygments import highlight
-from mathematica.lexer import MathematicaLexer, MToken
+from pygments import highlight, lex
+from mathematica.lexer import MathematicaLexer
 
 mma_lexer = MathematicaLexer()
 
@@ -30,7 +30,7 @@
     # Keyword,
     Name,
     Literal,
-    # Operator,
+    Operator,
     # String,
     Token,
 )
@@ -40,7 +40,7 @@
 color_scheme = TERMINAL_COLORS.copy()
 color_scheme[Token.Name] = ("yellow", "ansibrightyellow")
 color_scheme[Name.Function] = ("ansigreen", "ansibrightgreen")
-color_scheme[Name.NameSpace] = ("magenta", "ansibrightmagenta")
+color_scheme[Operator] = ("magenta", "ansibrightmagenta")
 color_scheme[Literal.Number] = ("ansiblue", "ansibrightblue")
 
 from colorama import init as colorama_init
@@ -152,6 +152,9 @@ def __init__(
 
         self.pygments_style = style
         self.definitions = definitions
+        self.definitions.set_ownvalue(
+            "Settings`$PygmentsShowTokens", from_python(False)
+        )
         self.definitions.set_ownvalue("Settings`$PygmentsStyle", from_python(style))
         self.definitions.set_ownvalue(
             "Settings`PygmentsStylesAvailable", from_python(ALL_PYGMENTS_STYLES)
@@ -214,15 +217,13 @@ def read_line(self, prompt):
             return self.rl_read_line(prompt)
         return input(prompt)
 
-    def print_result(self, result, output_style="", debug_pygments=False):
+    def print_result(self, result, output_style=""):
         if result is not None and result.result is not None:
             out_str = str(result.result)
             if self.terminal_formatter:  # pygmentize
-                from pygments import lex
-
-                if debug_pygments:
-                    print(list(lex(out_str, mma_lexer)))
-
+                show_pygments_tokens = self.definitions.get_ownvalue(
+                    "Settings`$PygmentsShowTokens"
+                ).replace.to_python()
                 pygments_style = self.definitions.get_ownvalue(
                     "Settings`$PygmentsStyle"
                 ).replace.get_string_value()
@@ -232,6 +233,8 @@ def print_result(self, result, output_style="", debug_pygments=False):
                             "Settings`$PygmentsStyle", String(self.pygments_style)
                         )
 
+                if show_pygments_tokens:
+                    print(list(lex(out_str, mma_lexer)))
                 out_str = highlight(out_str, mma_lexer, self.terminal_formatter)
             output = self.to_output(out_str)
             print(self.get_out_prompt(output_style) + output + "\n")