[std, macros] Use "environ" for grammar extraction.

This replaces the use of the "extract" package with the simpler
"environ" package, where the grammar contents are extracted from the
"collected body" \BODY macro via the TeX `\meaning` command. This does
not preserve whitespace as the "extact" approach did, but if we just
spell out whitespace (instead of relying on \obeyspaces), we get the
same results.

Author: tkoeppe

.github/workflows/check.yml

@@ -53,7 +53,7 @@ jobs:
           eval "$(/usr/libexec/path_helper)"
           echo "PATH=${PATH}" >> ${GITHUB_ENV}
           sudo tlmgr update --self
-          sudo tlmgr install latexmk isodate substr relsize ulem fixme rsfs extract layouts enumitem l3packages l3kernel imakeidx splitindex xstring
+          sudo tlmgr install latexmk isodate substr relsize ulem fixme rsfs environ layouts enumitem l3packages l3kernel imakeidx splitindex xstring
 
       - name: make (Linux)
         if: matrix.cfg.os == 'ubuntu-24.04'

source/grammar.tex

@@ -58,4 +58,4 @@
 	identifier
 \end{ncbnf}
 
-\FlushAndPrintGrammar
+\input{std-gram.ext}

source/macros.tex

@@ -39,20 +39,18 @@
 \newenvironment{addedblock}{\color{addclr}}{\color{black}}
 \newenvironment{removedblock}{\color{remclr}}{\color{black}}
 
-%%--------------------------------------------------
+%%-------------------------------------------------------------
 %% Grammar extraction.
-%%--------------------------------------------------
-\def\gramSec[#1]#2{}
-
+%% Assumes that the output file \gramout is managed externally.
+%%-------------------------------------------------------------
 \makeatletter
-\newcommand{\FlushAndPrintGrammar}{%
-\immediate\closeout\XTR@out%
-\immediate\openout\XTR@out=std-gram-dummy.tmp%
-\def\gramSec[##1]##2{\rSec1[##1]{##2}}%
-\input{std-gram.ext}%
-}
+\newcommand{\gramWrite}[1]{\protected@write\gramout{}{#1}}
+\newcommand{\meaningbody}[1]{\expandafter\strip@prefix\meaning#1}
 \makeatother
 
+\newcommand{\gramSec}[2][]{\gramWrite{%
+\string\rSec1\string[\string#1\string]\string{\string#2\string}}}
+
 %%--------------------------------------------------
 % Escaping for index entries. Replaces ! with "! throughout its argument.
 %%--------------------------------------------------
@@ -701,7 +699,12 @@
  \nonfrenchspacing
  }
 
-\newenvironment{simplebnf}
+% "ncbnf" is the non-copied "base" versions of the bnf environment;
+% instances of the full "bnf" environment is copied to the grammar
+% extraction file.
+% (Similarly for "ncsimplebnf", though in fact we never extract any
+% hypothetical "simplebnf" environments.)
+\newenvironment{ncsimplebnf}
 {
  \begin{bnfbase}
  \BnfNontermshape
@@ -712,7 +715,7 @@
  \end{bnfbase}
 }
 
-\newenvironment{bnf}
+\newenvironment{ncbnf}
 {
  \begin{bnfbase}
  \begin{bnflist}
@@ -724,6 +727,7 @@
  \end{bnfbase}
 }
 
+% The regex grammar is never copied.
 \newenvironment{ncrebnf}
 {
  \begin{bnfbase}
@@ -737,11 +741,10 @@
  \end{bnfbase}
 }
 
-% non-copied versions of bnf environments
-\let\ncsimplebnf\simplebnf
-\let\endncsimplebnf\endsimplebnf
-\let\ncbnf\bnf
-\let\endncbnf\endbnf
+\NewEnviron{bnf}{\begin{ncbnf}%
+\BODY%
+\gramWrite{\string\begin{ncbnf}\meaningbody\BODY\string\end{ncbnf}}%
+\end{ncbnf}}{}
 
 %%--------------------------------------------------
 %% Environment for imported graphics

source/std.tex

@@ -46,7 +46,7 @@
             urlcolor=blue,    % ISO/IEC Directives, Part 2, section 6.5
             plainpages=false}
 \usepackage{memhfixc}     % fix interactions between hyperref and memoir
-\usepackage[active,header=false,handles=false,copydocumentclass=false,generate=std-gram.ext,extract-cmdline={gramSec},extract-env={bnf,simplebnf}]{extract} % Grammar extraction
+\usepackage{environ}
 \usepackage{expl3}
 \usepackage{xparse}
 \usepackage{xstring}
@@ -57,6 +57,10 @@
 
 \renewcommand\RSsmallest{5.5pt}  % smallest font size for relsize
 
+% Begin grammar extraction...
+\newwrite\gramout
+\immediate\openout\gramout=std-gram.ext
+
 \input{layout}
 \input{styles}
 \input{macros}
@@ -154,6 +158,9 @@
 
 \numberwithin{table}{chapter}
 
+% ... end grammar extraction.
+\immediate\closeout\gramout
+
 \include{grammar}
 \include{limits}
 \include{compatibility}