Update highlighter and Docker to Python 3

This also updates the install process for the highlighter to pull from pip.

Author: sideshowbarker

.gitmodules

@@ -1,16 +1,17 @@
 FROM debian:stable
 
-## dependency installation: nginx, wattsi, and other build tools
-## cleanup freepascal since it is no longer needed after wattsi build
+## dependency installation: nginx and other build tools
 RUN apt-get update && \
-    apt-get install -y ca-certificates curl git unzip nginx python2.7 python-pip && \
+    apt-get install -y ca-certificates curl git unzip nginx python3 python3-pip && \
     rm -rf /etc/nginx/sites-enabled/* && \
     rm -rf /var/lib/apt/lists/*
 
 COPY --from=whatwg/wattsi:latest /whatwg/wattsi/bin/wattsi /bin/wattsi
 
 ADD . /whatwg/build
 
+RUN pip3 install bs-highlighter
+
 ARG html_source_dir
 ADD $html_source_dir /whatwg/html
 ENV HTML_SOURCE /whatwg/html

Dockerfile

@@ -8,7 +8,7 @@ Make sure you have `git` installed on your system, and you are using a Bash shel
 
 Then, clone this ([html-build](https://github.com/whatwg/html-build)) repo:
 
-```
+```bash
 git clone https://github.com/whatwg/html-build.git && cd html-build
 ```
 
@@ -22,13 +22,13 @@ To build locally, you'll need the following commands installed on your system:
 
 - `curl`, `grep`, `perl`, `unzip`
 
-Optionally, for faster builds, you can install [Wattsi](https://github.com/whatwg/wattsi) and Python 2.7 (necessary for applying syntax highlighting to `pre` contents). If you don't bother with that, the build will use [Wattsi Server](https://github.com/domenic/wattsi-server), which requires an internet connection.
+Optionally, for faster builds, you can install [Wattsi](https://github.com/whatwg/wattsi) and Python 3.7+ (necessary for applying syntax highlighting to `pre` contents). If you don't bother with that, the build will use [Wattsi Server](https://github.com/domenic/wattsi-server), which requires an internet connection.
 
 ### Running the build
 
 Run the `build.sh` script from inside your `html-build` working directory, like this:
 
-```
+```bash
 ./build.sh
 ```
 
@@ -59,7 +59,7 @@ The Dockerized version of the build allows you to run the build entirely inside
 
 To perform a Dockerized build, use the `--docker` flag:
 
-```
+```bash
 ./build.sh --docker
 ```
 
@@ -73,13 +73,13 @@ Note that due to the way Docker works, the HTML source repository must be contai
 
 Your clone doesn't need the HTML standard's complete revision history just for you to build the spec and contribute patches. So, if you use `build.sh` to create the clone, we don't start you out with a clone of the history. That makes your first build finish much faster. And if later you decide you do want to clone the complete history, you can still get it, by doing this:
 
-```
+```bash
 cd ./html && git fetch --unshallow
 ```
 
 That said, if you really do want to *start out* with the complete history of the repo, then run the build script for the first time like this:
 
-```
+```bash
 HTML_GIT_CLONE_OPTIONS="" ./build.sh
 ```
 

README.md

@@ -36,16 +36,19 @@ export HTML_TEMP
 # Used specifically when the Dockerfile calls this script
 SKIP_BUILD_UPDATE_CHECK=${SKIP_BUILD_UPDATE_CHECK:-false}
 SHA_OVERRIDE=${SHA_OVERRIDE:-}
-HIGHLIGHT_SERVER_URL="http://127.0.0.1:8080" # this needs to be coordinated with the highlighter submodule
+HIGHLIGHT_SERVER_URL="http://127.0.0.1:8080" # this needs to be coordinated with the bs-highlighter package
 
 function main {
   processCommandLineArgs "$@"
 
   # $SKIP_BUILD_UPDATE_CHECK is set inside the Dockerfile so that we don't check for updates both inside and outside
   # the Docker container.
   if [[ $DO_UPDATE == "true" && $SKIP_BUILD_UPDATE_CHECK != "true" ]]; then
-    git submodule update --init
     checkHTMLBuildIsUpToDate
+    # If we're using Docker then this will be installed inside the container.
+    if [[ $USE_DOCKER != "true" ]]; then
+      pip3 install bs-highlighter
+    fi
   fi
 
   findHTMLSource
@@ -627,14 +630,10 @@ function generateBacklinks {
 # - A server will be running in the background, at $HIGHLIGHT_SERVER_URL
 # - $HIGHLIGHT_SERVER_PID will be set for later use by stopHighlightServer
 function startHighlightServer {
-  # Setting PYTHONPATH is a workaround for https://github.com/whatwg/html-build/issues/169.
-  # See also https://github.com/tabatkins/highlighter/issues/5 and
-  # https://bitbucket.org/birkenfeld/pygments-main/issues/1448.
-  export PYTHONPATH="$DIR/highlighter/highlighter/pygments${PYTHONPATH:+:$PYTHONPATH}"
   HIGHLIGHT_SERVER_ARGS=()
   $QUIET && HIGHLIGHT_SERVER_ARGS+=( --quiet )
   # shellcheck disable=SC2068
-  "$DIR/highlighter/server.py" ${HIGHLIGHT_SERVER_ARGS[@]+"${HIGHLIGHT_SERVER_ARGS[@]}"} &
+  bs-highlighter-server ${HIGHLIGHT_SERVER_ARGS[@]+"${HIGHLIGHT_SERVER_ARGS[@]}"} &
   HIGHLIGHT_SERVER_PID=$!
 
   trap stopHighlightServer EXIT

build.sh

@@ -6,7 +6,7 @@ RUN apt-get update && \
     apt-get install -y ca-certificates curl rsync git unzip \
     default-jre                                             \
     libfontconfig1 libgomp1 libxml2                         \
-    python2.7 python-pip                                    \
+    python3 python3-pip                                     \
     fonts-dejavu fonts-droid-fallback fonts-liberation fonts-symbola fonts-unfonts-core
 
 # Dependency lines above are:
@@ -16,6 +16,8 @@ RUN apt-get update && \
 # - Highlighter
 # - fonts
 
+RUN pip3 install bs-highlighter
+
 # Dependecies of prince_11.3-1_debian8.0_amd64.deb (not used) are libc6 libcurl3 libfontconfig1
 # libfreetype6 libgif4 libgomp1 libjpeg62-turbo libpng12-0 libssl1.0.0 libtiff5 libxml2 zlib1g.
 # Above we install only the subset that's needed to make Prince work.

ci-deploy/Dockerfile

@@ -17,13 +17,6 @@ IS_TEST_OF_HTML_BUILD_ITSELF=${IS_TEST_OF_HTML_BUILD_ITSELF:-false}
 # - DOCKER_PASSWORD is set from the outside
 # - ENCRYPTION_LABEL is set from the outside
 
-# Initialize the highlighter submodule for html-build
-(
-  cd html-build
-  git submodule init
-  git submodule update
-)
-
 git clone --depth 1 https://github.com/pts/pdfsizeopt.git pdfsizeopt
 
 # Copy the Docker-related stuff into the working (grandparent) directory.