Merge branch 'master' of github.com:RunestoneInteractive/RunestoneServer

Author: bnmnetp

.github/workflows/python-app.yml

@@ -35,12 +35,12 @@ jobs:
                 id: docker-up
                 run: |
                     cd RunestoneServer
-                    docker-compose up -d
-                    docker-compose logs --follow &
+                    docker compose up -d
+                    docker compose logs --follow &
                     # Wait for the container to start and create a basic venv before running docker tools inside it.
                     sleep 5
                     # Invoke this directly, since the ``docker-tools`` script isn't installed yet. After this finishes, the docker tools are installed.
-                    docker exec -t runestoneserver_runestone_1 bash -c "\$RUNESTONE_PATH/.venv/bin/python \$RUNESTONE_PATH/docker/docker_tools.py wait"
+                    docker exec -t runestoneserver-runestone-1 bash -c "\$RUNESTONE_PATH/.venv/bin/python \$RUNESTONE_PATH/docker/docker_tools.py wait"
             -   name: Test
                 id: pytest
                 run: |

docker/README.rst

@@ -206,23 +206,23 @@ For the use case of running the server, you will need to modify these settings t
 
 Pre-start
 ^^^^^^^^^
-Once your environment is ready to go, you can use ``docker-compose`` to bring the containers up. This command will create four containers to run different parts of the application stack (the Runestone server, redis cache, postgres DB, jobe code testing environment).
+Once your environment is ready to go, you can use ``docker compose`` to bring the containers up. This command will create four containers to run different parts of the application stack (the Runestone server, redis cache, postgres DB, jobe code testing environment).
 
 For the use case of running the server, execute:
 
     .. code-block:: bash
 
-        docker-compose up -d
+        docker compose up -d
 
-    This run the container in the background (detached mode). Use ``docker-compose logs --follow`` to view logging data as the container starts up and runs.
+    This run the container in the background (detached mode). Use ``docker compose logs --follow`` to view logging data as the container starts up and runs.
 
 **OR**
 
 For the developer use case, execute:
 
     .. code-block:: bash
 
-        docker-compose up
+        docker compose up
 
     This displays logging data from the container in the terminal. To Stop the container, press when ctrl-c.
 
@@ -254,11 +254,11 @@ No books are installed by default; you must add books using the following proces
 
     rsmanage build --course thinkcspy --clone https://github.com/RunestoneInteractive/thinkcspy.git
 
+After cloning a book, you may need to add it to the database.  Most of the standard books are already there, but you can use ``rsmanage addcourse`` to add it if needed.
 
 .. note::
 
-    TODO: Edit/improve the docs from this point forward.
-
+    PreTeXt authors, see `Publishing to Runestone Academy <https://pretextbook.org/doc/guide/html/sec-publishing-to-runestone-academy.html>`_. The following information applies only *authoring* books using the Runestone.
 
 .. warning::
 
@@ -268,8 +268,6 @@ No books are installed by default; you must add books using the following proces
    If there is a mismatch, you will want to rename the folder you cloned the code into so that it
    matches the ``project_name``.
 
-After cloning a book, you may need to add it to the database.  Most of the standard books are already there, but you can use ``rsmanage addcourse`` to add it if needed.
-
 .. note::
 
    Most Runestone books set ``master_url`` to ``get_master_url()`` in their ``pavement.py`` file. However, if the book
@@ -349,7 +347,7 @@ trying to load the same records and entering a restart loop because the records
 
 Operation
 ---------
-The containerized application is configured to automatically start as soon as Docker / the Docker Desktop is started. Therefore, on OS X or Windows (when using WSL2): after a reboot or after manually shutting down the Docker Desktop, remember to start the Docker Desktop application.
+The containerized application is configured to automatically start as soon as Docker / the Docker Desktop is started. Therefore, on OS X or Windows (when using WSL2): after a reboot or after manually shutting down the Docker Desktop, **remember to start the Docker Desktop application**.
 
 
 Other Tips & Tricks
@@ -423,12 +421,12 @@ Testing the Entrypoint
 **********************************
 
 If you want to test the script, the easiest thing
-to do is add a command to the docker-compose to disable it, and then run commands
+to do is add a command to the ``docker compose`` to disable it, and then run commands
 interactively by shelling into the container.
 
 Bring up the containers and then shell inside. Once inside, you can then issue commands
 to test the entry point script - since the other containers were started
-with docker-compose everything in them is ready to go.
+with ``docker compose`` everything in them is ready to go.
 
 File Permissions
 **********************************
@@ -441,23 +439,17 @@ container enough privileges to do your work.
 Writing Your Own Book
 **********************************
 
+.. note::
+
+    PreTeXt authors, see `Publishing to Runestone Academy <https://pretextbook.org/doc/guide/html/sec-publishing-to-runestone-academy.html>`_. The following information applies only *authoring* books using the Runestone.
+
 If you are writing your own book you will want to get that book set up properly in the Runestone
 system. You need to do the following:
 
 #.  Run the command ``rsmanage addcourse``. Use the project name you configured in ``pavement.py`` as the name of BOTH the course and the basecourse when it asks.
 
 #.  Now that your course is registered, rebuild it using the command ``rsmanage build --course <book_name>`` command.
 
-#.  If this book is a PreTeXt book you will need to navigate to the directory that contains the ``runestone-manifest.xml`` file and run the command:
-
-    .. code-block:: bash
-
-        runestone process-manifest --course <yourcourse> --manifest runestone-manifest.xml
-
-.. note::
-
-    If you are missing ``runestone-manifest.xml`` then you need to rebuild your PreTeXt
-    book with ``runestone`` as the publisher. See the PreTeXt docs for how do do this.
 
 Changing dependencies
 *********************

docker/docker_tools.py

@@ -67,7 +67,7 @@
 # Local application bootstrap
 # ---------------------------
 # Everything after this depends on Unix utilities. We can't use ``is_win`` because we don't know if ``ci_utils`` is available.
-if sys.platform == "win32":
+if sys.platform == "win32x":
     sys.exit("ERROR: You must run this program in WSL/VirtualBox/VMWare/etc.")
 
 # See if we're root.
@@ -180,16 +180,6 @@ def pip_user() -> str:
     print("Note: this must be an initial install; additional commands missing.")
 
 
-# Global variables
-# ================
-# .. note::
-#
-#   Update these regularly!
-#
-# See the `Docker compose release page <https://github.com/docker/compose/releases>`_ for the latest version.
-DOCKER_COMPOSE_VERSION = "2.29.2"
-
-
 # CLI
 # ===
 # Create a series of subcommands for this CLI.
@@ -207,15 +197,6 @@ def cli() -> None:
 
 # ``init`` command
 # ================
-# Check if we're running in WSL or OS X -- don't install Docker if so.
-def check_requires_docker_desktop() -> None:
-    if (is_linux and "WSL" in Path("/proc/version").read_text()) or is_darwin:
-        sys.exit(
-            "ERROR: Docker Desktop not detected. You must install and run this\n"
-            "before proceeding."
-        )
-
-
 @cli.command()
 @click.option(
     "--clone-rs",
@@ -237,9 +218,16 @@ def init(
     try:
         xqt("docker --version")
     except subprocess.CalledProcessError as e:
-        check_requires_docker_desktop()
+        print(f"Unable to run docker: {e}")
+        # Ensure the Docker Desktop is running if we're running in WSL. On Windows, the ``docker`` command doesn't exist when the Docker Desktop isn't running.
+        if is_linux and "WSL" in Path("/proc/version").read_text():
+            sys.exit(
+                "ERROR: Docker Desktop not detected. You must install and run this\n"
+                "before proceeding."
+            )
+
         check_install_curl()
-        print(f"Unable to run docker: {e} Installing Docker...")
+        print("Installing Docker...")
         # Use the `convenience script <https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script>`_.
         xqt(
             "curl -fsSL https://get.docker.com -o get-docker.sh",
@@ -253,21 +241,24 @@ def init(
             if is_darwin:
                 xqt("sudo dscl . append /Groups/docker GroupMembership $USER")
             else:
-                xqt("sudo usermod -aG docker ${USER}")
+                xqt("sudo usermod -a -G docker ${USER}")
 
             # The group add doesn't take effect until the user logs out then back in. Work around it for now.
             did_group_add = True
 
-    # Ensure docker-compose is installed.
+    # Ensure the Docker Desktop is running if this is OS X. On OS X, the ``docker`` command exists, but can't run the hello, world script. It also serves as a sanity check for the other platforms.
+    print("Checking that Docker works...")
     try:
-        xqt("docker-compose --version")
+        xqt("docker run hello-world")
     except subprocess.CalledProcessError as e:
-        check_requires_docker_desktop()
-        print(f"Unable to run docker-compose: {e} Installing...")
-        # This is from the `docker-compose install instructions <https://docs.docker.com/compose/install/#install-compose-on-linux-systems>`_.
-        xqt(
-            f'sudo curl -L "https://github.com/docker/compose/releases/download/{DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose',
-            "sudo chmod +x /usr/local/bin/docker-compose",
+        print(f"Unable to execute docker run hello-world: {e}")
+        sys.exit(
+            (
+                "ERROR: Docker Desktop not detected. You must install and run this\n"
+                "before proceeding."
+            )
+            if is_darwin
+            else "ERROR: Unable to run a basic Docker application."
         )
 
     # Make sure git's installed.
@@ -303,6 +294,7 @@ def init(
         if is_darwin:
             xqt(
                 "sudo dscl . create /Groups/www-data",
+                "sudo dscl . create /Groups/www-data gid 799",
                 "sudo dseditgroup -o edit -a $USER -t user www-data",
                 "sudo dscl . append /Groups/www-data GroupMembership $USER",
             )

docker/docker_tools_misc.py

@@ -88,7 +88,9 @@ def _start_servers(dev: bool) -> None:
     xqt(
         'sudo -u www-data env "PATH=$PATH" "REDIS_URI=$REDIS_URI" '
         "poetry run celery --app=scheduled_builder worker --pool=threads "
-        "--concurrency=3 --loglevel=info &",
+        "--concurrency=3 --loglevel=info "
+        # This redirect ensures output ends up in the Docker log even if the servers are restarted. Sending to ``/dev/stdout`` only works at initial startup, but doesn't redirect after the servers are restarted. For more discussion, see `Github <https://github.com/moby/moby/issues/19616#issuecomment-174355979>`_.
+        "--logfile=/proc/1/fd/1 &",
         cwd=f"{env.RUNESTONE_PATH}/modules",
     )
 
@@ -98,16 +100,16 @@ def _start_servers(dev: bool) -> None:
         "--error_path /tmp "
         "--gconfig $RUNESTONE_PATH/docker/gunicorn_config/fastapi_config.py "
         # This much match the address in `./nginx/sites-available/runestone.template`.
-        "--bind unix:/run/fastapi.sock "
-        + ("--reload " if dev else "")
-        + "2>&1 > /proc/1/fd/1 &",  # This redirect ensures output ends up in the docker log
+        f"--bind unix:/run/fastapi.sock {'--reload ' if dev else ''}"
+        # See previous comment on redirecting output to ``/dev/stdout`` even after server restarts.
+        + "2>&1 > /proc/1/fd/1 &",
         "service nginx start",
         "poetry run gunicorn -D --config $RUNESTONE_PATH/docker/gunicorn_config/web2py_config.py &",
         cwd=f"{env.RUNESTONE_PATH}/docker/gunicorn_config",
     )
 
     # Start the script to collect tickets and store them in the database. Most useful
-    # for a production environment with several worker containers
+    # for a production environment with several worker containers.
     xqt(
         f"cp {env.RUNESTONE_PATH}/scripts/tickets2db.py {env.WEB2PY_PATH}",
         "python web2py.py -M -S runestone --run tickets2db.py &",
@@ -137,6 +139,8 @@ def _stop_servers() -> None:
     )
 
 
+# ``restart_servers``
+# -------------------
 @click.command()
 @click.option(
     "--dev/--no-dev",
@@ -152,6 +156,8 @@ def restart_servers(dev):
     _start_servers()
 
 
+# ``reloadbks``
+# -------------
 @click.command()
 def reloadbks() -> None:
     """
@@ -162,7 +168,8 @@ def reloadbks() -> None:
         pid = pfile.read().strip()
 
     pid = int(pid)
-    os.kill(pid, 1)  # send the HUP signal to bookserver
+    # send the HUP signal to the BookServer.
+    os.kill(pid, 1)
 
 
 # ``test``
@@ -179,12 +186,17 @@ def reloadbks() -> None:
 @click.argument("passthrough", nargs=-1, type=click.UNPROCESSED)
 def test(bks: bool, rc: bool, rs: bool, passthrough: Tuple) -> None:
     """
-    Run unit tests.
+    Run unit tests. All tests are disabled by default; manually select which test to run.
 
         PASSTHROUGH: These arguments are passed directly to the underlying "pytest" command. To pass options to this command, prefix this argument with "--". For example, use "docker_tools.py test -- -k test_just_this" instead of "docker_tools.py test -k test_just_this" (which produces an error).
 
     """
     ensure_in_docker()
+    if not bks and not rc and not rs:
+        sys.exit(
+            "ERROR: No tests selected to run. Pass any combination of --rs, --rs,\n"
+            "and/or --bks."
+        )
     _stop_servers()
     pytest = "$RUNESTONE_PATH/.venv/bin/pytest"
     passthrough_args = " ".join(passthrough)
@@ -261,7 +273,7 @@ def ensure_in_docker(
         return True
     # Get the name of the container running the Runestone servers.
     res = subprocess.run(
-        'docker ps --filter "ancestor=runestone/server"  --format "{{.Names}}"',
+        'docker ps --filter "ancestor=runestone/server" --format "{{.Names}}"',
         shell=True,
         capture_output=True,
         text=True,

pyproject.toml

@@ -36,6 +36,7 @@ pathlib2 = "^2.0.0"
 Paver = ">= 1.2.4"
 # This is actually an rsmanage dependency. It's here so that rsmanage can be installed without installing this, which requires Postgres (available only in the container).
 pgcli = "^3.0.0"
+pretextbook = "^0.7"
 psycopg2-binary = "^2.0.0"
 pylint = ">= 1.2.1"
 python = "^3.8"