feat: AppArmor confinement for codejail-service

codejail.profile

@@ -0,0 +1,100 @@
+# AppArmor profile for running codejail-service in devstack.
+#
+#                         #=========#
+#                         # WARNING #
+#                         #=========#
+#
+# This is not a complete and secure apparmor profile! Do not use this
+# in any deployed environment (even a staging environment) without
+# careful inspection and modification to fit your needs.
+#
+# Failure to apply a secure apparmor profile *will* likely result in a
+# compromise of your environment by an attacker.
+#
+# We may at some point make this file good enough for confinement in
+# production, but for now it is only intended to be used in devstack.
+
+# Sets standard variables used by abstractions/base, later. Controlled
+# by OS, see /etc/apparmor.d/tunables/global for contents.
+include <tunables/global>
+
+# Require that the system understands the feature set that this policy was written
+# for. If we didn't include this, then on Ubuntu >= 22.04, AppArmor might assume
+# the wrong feature set was requested, and some rules might become too permissive.
+# See https://github.com/netblue30/firejail/issues/3659#issuecomment-711074899
+abi <abi/3.0>,
+
+# This outer profile applies to the entire container, and isn't as
+# important. If the sandbox profile doesn't work, it's not likely that
+# the outer one is going to help. But there may be some small value in
+# defense-in-depth, as it's possible that a bug in the child (sandbox)
+# profile isn't present in the outer one.
+profile codejail_service flags=(mediate_deleted) {
+
+    # Allow access to a variety of commonly needed, generally safe things
+    # (such as reading /dev/random, free memory, etc.)
+    #
+    # Manpage: "Includes files that should be readable and writable in all profiles."
+    include <abstractions/base>
+
+    # Filesystem access -- self-explanatory
+    file,
+
+    # `network` is required for sudo
+    # TODO: Restrict this so that general network access is not permitted
+    network,
+
+    # Various capabilities required for sudoing to sandbox (setuid,
+    # setgid, audit_write) and for sending a kill signal (kill).
+    capability setuid setgid audit_write kill,
+
+    # Allow sending a kill signal to the sandbox when the execution
+    # runs beyond time limits.
+    signal (send) set=(kill) peer=codejail_service//child,
+
+    # The core of the confinement: When the sandbox Python is executed, switch to
+    # the (extremely constrained) child profile.
+    #
+    # This path needs to be coordinated with the Dockerfile and Django settings.
+    #
+    # Manpage: "Cx: transition to subprofile on execute -- scrub the environment"
+    /sandbox/venv/bin/python Cx -> child,
+
+    # This is the important apparmor profile -- the one that actually
+    # constrains the sandbox Python process.
+    #
+    # mediate_deleted is not well documented, but it seems to indicate that
+    # apparmor will continue to make policy decisions in cases where a confined
+    # executable has a handle to a file's inode even after the file is removed
+    # from the filesystem.
+    profile child flags=(mediate_deleted) {
+
+        # This inner profile also gets general access to "safe"
+        # actions; we could list those explicitly out of caution but
+        # it could get pretty verbose.
+        include <abstractions/base>
+
+        # Read and run binaries and libraries in the virtualenv. This
+        # includes the sandbox's copy of Python as well as any
+        # dependencies that have been installed for inclusion in
+        # sandboxes.
+        /sandbox/venv/** rm,
+
+        # Allow access to the temporary directories that are set up by
+        # codejail, one for each code-exec call. Each /tmp/code-XXXXX
+        # contains one execution.
+        #
+        # Codejail has a hardcoded reference to this file path, although the
+        # use of /tmp specifically may be controllable with environment variables:
+        # https://github.com/openedx/codejail/blob/0165d9ca351/codejail/util.py#L15
+        /tmp/codejail-*/ r,
+        /tmp/codejail-*/** rw,
+
+        # Allow interactive terminal in devstack.
+        /dev/pts/* rw,
+
+        # Allow receiving a kill signal from the webapp when the execution
+        # runs beyond time limits.
+        signal (receive) set=(kill) peer=codejail_service,
+    }
+}

docker-compose.yml

@@ -726,11 +726,13 @@ services:
     hostname: codejail.devstack.edx
     stdin_open: true
     tty: true
-    image: edxops/codejail-dev:latest
+    image: edxops/codejail-service-dev:latest
     environment:
       DJANGO_SETTINGS_MODULE: codejail_service.settings.devstack
     ports:
       - "18030:8080"
+    security_opt:
+      - apparmor=codejail_service
 
   xqueue:
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue"

docs/codejail.rst

@@ -0,0 +1,43 @@
+Codejail service
+################
+
+The ``codejail`` devstack component (codejail-service) requires some additional configuration before it can be enabled. This page describes how to set it up and debug it.
+
+Background
+**********
+
+Both LMS and CMS can run Python code submitted by instructors and learners in order to implement custom Python-graded problems. By default this involves running the code on the same host as edxapp itself. Ordinarily this would be quite dangerous, but we use a sandboxing library called `codejail <https://github.com/openedx/codejail>`__ in order to confine the code execution in terms of disk and network access as well as memory, CPU, and other resource limits. Part of these restrictions are implemented via AppArmor, a utility available in some Linux distributions (including Debian and Ubuntu).
+
+While AppArmor provides good protection, a sandbox escape could still be possible due to misconfiguration or bugs in AppArmor. For defense in depth, we're setting up a dedicated `codejail service <https://github.com/openedx/codejail-service>`__ that will perform code execution for edxapp and which will allow further isolation.
+
+The default edxapp codejail defaults to unsafe, direct execution of Python code, and this remains true in devstack. We don't even have a way to run on-host codejail securely in devstack. In constrast, the codejail service refuses to run if codejail has not been configured properly, and we've included a way to run it in devstack.
+
+Configuration
+*************
+
+In order to run the codejail devstack component:
+
+1. Install AppArmor: ``sudo apt install apparmor``
+2. Add the codejail AppArmor profile to your OS, or update it: ``sudo apparmor_parser --replace -W codejail.profile``
+3. Configure LMS and CMS to use the codejail-service by uncommenting ``# ENABLE_CODEJAIL_REST_SERVICE = True`` in ``py_configuration_files/{lms,cms}.py``
+4. Run ``make codejail-up``
+
+The service does not need any provisioning, and does not have dependencies.
+
+Development
+***********
+
+Changes to the AppArmor profile must be coordinated with changes to the Dockerfile, as they need to agree on filesystem paths.
+
+Any time you update the profile, you'll need to re-run the command to apply the profile.
+
+The profile file contains the directive ``profile codejail_service``. That defines the name of the profile when it is installed into the kernel. In order to change that name, you must first remove the profile **under the old name**, then install a new profile under the new name. To remove a profile, use the ``--remove`` action instead of the ``-replace`` action: : ``sudo apparmor_parser --remove -W codejail.profile``
+
+The profile name must also agree with the relevant ``security_opt`` line in devstack's ``docker-compose.yml``.
+
+Debugging
+*********
+
+To check whether the profile has been applied, run ``sudo aa-status | grep codejail``. This won't tell you if the profile is out of date, but it will tell you if you have *some* version of it installed.
+
+If you need to debug the confinement, either because it is restricting too much or too little, a good strategy is to run ``tail -F /var/log/kern.log | grep codejail`` and watch for ``DENIED`` lines. You should expect to see several appear during service startup, as the service is designed to probe the confinement as part of its initial healthcheck.

docs/index.rst

@@ -27,3 +27,4 @@ Contents
    troubleshoot_general_tips
    manual_upgrades
    advanced_configuration
+   codejail

py_configuration_files/cms.py

@@ -312,6 +312,14 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
 xblock_duplicated_event_setting = EVENT_BUS_PRODUCER_CONFIG['org.openedx.content_authoring.xblock.duplicated.v1']
 xblock_duplicated_event_setting['course-authoring-xblock-lifecycle']['enabled'] = True
 
+############################ Codejail ############################
+
+# Disabled by default since codejail service needs to be configured
+# and started separately. See docs/codejail.rst for details.
+
+# ENABLE_CODEJAIL_REST_SERVICE = True
+CODE_JAIL_REST_SERVICE_HOST = "http://edx.devstack.codejail:8080"
+
 
 ################# New settings must go ABOVE this line #################
 ########################################################################

py_configuration_files/codejail.py

@@ -1,3 +1,28 @@
 """Settings for devstack use."""
 
 from codejail_service.settings.local import *  # pylint: disable=wildcard-import
+
+ALLOWED_HOSTS = [
+    # When called from outside of docker's network (dev's terminal)
+    'localhost',
+    # When called from another container (lms, cms)
+    'edx.devstack.codejail',
+]
+
+CODEJAIL_ENABLED = True
+
+CODE_JAIL = {
+    # These values are coordinated with the Dockerfile and AppArmor profile
+    'python_bin': '/sandbox/venv/bin/python',
+    'user': 'sandbox',
+
+    # Configurable limits.
+    'limits': {
+        # CPU-seconds
+        'CPU': 3,
+        # 100 MiB memory
+        'VMEM': 100 * 1024 * 1024,
+        # Clock seconds
+        'REALTIME': 3,
+    },
+}

py_configuration_files/lms.py

@@ -554,6 +554,13 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
     'http://localhost:1996',  # frontend-app-learner-dashboard
 ]
 
+############################ Codejail ############################
+
+# Disabled by default since codejail service needs to be configured
+# and started separately. See docs/codejail.rst for details.
+
+# ENABLE_CODEJAIL_REST_SERVICE = True
+CODE_JAIL_REST_SERVICE_HOST = "http://edx.devstack.codejail:8080"
 
 ################# New settings must go ABOVE this line #################
 ########################################################################