Merge pull request #313 from mishaschwartz/v2.1.0

mishaschwartz · web-flow · commit 799c5b168816 · 2021-11-15T16:03:36.000-05:00
v2.1.0
diff --git a/Changelog.md b/Changelog.md
@@ -1,7 +1,8 @@
 # CHANGELOG
 All notable changes to this project will be documented here.
 
-## [unreleased]
+## [v2.1.0]
+- Add R tester (#310)
 
 ## [v2.0.2]
 - Keep result object alive for longer than the default 500 seconds (#302)
diff --git a/README.md b/README.md
@@ -138,6 +138,8 @@ The autotester currently supports testers for the following languages and testin
     - [Jupyter Notebook](https://jupyter.org/)
 - `racket`
     - [RackUnit](https://docs.racket-lang.org/rackunit/)
+- `R`
+    - [TestThat](https://testthat.r-lib.org/)
 - `custom`
     - see more information [here](#the-custom-tester)
 
@@ -158,6 +160,8 @@ Installing each tester will also install the following additional packages (syst
     - none
 - `racket`
     - racket
+- `R`
+    - R
 - `custom`
     - none
 
diff --git a/client/autotest_client/__init__.py b/client/autotest_client/__init__.py
@@ -308,7 +308,7 @@ def get_statuses(settings_id, **_kw):
 
 @app.route("/settings/<settings_id>/tests/cancel", methods=["DELETE"])
 @authorize
-def cancel_tests(settings_id):
+def cancel_tests(settings_id, **_kw):
     test_ids = request.json["test_ids"]
     result = {}
     for id_, job in zip(test_ids, _get_jobs(test_ids, settings_id)):
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -8,7 +8,8 @@ services:
       args:
         UBUNTU_VERSION: '18.04'
         LOGIN_USER: 'docker'
-    image: markus-autotest-server-dev:1.0.0
+        WORKSPACE: '/home/docker/.autotesting'
+    image: markus-autotest-server-dev:1.0.1
     volumes:
       - ./server:/app:cached
       - workspace:/home/docker/.autotesting:rw
diff --git a/server/.dockerfiles/Dockerfile b/server/.dockerfiles/Dockerfile
@@ -5,6 +5,7 @@ FROM ubuntu:$UBUNTU_VERSION as base
 ENV DEBIAN_FRONTEND=noninteractive 
 
 ARG LOGIN_USER
+ARG WORKSPACE
 
 RUN apt-get update -y && \
     apt-get -y install software-properties-common && \
@@ -38,6 +39,8 @@ RUN python3.9 -m venv /markus_venv && \
     find /app/autotest_server/testers -name requirements.system -exec {} \; && \
     rm -rf /app/*
 
+RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE}
+
 WORKDIR /home/${LOGIN_USER}
 
 USER ${LOGIN_USER}
diff --git a/server/autotest_server/__init__.py b/server/autotest_server/__init__.py
@@ -82,7 +82,7 @@ def _kill_user_processes(test_username: str) -> None:
     subprocess.run(kill_cmd, shell=True)
 
 
-def _create_test_script_command(env_dir: str, tester_type: str) -> str:
+def _create_test_script_command(tester_type: str) -> str:
     """
     Return string representing a command line command to
     run tests.
@@ -95,9 +95,8 @@ def _create_test_script_command(env_dir: str, tester_type: str) -> str:
         "from testers.specs import TestSpecs",
         f"Tester(specs=TestSpecs.from_json(sys.stdin.read())).run()",
     ]
-    python_ex = os.path.join(os.path.join(TEST_SCRIPT_DIR, env_dir), "bin", "python")
     python_str = "; ".join(python_lines)
-    return f"{python_ex} -c '{python_str}'"
+    return f"\"${{PYTHON}}\" -c '{python_str}'"
 
 
 def get_available_port(min_, max_, host: str = "localhost") -> str:
@@ -174,9 +173,8 @@ def _run_test_specs(
 
     for settings in test_settings["testers"]:
         tester_type = settings["tester_type"]
-        env_dir = settings.get("_env_loc")
 
-        cmd_str = _create_test_script_command(env_dir, tester_type)
+        cmd_str = _create_test_script_command(tester_type)
         args = cmd.format(cmd_str)
 
         for test_data in settings["test_data"]:
@@ -198,7 +196,7 @@ def _run_test_specs(
                         stdin=subprocess.PIPE,
                         preexec_fn=set_rlimits_before_test,
                         universal_newlines=True,
-                        env={**os.environ, **env_vars},
+                        env={**os.environ, **env_vars, **settings["_env"]},
                     )
                     try:
                         settings_json = json.dumps({**settings, "test_data": test_data})
@@ -356,20 +354,15 @@ def update_test_settings(user, settings_id, test_settings, file_url):
             tester_type = tester_settings["tester_type"]
             if tester_type not in installed_testers:
                 raise Exception(f"tester {tester_type} is not installed")
+            env_dir = os.path.join(settings_dir, f"{tester_type}_{i}")
             tester_install = importlib.import_module(f"autotest_server.testers.{tester_type}.setup")
-            if tester_settings.get("env_data"):
-                env_dir = os.path.join(settings_dir, f"{tester_type}_{i}")
-                tester_settings["_env_loc"] = env_dir
-                try:
-                    tester_install.create_environment(tester_settings)
-                except Exception as e:
-                    raise Exception(f"create tester environment failed:\n{e}") from e
-            else:
-                default_env = os.path.join(TEST_SCRIPT_DIR, DEFAULT_ENV_DIR)
-                if not os.path.isdir(default_env):
-                    subprocess.run([sys.executable, "-m", "venv", default_env], check=True)
-
-                tester_settings["_env_loc"] = default_env
+            default_env = os.path.join(TEST_SCRIPT_DIR, DEFAULT_ENV_DIR)
+            if not os.path.isdir(default_env):
+                subprocess.run([sys.executable, "-m", "venv", default_env], check=True)
+            try:
+                tester_settings["_env"] = tester_install.create_environment(tester_settings, env_dir, default_env)
+            except Exception as e:
+                raise Exception(f"create tester environment failed:\n{e}") from e
             test_settings["testers"][i] = tester_settings
         test_settings["_files"] = files_dir
         test_settings.pop("_error", None)
diff --git a/server/autotest_server/testers/__init__.py b/server/autotest_server/testers/__init__.py
@@ -1,6 +1,6 @@
 import os
 
-_TESTERS = ("custom", "haskell", "java", "jupyter", "py", "pyta", "racket")
+_TESTERS = ("custom", "haskell", "java", "jupyter", "py", "pyta", "r", "racket")
 
 
 def install(testers=_TESTERS):
diff --git a/server/autotest_server/testers/custom/setup.py b/server/autotest_server/testers/custom/setup.py
@@ -2,8 +2,8 @@
 import json
 
 
-def create_environment(*_args, **_kwargs):
-    """no op"""
+def create_environment(_settings, _env_dir, default_env_dir):
+    return {"PYTHON": os.path.join(default_env_dir, 'bin', 'python3')}
 
 
 def install():
diff --git a/server/autotest_server/testers/haskell/setup.py b/server/autotest_server/testers/haskell/setup.py
@@ -3,8 +3,8 @@
 import subprocess
 
 
-def create_environment(*_args, **_kwargs):
-    """no op"""
+def create_environment(_settings, _env_dir, default_env_dir):
+    return {"PYTHON": os.path.join(default_env_dir, 'bin', 'python3')}
 
 
 def install():
diff --git a/server/autotest_server/testers/java/setup.py b/server/autotest_server/testers/java/setup.py
@@ -4,8 +4,8 @@
 import requests
 
 
-def create_environment(*_args, **_kwargs):
-    """no op"""
+def create_environment(_settings, _env_dir, default_env_dir):
+    return {"PYTHON": os.path.join(default_env_dir, 'bin', 'python3')}
 
 
 def install():
diff --git a/server/autotest_server/testers/jupyter/settings_schema.json b/server/autotest_server/testers/jupyter/settings_schema.json
@@ -55,7 +55,7 @@
                 "test_merge": {
                   "title": "Test that files can be merged",
                   "type": "boolean",
-                  "default": "false"
+                  "default": false
                 }
               }
             },
diff --git a/server/autotest_server/testers/jupyter/setup.py b/server/autotest_server/testers/jupyter/setup.py
@@ -4,16 +4,15 @@
 import subprocess
 
 
-def create_environment(settings_):
-    env_loc = settings_["_env_loc"]
+def create_environment(settings_, env_dir, _default_env_dir):
     env_data = settings_.get("env_data", {})
     python_version = env_data.get("python_version", "3")
     pip_requirements = ["wheel"] + env_data.get("pip_requirements", "").split()
     requirements = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.txt")
-    pip = os.path.join(env_loc, "bin", "pip")
-    subprocess.run([f"python{python_version}", "-m", "venv", "--clear", env_loc], check=True)
+    pip = os.path.join(env_dir, "bin", "pip")
+    subprocess.run([f"python{python_version}", "-m", "venv", "--clear", env_dir], check=True)
     subprocess.run([pip, "install", "-r", requirements, *pip_requirements], check=True)
-    return True
+    return {"PYTHON": os.path.join(env_dir, 'bin', 'python3')}
 
 
 def settings():
diff --git a/server/autotest_server/testers/py/setup.py b/server/autotest_server/testers/py/setup.py
@@ -4,16 +4,15 @@
 import subprocess
 
 
-def create_environment(settings_):
-    env_loc = settings_["_env_loc"]
+def create_environment(settings_, env_dir, _default_env_dir):
     env_data = settings_.get("env_data", {})
     python_version = env_data.get("python_version", "3")
     pip_requirements = ["wheel"] + env_data.get("pip_requirements", "").split()
     requirements = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.txt")
-    pip = os.path.join(env_loc, "bin", "pip")
-    subprocess.run([f"python{python_version}", "-m", "venv", "--clear", env_loc], check=True)
+    pip = os.path.join(env_dir, "bin", "pip")
+    subprocess.run([f"python{python_version}", "-m", "venv", "--clear", env_dir], check=True)
     subprocess.run([pip, "install", "-r", requirements, *pip_requirements], check=True)
-    return True
+    return {"PYTHON": os.path.join(env_dir, 'bin', 'python3')}
 
 
 def settings():
diff --git a/server/autotest_server/testers/pyta/setup.py b/server/autotest_server/testers/pyta/setup.py
@@ -4,16 +4,15 @@
 import subprocess
 
 
-def create_environment(settings_):
-    env_loc = settings_["_env_loc"]
+def create_environment(settings_, env_dir, _default_env_dir):
     env_data = settings_.get("env_data", {})
     python_version = env_data.get("python_version", "3")
     pip_requirements = ["wheel"] + env_data.get("pip_requirements", "").split()
     requirements = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.txt")
-    pip = os.path.join(env_loc, "bin", "pip")
-    subprocess.run([f"python{python_version}", "-m", "venv", "--clear", env_loc], check=True)
+    pip = os.path.join(env_dir, "bin", "pip")
+    subprocess.run([f"python{python_version}", "-m", "venv", "--clear", env_dir], check=True)
     subprocess.run([pip, "install", "-r", requirements, *pip_requirements], check=True)
-    return True
+    return {"PYTHON": os.path.join(env_dir, 'bin', 'python3')}
 
 
 def settings():
diff --git a/server/autotest_server/testers/r/__init__.py b/server/autotest_server/testers/r/__init__.py
diff --git a/server/autotest_server/testers/r/lib/r_tester.R b/server/autotest_server/testers/r/lib/r_tester.R
@@ -0,0 +1,15 @@
+sink(file="/dev/null")
+library(testthat)
+library(rjson)
+args <- commandArgs(TRUE)
+test_results <- test_file(args[1], reporter = ListReporter)
+for (i in 1:length(test_results)) {
+  for (j in 1:length(test_results[[i]]$results)) {
+    result <- test_results[[i]]$results[[j]]
+    expectation <- class(result)[1]
+    test_results[[i]]$results[[j]]$type <- expectation
+  }
+}
+json <- toJSON(test_results)
+sink()
+cat(json)
diff --git a/server/autotest_server/testers/r/r_tester.py b/server/autotest_server/testers/r/r_tester.py
@@ -0,0 +1,99 @@
+import subprocess
+import os
+import json
+from typing import Dict, Optional, IO, Type, List, Union
+
+from ..tester import Tester, Test, TestError
+from ..specs import TestSpecs
+
+
+class RTest(Test):
+    def __init__(
+        self,
+        tester: "RTester",
+        test_file: str,
+        result: Dict,
+        feedback_open: Optional[IO] = None,
+    ) -> None:
+        """
+        Initialize a R test created by tester.
+
+        The result was created after running the tests in test_file and test feedback
+        will be written to feedback_open.
+        """
+        self._test_name = ':'.join(info for info in [test_file, result.get('context'), result['test']] if info)
+        self.result = result["results"]
+        super().__init__(tester, feedback_open)
+        self.points_total = 0
+
+    @property
+    def test_name(self):
+        return self._test_name
+
+    @Test.run_decorator
+    def run(self):
+        messages = []
+        successes = 0
+        for result in self.result:
+            messages.append(result["message"])
+            if result["type"] == "expectation_success":
+                self.points_total += 1
+                successes += 1
+            elif result["type"] == "expectation_failure":
+                self.points_total += 1
+
+        message = "\n\n".join(messages)
+        if successes == self.points_total:
+            return self.passed(message=message)
+        elif successes > 0:
+            return self.partially_passed(points_earned=successes, message=message)
+        else:
+            return self.failed(message=message)
+
+
+class RTester(Tester):
+    def __init__(
+        self,
+        specs: TestSpecs,
+        test_class: Type[RTest] = RTest,
+    ) -> None:
+        """
+        Initialize a R tester using the specifications in specs.
+
+        This tester will create tests of type test_class.
+        """
+        super().__init__(specs, test_class)
+
+    def run_r_tests(self) -> Dict[str, List[Dict[str, Union[int, str]]]]:
+        """
+        Return test results for each test file. Results contain a list of parsed test results.
+        """
+        results = {}
+        r_tester = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib', 'r_tester.R')
+        for test_file in self.specs["test_data", "script_files"]:
+            proc = subprocess.run(['Rscript', r_tester, test_file],
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE,
+                                  universal_newlines=True)
+            if not results.get(test_file):
+                results[test_file] = []
+            if proc.returncode == 0:
+                results[test_file].extend(json.loads(proc.stdout))
+            else:
+                raise TestError(proc.stderr)
+        return results
+
+    @Tester.run_decorator
+    def run(self) -> None:
+        """
+        Runs all tests in this tester.
+        """
+        try:
+            results = self.run_r_tests()
+        except subprocess.CalledProcessError as e:
+            raise TestError(e.stderr) from e
+        with self.open_feedback() as feedback_open:
+            for test_file, result in results.items():
+                for res in result:
+                    test = self.test_class(self, test_file, res, feedback_open)
+                    print(test.run(), flush=True)
diff --git a/server/autotest_server/testers/r/requirements.system b/server/autotest_server/testers/r/requirements.system
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+if ! dpkg -l r-base &> /dev/null; then
+  apt-get -y update
+  DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' r-base
+fi
diff --git a/server/autotest_server/testers/r/settings_schema.json b/server/autotest_server/testers/r/settings_schema.json
diff --git a/server/autotest_server/testers/r/setup.py b/server/autotest_server/testers/r/setup.py
diff --git a/server/autotest_server/testers/racket/setup.py b/server/autotest_server/testers/racket/setup.py
