Remove i18n from judge

README.md

@@ -91,3 +91,11 @@ Since the tests are mostly just jUnit tests, IntelliJ and other IDE's provide su
 Per exercise, create a new project. The `config.json` file should be in the project root. Mark (or create) the `workdir`, `evaluation` and `solution` directories as "sources root". Add jUnit 4 as a dependency of the project.
 
 If your exercises use some Dodona-specific features, such as the `TabTitle` or the `AssertionStubber`, add the Judge as a dependency. Opening this repository as an IntelliJ project should allow you to create a JAR.
+
+## Developing the judge
+
+While developing, you can use `./integration-tests/run` to validate whether the judge is working properly. It checks the judge's output JSON against previous results (stored as `result.json`) and monitors whether something has changed.
+
+You can also use `./integration-tests/run <path-to-repo>` to run the same checks on an exercise repository, which is useful to validate a larger set op exercises.
+
+With `---overwrite`, you can overwrite previous `result.json` files with the current output, and with `-v` you can check for judge error output and view the exact changes when output differs.

integration-tests/correct/config.json

@@ -0,0 +1 @@
+../with-package/config.json

integration-tests/correct/evaluation

@@ -0,0 +1 @@
+../with-package/evaluation/

integration-tests/i18n-correct-nl/result.json → integration-tests/correct/result.json

@@ -13,12 +13,12 @@
   "command": "start-tab",
   "hidden": false,
   "permission": "student",
-  "title": "Vertaalde titel"
+  "title": "Tab title"
 }
 {
   "command": "start-context",
   "description": {
-    "description": "Vertaalde Test",
+    "description": "Test description",
     "format": "code"
   }
 }

integration-tests/forgotten-semicolon/config.json

@@ -0,0 +1 @@
+../with-package/config.json

integration-tests/forgotten-semicolon/evaluation

@@ -0,0 +1 @@
+../with-package/evaluation/

integration-tests/run

@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 set -e
+shopt -s globstar
 
 # Default values
 target_dir="integration-tests"
 overwrite=0
+verbose=0
 
 # Parse arguments
 while [ "$#" -gt 0 ]; do
     case "$1" in
         --overwrite)
             overwrite=1
             ;;
+        -v|--verbose)
+            verbose=1
+            ;;
         *)
             target_dir="$1"
             ;;
@@ -25,8 +30,13 @@ description_changed=0
 minor_change=0
 major_change=0
 
+err_out="/dev/null"
+if [ $verbose -gt 0 ]; then
+    err_out="/dev/stderr"
+fi
+
 # Find all config.json files in target directory
-for config in "$target_dir"/**/config.json; do
+for config in $target_dir/**/config.json; do
     test_dir="$(dirname "$config")"
 
     # Extract config values
@@ -69,7 +79,7 @@ for config in "$target_dir"/**/config.json; do
           , "memory_limit": 1000000000
           , "source": "'"$judge/$source"'"
           }' \
-          | (timeout -k 10s 60s "$judge/run" 2> /dev/null) \
+          | (timeout -k 10s 60s "$judge/run" 2> "$err_out") \
           | jq --sort-keys 'if(.command == "append-message")
                 then .message.description |= gsub("\n at [^\n]+\\([^)]+\\)"; "")
                 else .
@@ -93,10 +103,11 @@ for config in "$target_dir"/**/config.json; do
 
 
         # First check if files are exact using diff, then check if they differ except '.description', then check they differ in accepted or failed
-        if diff "$result_file" <(echo "$output") > /dev/null; then
+
+        if diff -c "$result_file" <(echo "$output") > "$err_out"; then
             echo "[EXACT MATCH]"
             exact_match=$((exact_match + 1))
-        elif diff <(jq "del(.description)" "$result_file") <(echo "$output" | jq "del(.description)") > /dev/null; then
+        elif diff -c <(jq "del(.description)" "$result_file") <(echo "$output" | jq "del(.description)") > "$err_out"; then
             echo "[DESCRIPTION CHANGED]"
             description_changed=$((description_changed + 1))
         elif [ "$accepted_output" -eq "$accepted_result" ] && [ "$failed_output" -eq "$failed_result" ]; then

integration-tests/stackoverflow/config.json

@@ -0,0 +1 @@
+../with-package/config.json

integration-tests/stackoverflow/evaluation

@@ -0,0 +1 @@
+../with-package/evaluation/

integration-tests/i18n-stackoverflow/result.json → integration-tests/stackoverflow/result.json

@@ -13,12 +13,12 @@
   "command": "start-tab",
   "hidden": false,
   "permission": "student",
-  "title": "Translated title"
+  "title": "Tab title"
 }
 {
   "command": "start-context",
   "description": {
-    "description": "Translated Test",
+    "description": "Test description",
     "format": "code"
   }
 }