Remove i18n from judge

README.md

@@ -91,3 +91,11 @@ Since the tests are mostly just jUnit tests, IntelliJ and other IDE's provide su
 Per exercise, create a new project. The `config.json` file should be in the project root. Mark (or create) the `workdir`, `evaluation` and `solution` directories as "sources root". Add jUnit 4 as a dependency of the project.
 
 If your exercises use some Dodona-specific features, such as the `TabTitle` or the `AssertionStubber`, add the Judge as a dependency. Opening this repository as an IntelliJ project should allow you to create a JAR.
+
+## Developing the judge
+
+While developing, you can use `./integration-tests/run` to validate whether the judge is working properly. It checks the judge's output JSON against previous results (stored as `result.json`) and monitors whether something has changed.
+
+You can also use `./integration-tests/run <path-to-repo>` to run the same checks on an exercise repository, which is useful to validate a larger set op exercises.
+
+With `---overwrite`, you can overwrite previous `result.json` files with the current output, and with `-v` you can check for judge error output and view the exact changes when output differs.

integration-tests/correct/config.json

@@ -0,0 +1 @@
+../with-package/config.json

integration-tests/correct/evaluation

@@ -0,0 +1 @@
+../with-package/evaluation/

integration-tests/i18n-correct-nl/result.json → integration-tests/correct/result.json

@@ -13,12 +13,12 @@
   "command": "start-tab",
   "hidden": false,
   "permission": "student",
-  "title": "Vertaalde titel"
+  "title": "Tab title"
 }
 {
   "command": "start-context",
   "description": {
-    "description": "Vertaalde Test",
+    "description": "Test description",
     "format": "code"
   }
 }

integration-tests/forgotten-semicolon/config.json

@@ -0,0 +1 @@
+../with-package/config.json

integration-tests/forgotten-semicolon/evaluation

@@ -0,0 +1 @@
+../with-package/evaluation/

integration-tests/run

@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 set -e
+shopt -s globstar
 
 # Default values
 target_dir="integration-tests"
 overwrite=0
+err_out="/dev/null"
 
 # Parse arguments
 while [ "$#" -gt 0 ]; do
     case "$1" in
         --overwrite)
             overwrite=1
             ;;
+        -v|--verbose)
+            err_out="/dev/stderr"
+            ;;
         *)
             target_dir="$1"
             ;;
@@ -26,7 +31,7 @@ minor_change=0
 major_change=0
 
 # Find all config.json files in target directory
-for config in "$target_dir"/**/config.json; do
+for config in $target_dir/**/config.json; do
     test_dir="$(dirname "$config")"
 
     # Extract config values
@@ -69,7 +74,7 @@ for config in "$target_dir"/**/config.json; do
           , "memory_limit": 1000000000
           , "source": "'"$judge/$source"'"
           }' \
-          | (timeout -k 10s 60s "$judge/run" 2> /dev/null) \
+          | (timeout -k 10s 60s "$judge/run" 2> "$err_out") \
           | jq --sort-keys 'if(.command == "append-message")
                 then .message.description |= gsub("\n at [^\n]+\\([^)]+\\)"; "")
                 else .
@@ -80,30 +85,30 @@ for config in "$target_dir"/**/config.json; do
     rm -r "$workdir"
 
     # Count accepted and failed in output
-    accepted_output="$(echo "$output" | jq -s 'map(select(.accepted == true)) | length')"
-    failed_output="$(echo "$output" | jq -s 'map(select(.accepted == false)) | length')"
-    echo -en "$accepted_output accepted, $failed_output failed\t"
+    accepted_actual="$(echo "$output" | jq -s 'map(select(.accepted == true)) | length')"
+    failed_actual="$(echo "$output" | jq -s 'map(select(.accepted == false)) | length')"
+    echo -en "$accepted_actual accepted, $failed_actual failed\t"
 
 
 
     if [ -f "$result_file" ]; then
         # Count accepted and failed in result
-        accepted_result="$(jq -s 'map(select(.accepted == true)) | length' "$result_file" )"
-        failed_result="$(jq -s 'map(select(.accepted == true)) | length' "$result_file" )"
-
+        accepted_expected="$(jq -s 'map(select(.accepted == true)) | length' "$result_file" )"
+        failed_expected="$(jq -s 'map(select(.accepted == false)) | length' "$result_file" )"
 
         # First check if files are exact using diff, then check if they differ except '.description', then check they differ in accepted or failed
-        if diff "$result_file" <(echo "$output") > /dev/null; then
+
+        if diff -c "$result_file" <(echo "$output") > "$err_out"; then
             echo "[EXACT MATCH]"
             exact_match=$((exact_match + 1))
-        elif diff <(jq "del(.description)" "$result_file") <(echo "$output" | jq "del(.description)") > /dev/null; then
+        elif diff -c <(jq "del(.description)" "$result_file") <(echo "$output" | jq "del(.description)") > /dev/null; then
             echo "[DESCRIPTION CHANGED]"
             description_changed=$((description_changed + 1))
-        elif [ "$accepted_output" -eq "$accepted_result" ] && [ "$failed_output" -eq "$failed_result" ]; then
+        elif [ "$accepted_actual" -eq "$accepted_expected" ] && [ "$failed_actual" -eq "$failed_expected" ]; then
             echo "[MINOR CHANGE] (accepted/failed unchanged)"
             minor_change=$((minor_change + 1))
         else
-            echo "[MAJOR CHANGE] (accepted/failed changed, was $accepted_result/$failed_result)"
+            echo "[MAJOR CHANGE] (accepted/failed changed, expected $accepted_expected/$failed_expected, actual $accepted_actual/$failed_actual)"
             major_change=$((major_change + 1))
         fi
     fi

integration-tests/stackoverflow/config.json

@@ -0,0 +1 @@
+../with-package/config.json

integration-tests/stackoverflow/evaluation

@@ -0,0 +1 @@
+../with-package/evaluation/

integration-tests/i18n-stackoverflow/result.json → integration-tests/stackoverflow/result.json

@@ -13,12 +13,12 @@
   "command": "start-tab",
   "hidden": false,
   "permission": "student",
-  "title": "Translated title"
+  "title": "Tab title"
 }
 {
   "command": "start-context",
   "description": {
-    "description": "Translated Test",
+    "description": "Test description",
     "format": "code"
   }
 }