diff --git a/README.md b/README.md index f3dd869..b9f4129 100644 --- a/README.md +++ b/README.md @@ -91,3 +91,11 @@ Since the tests are mostly just jUnit tests, IntelliJ and other IDE's provide su Per exercise, create a new project. The `config.json` file should be in the project root. Mark (or create) the `workdir`, `evaluation` and `solution` directories as "sources root". Add jUnit 4 as a dependency of the project. If your exercises use some Dodona-specific features, such as the `TabTitle` or the `AssertionStubber`, add the Judge as a dependency. Opening this repository as an IntelliJ project should allow you to create a JAR. + +## Developing the judge + +While developing, you can use `./integration-tests/run` to validate whether the judge is working properly. It checks the judge's output JSON against previous results (stored as `result.json`) and monitors whether something has changed. + +You can also use `./integration-tests/run ` to run the same checks on an exercise repository, which is useful to validate a larger set op exercises. + +With `---overwrite`, you can overwrite previous `result.json` files with the current output, and with `-v` you can check for judge error output and view the exact changes when output differs. \ No newline at end of file diff --git a/i18n/en b/i18n/en deleted file mode 100644 index 2a4567c..0000000 --- a/i18n/en +++ /dev/null @@ -1,18 +0,0 @@ -i18n_error='error' -i18n_errors='errors' -i18n_warning='warning' -i18n_warnings='warnings' -i18n_wrong_class_name='Your class should be called "%s".' -i18n_forgot_import='You are trying to use an unknown class "%s". Might you have forgotten an import?' -i18n_assign_to_final='Variables declared to be "final" can be assigned to only once. Remove the keyword "final" from the variable "%s" to solve this.' -i18n_raw_type='Generic classes should be passed a type.' -i18n_compilation_error='Compilation error' -i18n_compilation_warning='Compilation warning' -i18n_and='and' -i18n_workdir_compilation_message='Something went wrong while compiling the base code for this exercise. Contact your teacher.' -i18n_workdir_compilation_summary='Error in base code' -i18n_user_compilation_message='Your code cannot be compiled and therefore not tested. The compiler reported %s.' -i18n_class_not_submitted='Your submission does not contain a class "%s", so your code could not be tested.' -i18n_default_package='Are you sure you put your submitted class in the default package? Your class will be put in the default package if you remove the package statement.' -i18n_test_compilation_message='Something went wrong while compiling the tests for this exercise.' -i18n_test_compilation_summary='Error in the tests' diff --git a/i18n/nl b/i18n/nl deleted file mode 100644 index 92e20ba..0000000 --- a/i18n/nl +++ /dev/null @@ -1,18 +0,0 @@ -i18n_error='fout' -i18n_errors='fouten' -i18n_warning='waarschuwing' -i18n_warnings='waarschuwingen' -i18n_wrong_class_name='De naam van je klasse hoort "%s" te zijn.' -i18n_forgot_import='Je probeert een onbekende klasse "%s" te gebruiken. Mogelijks ben je de noodzakelijke import vergeten.' -i18n_assign_to_final='Variabelen die als "final" zijn gedeclareerd, kunnen niet meer worden aangepast eens ze een waarde hebben. Verwijder het keyword "final" bij de variabele "%s" om dit op te lossen.' -i18n_raw_type='Het is aangeraden een type mee te geven aan generieke klassen.' -i18n_compilation_error='Compilatiefout' -i18n_compilation_warning='Compilatiewaarschuwing' -i18n_and='en' -i18n_workdir_compilation_message='Er ging iets mis tijdens het compileren van de startcode voor deze oefening. Contacteer je lesgever.' -i18n_workdir_compilation_summary='Fout in de startcode' -i18n_user_compilation_message='Je code kon niet worden gecompileerd en bijgevolg niet worden getest. De compiler rapporteerde %s.' -i18n_class_not_submitted='Je diende geen %s-klasse in, waardoor de testen niet uitgevoerd kunnen worden.' -i18n_default_package='Ben je zeker dat je de ingediende klasse in het default package plaatste? Je klasse komt in het default package terecht als je jouw package statement verwijdert.' -i18n_test_compilation_message='Er ging iets mis tijdens het compileren van de testen voor deze oefening.' -i18n_test_compilation_summary='Fout in de testen' diff --git a/integration-tests/correct/config.json b/integration-tests/correct/config.json new file mode 120000 index 0000000..6f8e641 --- /dev/null +++ b/integration-tests/correct/config.json @@ -0,0 +1 @@ +../with-package/config.json \ No newline at end of file diff --git a/integration-tests/correct/evaluation b/integration-tests/correct/evaluation new file mode 120000 index 0000000..99f1c59 --- /dev/null +++ b/integration-tests/correct/evaluation @@ -0,0 +1 @@ +../with-package/evaluation/ \ No newline at end of file diff --git a/integration-tests/i18n-correct-nl/result.json b/integration-tests/correct/result.json similarity index 85% rename from integration-tests/i18n-correct-nl/result.json rename to integration-tests/correct/result.json index 4a71397..b5e071e 100644 --- a/integration-tests/i18n-correct-nl/result.json +++ b/integration-tests/correct/result.json @@ -13,12 +13,12 @@ "command": "start-tab", "hidden": false, "permission": "student", - "title": "Vertaalde titel" + "title": "Tab title" } { "command": "start-context", "description": { - "description": "Vertaalde Test", + "description": "Test description", "format": "code" } } diff --git a/integration-tests/i18n-correct-en/submission.java b/integration-tests/correct/submission.java similarity index 100% rename from integration-tests/i18n-correct-en/submission.java rename to integration-tests/correct/submission.java diff --git a/integration-tests/forgotten-semicolon/config.json b/integration-tests/forgotten-semicolon/config.json new file mode 120000 index 0000000..6f8e641 --- /dev/null +++ b/integration-tests/forgotten-semicolon/config.json @@ -0,0 +1 @@ +../with-package/config.json \ No newline at end of file diff --git a/integration-tests/forgotten-semicolon/evaluation b/integration-tests/forgotten-semicolon/evaluation new file mode 120000 index 0000000..99f1c59 --- /dev/null +++ b/integration-tests/forgotten-semicolon/evaluation @@ -0,0 +1 @@ +../with-package/evaluation/ \ No newline at end of file diff --git a/integration-tests/i18n-forgotten-semicolon/result.json b/integration-tests/forgotten-semicolon/result.json similarity index 100% rename from integration-tests/i18n-forgotten-semicolon/result.json rename to integration-tests/forgotten-semicolon/result.json diff --git a/integration-tests/i18n-forgotten-semicolon/submission.java b/integration-tests/forgotten-semicolon/submission.java similarity index 100% rename from integration-tests/i18n-forgotten-semicolon/submission.java rename to integration-tests/forgotten-semicolon/submission.java diff --git a/integration-tests/i18n-correct-en/config.json b/integration-tests/i18n-correct-en/config.json deleted file mode 120000 index 77b3a67..0000000 --- a/integration-tests/i18n-correct-en/config.json +++ /dev/null @@ -1 +0,0 @@ -../i18n-with-package-en/config.json \ No newline at end of file diff --git a/integration-tests/i18n-correct-en/evaluation b/integration-tests/i18n-correct-en/evaluation deleted file mode 120000 index 558ef2e..0000000 --- a/integration-tests/i18n-correct-en/evaluation +++ /dev/null @@ -1 +0,0 @@ -../i18n-with-package-en/evaluation \ No newline at end of file diff --git a/integration-tests/i18n-correct-en/result.json b/integration-tests/i18n-correct-en/result.json deleted file mode 100644 index c90e4a3..0000000 --- a/integration-tests/i18n-correct-en/result.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "command": "start-judgement" -} -{ - "command": "start-tab", - "hidden": true, - "title": "Compiler" -} -{ - "command": "close-tab" -} -{ - "command": "start-tab", - "hidden": false, - "permission": "student", - "title": "Translated title" -} -{ - "command": "start-context", - "description": { - "description": "Translated Test", - "format": "code" - } -} -{ - "accepted": true, - "command": "close-context" -} -{ - "command": "close-tab" -} -{ - "command": "close-judgement" -} diff --git a/integration-tests/i18n-correct-nl/config.json b/integration-tests/i18n-correct-nl/config.json deleted file mode 100644 index 817a980..0000000 --- a/integration-tests/i18n-correct-nl/config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "filename": "Translated.java", - "natural_language": "nl" -} \ No newline at end of file diff --git a/integration-tests/i18n-correct-nl/evaluation b/integration-tests/i18n-correct-nl/evaluation deleted file mode 120000 index 558ef2e..0000000 --- a/integration-tests/i18n-correct-nl/evaluation +++ /dev/null @@ -1 +0,0 @@ -../i18n-with-package-en/evaluation \ No newline at end of file diff --git a/integration-tests/i18n-correct-nl/submission.java b/integration-tests/i18n-correct-nl/submission.java deleted file mode 100644 index fc744e9..0000000 --- a/integration-tests/i18n-correct-nl/submission.java +++ /dev/null @@ -1,7 +0,0 @@ -public class Translated { - - public String getLanguage() { - return "nederlands"; - } - -} diff --git a/integration-tests/i18n-forgotten-semicolon/config.json b/integration-tests/i18n-forgotten-semicolon/config.json deleted file mode 120000 index 77b3a67..0000000 --- a/integration-tests/i18n-forgotten-semicolon/config.json +++ /dev/null @@ -1 +0,0 @@ -../i18n-with-package-en/config.json \ No newline at end of file diff --git a/integration-tests/i18n-forgotten-semicolon/evaluation b/integration-tests/i18n-forgotten-semicolon/evaluation deleted file mode 120000 index 558ef2e..0000000 --- a/integration-tests/i18n-forgotten-semicolon/evaluation +++ /dev/null @@ -1 +0,0 @@ -../i18n-with-package-en/evaluation \ No newline at end of file diff --git a/integration-tests/i18n-stackoverflow/config.json b/integration-tests/i18n-stackoverflow/config.json deleted file mode 120000 index 77b3a67..0000000 --- a/integration-tests/i18n-stackoverflow/config.json +++ /dev/null @@ -1 +0,0 @@ -../i18n-with-package-en/config.json \ No newline at end of file diff --git a/integration-tests/i18n-stackoverflow/evaluation b/integration-tests/i18n-stackoverflow/evaluation deleted file mode 120000 index 558ef2e..0000000 --- a/integration-tests/i18n-stackoverflow/evaluation +++ /dev/null @@ -1 +0,0 @@ -../i18n-with-package-en/evaluation \ No newline at end of file diff --git a/integration-tests/i18n-with-package-en/evaluation/TranslatedTest.java b/integration-tests/i18n-with-package-en/evaluation/TranslatedTest.java deleted file mode 100644 index 62ea954..0000000 --- a/integration-tests/i18n-with-package-en/evaluation/TranslatedTest.java +++ /dev/null @@ -1,22 +0,0 @@ -import dodona.i18n.Language; -import dodona.i18n.I18nTabTitle; -import dodona.i18n.I18nTestDescription; - -import org.junit.Assert; -import org.junit.Test; - -@I18nTabTitle("tab_title") -public class TranslatedTest { - - @Test - @I18nTestDescription("translated_description") - public void test() { - String language = new Translated().getLanguage(); - if(Language.DUTCH == Language.current()) { - Assert.assertEquals("nederlands", language); - } else { - Assert.assertEquals("english", language); - } - } - -} diff --git a/integration-tests/i18n-with-package-en/evaluation/properties/descriptions.en.properties b/integration-tests/i18n-with-package-en/evaluation/properties/descriptions.en.properties deleted file mode 100644 index a04621c..0000000 --- a/integration-tests/i18n-with-package-en/evaluation/properties/descriptions.en.properties +++ /dev/null @@ -1,2 +0,0 @@ -translated_description=Translated Test -tab_title=Translated title diff --git a/integration-tests/i18n-with-package-en/evaluation/properties/descriptions.nl.properties b/integration-tests/i18n-with-package-en/evaluation/properties/descriptions.nl.properties deleted file mode 100644 index 77d6aa0..0000000 --- a/integration-tests/i18n-with-package-en/evaluation/properties/descriptions.nl.properties +++ /dev/null @@ -1,2 +0,0 @@ -translated_description=Vertaalde Test -tab_title=Vertaalde titel diff --git a/integration-tests/i18n-with-package-nl/config.json b/integration-tests/i18n-with-package-nl/config.json deleted file mode 100644 index 817a980..0000000 --- a/integration-tests/i18n-with-package-nl/config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "filename": "Translated.java", - "natural_language": "nl" -} \ No newline at end of file diff --git a/integration-tests/i18n-with-package-nl/evaluation b/integration-tests/i18n-with-package-nl/evaluation deleted file mode 120000 index 558ef2e..0000000 --- a/integration-tests/i18n-with-package-nl/evaluation +++ /dev/null @@ -1 +0,0 @@ -../i18n-with-package-en/evaluation \ No newline at end of file diff --git a/integration-tests/i18n-with-package-nl/result.json b/integration-tests/i18n-with-package-nl/result.json deleted file mode 100644 index 4da0d92..0000000 --- a/integration-tests/i18n-with-package-nl/result.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "command": "start-judgement" -} -{ - "command": "start-tab", - "hidden": true, - "title": "Compiler" -} -{ - "command": "start-context", - "description": { - "description": "Je diende geen Translated-klasse in, waardoor de testen niet uitgevoerd kunnen worden.", - "format": "plain" - } -} -{ - "accepted": false, - "command": "close-context" -} -{ - "command": "append-message", - "message": { - "description": "Ben je zeker dat je de ingediende klasse in het default package plaatste? Je klasse komt in het default package terecht als je jouw package statement verwijdert.", - "format": "callout" - } -} -{ - "command": "close-tab" -} -{ - "accepted": false, - "command": "close-judgement", - "status": { - "enum": "compilation error", - "human": "Compilatiefout" - } -} diff --git a/integration-tests/i18n-with-package-nl/submission.java b/integration-tests/i18n-with-package-nl/submission.java deleted file mode 100644 index e69911c..0000000 --- a/integration-tests/i18n-with-package-nl/submission.java +++ /dev/null @@ -1,9 +0,0 @@ -package /**/ translate; - -public class Translated { - - public String getLanguage() { - return "nederlands"; - } - -} diff --git a/integration-tests/run b/integration-tests/run index 5ea4ff7..e171a9c 100755 --- a/integration-tests/run +++ b/integration-tests/run @@ -1,9 +1,11 @@ #!/usr/bin/env bash set -e +shopt -s globstar # Default values target_dir="integration-tests" overwrite=0 +err_out="/dev/null" # Parse arguments while [ "$#" -gt 0 ]; do @@ -11,6 +13,9 @@ while [ "$#" -gt 0 ]; do --overwrite) overwrite=1 ;; + -v|--verbose) + err_out="/dev/stderr" + ;; *) target_dir="$1" ;; @@ -26,7 +31,7 @@ minor_change=0 major_change=0 # Find all config.json files in target directory -for config in "$target_dir"/**/config.json; do +for config in $target_dir/**/config.json; do test_dir="$(dirname "$config")" # Extract config values @@ -69,7 +74,7 @@ for config in "$target_dir"/**/config.json; do , "memory_limit": 1000000000 , "source": "'"$judge/$source"'" }' \ - | (timeout -k 10s 60s "$judge/run" 2> /dev/null) \ + | (timeout -k 10s 60s "$judge/run" 2> "$err_out") \ | jq --sort-keys 'if(.command == "append-message") then .message.description |= gsub("\n at [^\n]+\\([^)]+\\)"; "") else . @@ -80,30 +85,30 @@ for config in "$target_dir"/**/config.json; do rm -r "$workdir" # Count accepted and failed in output - accepted_output="$(echo "$output" | jq -s 'map(select(.accepted == true)) | length')" - failed_output="$(echo "$output" | jq -s 'map(select(.accepted == false)) | length')" - echo -en "$accepted_output accepted, $failed_output failed\t" + accepted_actual="$(echo "$output" | jq -s 'map(select(.accepted == true)) | length')" + failed_actual="$(echo "$output" | jq -s 'map(select(.accepted == false)) | length')" + echo -en "$accepted_actual accepted, $failed_actual failed\t" if [ -f "$result_file" ]; then # Count accepted and failed in result - accepted_result="$(jq -s 'map(select(.accepted == true)) | length' "$result_file" )" - failed_result="$(jq -s 'map(select(.accepted == true)) | length' "$result_file" )" - + accepted_expected="$(jq -s 'map(select(.accepted == true)) | length' "$result_file" )" + failed_expected="$(jq -s 'map(select(.accepted == false)) | length' "$result_file" )" # First check if files are exact using diff, then check if they differ except '.description', then check they differ in accepted or failed - if diff "$result_file" <(echo "$output") > /dev/null; then + + if diff -c "$result_file" <(echo "$output") > "$err_out"; then echo "[EXACT MATCH]" exact_match=$((exact_match + 1)) - elif diff <(jq "del(.description)" "$result_file") <(echo "$output" | jq "del(.description)") > /dev/null; then + elif diff -c <(jq "del(.description)" "$result_file") <(echo "$output" | jq "del(.description)") > /dev/null; then echo "[DESCRIPTION CHANGED]" description_changed=$((description_changed + 1)) - elif [ "$accepted_output" -eq "$accepted_result" ] && [ "$failed_output" -eq "$failed_result" ]; then + elif [ "$accepted_actual" -eq "$accepted_expected" ] && [ "$failed_actual" -eq "$failed_expected" ]; then echo "[MINOR CHANGE] (accepted/failed unchanged)" minor_change=$((minor_change + 1)) else - echo "[MAJOR CHANGE] (accepted/failed changed, was $accepted_result/$failed_result)" + echo "[MAJOR CHANGE] (accepted/failed changed, expected $accepted_expected/$failed_expected, actual $accepted_actual/$failed_actual)" major_change=$((major_change + 1)) fi fi diff --git a/integration-tests/stackoverflow/config.json b/integration-tests/stackoverflow/config.json new file mode 120000 index 0000000..6f8e641 --- /dev/null +++ b/integration-tests/stackoverflow/config.json @@ -0,0 +1 @@ +../with-package/config.json \ No newline at end of file diff --git a/integration-tests/stackoverflow/evaluation b/integration-tests/stackoverflow/evaluation new file mode 120000 index 0000000..99f1c59 --- /dev/null +++ b/integration-tests/stackoverflow/evaluation @@ -0,0 +1 @@ +../with-package/evaluation/ \ No newline at end of file diff --git a/integration-tests/i18n-stackoverflow/result.json b/integration-tests/stackoverflow/result.json similarity index 92% rename from integration-tests/i18n-stackoverflow/result.json rename to integration-tests/stackoverflow/result.json index da82e7a..95222eb 100644 --- a/integration-tests/i18n-stackoverflow/result.json +++ b/integration-tests/stackoverflow/result.json @@ -13,12 +13,12 @@ "command": "start-tab", "hidden": false, "permission": "student", - "title": "Translated title" + "title": "Tab title" } { "command": "start-context", "description": { - "description": "Translated Test", + "description": "Test description", "format": "code" } } diff --git a/integration-tests/i18n-stackoverflow/submission.java b/integration-tests/stackoverflow/submission.java similarity index 100% rename from integration-tests/i18n-stackoverflow/submission.java rename to integration-tests/stackoverflow/submission.java diff --git a/integration-tests/system-exit-nl/config.json b/integration-tests/system-exit-nl/config.json deleted file mode 100644 index 2b64065..0000000 --- a/integration-tests/system-exit-nl/config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "filename": "Exitter.java", - "natural_language": "nl" -} \ No newline at end of file diff --git a/integration-tests/system-exit-nl/evaluation b/integration-tests/system-exit-nl/evaluation deleted file mode 120000 index 19c0f24..0000000 --- a/integration-tests/system-exit-nl/evaluation +++ /dev/null @@ -1 +0,0 @@ -../system-exit-en/evaluation \ No newline at end of file diff --git a/integration-tests/system-exit-nl/result.json b/integration-tests/system-exit-nl/result.json deleted file mode 100644 index a3f25ac..0000000 --- a/integration-tests/system-exit-nl/result.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "command": "start-judgement" -} -{ - "command": "start-tab", - "hidden": true, - "title": "Compiler" -} -{ - "command": "close-tab" -} -{ - "command": "start-tab", - "hidden": false, - "permission": "student", - "title": "Test" -} -{ - "command": "start-context", - "description": { - "description": "exit1ShouldFail(Unaware)", - "format": "code" - } -} -{ - "command": "start-testcase", - "description": { - "description": "dodona.junit.ExitException: System.exit(1) called.", - "format": "code" - } -} -{ - "command": "escalate-status", - "status": { - "enum": "runtime error", - "human": "Uitvoeringsfout" - } -} -{ - "command": "append-message", - "message": { - "description": "Caused by dodona.junit.ExitException: System.exit(1) called.", - "format": "code" - } -} -{ - "accepted": false, - "command": "close-testcase" -} -{ - "accepted": false, - "command": "close-context" -} -{ - "command": "start-context", - "description": { - "description": "noExitShouldPass(Unaware)", - "format": "code" - } -} -{ - "accepted": true, - "command": "close-context" -} -{ - "command": "start-context", - "description": { - "description": "exit0ShouldFail(Unaware)", - "format": "code" - } -} -{ - "command": "start-testcase", - "description": { - "description": "dodona.junit.ExitException: System.exit(0) called.", - "format": "code" - } -} -{ - "command": "escalate-status", - "status": { - "enum": "runtime error", - "human": "Uitvoeringsfout" - } -} -{ - "command": "append-message", - "message": { - "description": "Caused by dodona.junit.ExitException: System.exit(0) called.", - "format": "code" - } -} -{ - "accepted": false, - "command": "close-testcase" -} -{ - "accepted": false, - "command": "close-context" -} -{ - "command": "close-tab" -} -{ - "command": "start-tab", - "hidden": false, - "permission": "student", - "title": "Test" -} -{ - "command": "start-context", - "description": { - "description": "exit0IsOK(Usage)", - "format": "code" - } -} -{ - "accepted": true, - "command": "close-context" -} -{ - "command": "start-context", - "description": { - "description": "exit1IsNotOK(Usage)", - "format": "code" - } -} -{ - "command": "escalate-status", - "status": { - "enum": "wrong", - "human": "Fout" - } -} -{ - "command": "start-testcase", - "description": { - "description": "expected:<0> but was:<1>", - "format": "code" - } -} -{ - "accepted": false, - "command": "close-testcase" -} -{ - "accepted": false, - "command": "close-context" -} -{ - "command": "close-tab" -} -{ - "command": "start-tab", - "hidden": false, - "permission": "student", - "title": "Test" -} -{ - "command": "start-context", - "description": { - "description": "noExitShouldPass(Interference)", - "format": "code" - } -} -{ - "accepted": true, - "command": "close-context" -} -{ - "command": "start-context", - "description": { - "description": "exit0IsOK(Interference)", - "format": "code" - } -} -{ - "accepted": true, - "command": "close-context" -} -{ - "command": "start-context", - "description": { - "description": "exit1shouldFail(Interference)", - "format": "code" - } -} -{ - "command": "escalate-status", - "status": { - "enum": "wrong", - "human": "Fout" - } -} -{ - "command": "start-testcase", - "description": { - "description": "Wrong exit status expected:<0> but was:<1>", - "format": "code" - } -} -{ - "accepted": false, - "command": "close-testcase" -} -{ - "accepted": false, - "command": "close-context" -} -{ - "command": "close-tab" -} -{ - "command": "close-judgement" -} diff --git a/integration-tests/system-exit-nl/submission.java b/integration-tests/system-exit-nl/submission.java deleted file mode 120000 index 836b00b..0000000 --- a/integration-tests/system-exit-nl/submission.java +++ /dev/null @@ -1 +0,0 @@ -../system-exit-en/submission.java \ No newline at end of file diff --git a/integration-tests/system-exit-en/config.json b/integration-tests/system-exit/config.json similarity index 100% rename from integration-tests/system-exit-en/config.json rename to integration-tests/system-exit/config.json diff --git a/integration-tests/system-exit-en/evaluation/Interference.java b/integration-tests/system-exit/evaluation/Interference.java similarity index 100% rename from integration-tests/system-exit-en/evaluation/Interference.java rename to integration-tests/system-exit/evaluation/Interference.java diff --git a/integration-tests/system-exit-en/evaluation/TestSuite.java b/integration-tests/system-exit/evaluation/TestSuite.java similarity index 100% rename from integration-tests/system-exit-en/evaluation/TestSuite.java rename to integration-tests/system-exit/evaluation/TestSuite.java diff --git a/integration-tests/system-exit-en/evaluation/Unaware.java b/integration-tests/system-exit/evaluation/Unaware.java similarity index 100% rename from integration-tests/system-exit-en/evaluation/Unaware.java rename to integration-tests/system-exit/evaluation/Unaware.java diff --git a/integration-tests/system-exit-en/evaluation/Usage.java b/integration-tests/system-exit/evaluation/Usage.java similarity index 100% rename from integration-tests/system-exit-en/evaluation/Usage.java rename to integration-tests/system-exit/evaluation/Usage.java diff --git a/integration-tests/system-exit-en/result.json b/integration-tests/system-exit/result.json similarity index 100% rename from integration-tests/system-exit-en/result.json rename to integration-tests/system-exit/result.json diff --git a/integration-tests/system-exit-en/submission.java b/integration-tests/system-exit/submission.java similarity index 100% rename from integration-tests/system-exit-en/submission.java rename to integration-tests/system-exit/submission.java diff --git a/integration-tests/i18n-with-package-en/config.json b/integration-tests/with-package/config.json similarity index 100% rename from integration-tests/i18n-with-package-en/config.json rename to integration-tests/with-package/config.json diff --git a/integration-tests/with-package/evaluation/SimpleTest.java b/integration-tests/with-package/evaluation/SimpleTest.java new file mode 100644 index 0000000..74fcba8 --- /dev/null +++ b/integration-tests/with-package/evaluation/SimpleTest.java @@ -0,0 +1,16 @@ +import org.junit.Assert; +import org.junit.Test; + +import dodona.junit.TabTitle; +import dodona.junit.TestDescription; + +@TabTitle("Tab title") +public class SimpleTest { + + @Test + @TestDescription("Test description") + public void test() { + Assert.assertEquals(new Translated().getLanguage(), "english"); + } + +} diff --git a/integration-tests/i18n-with-package-en/evaluation/TestSuite.java b/integration-tests/with-package/evaluation/TestSuite.java similarity index 84% rename from integration-tests/i18n-with-package-en/evaluation/TestSuite.java rename to integration-tests/with-package/evaluation/TestSuite.java index 9b1f72e..fdaec37 100644 --- a/integration-tests/i18n-with-package-en/evaluation/TestSuite.java +++ b/integration-tests/with-package/evaluation/TestSuite.java @@ -4,6 +4,6 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ - TranslatedTest.class, + SimpleTest.class, }) public class TestSuite {} diff --git a/integration-tests/i18n-with-package-en/result.json b/integration-tests/with-package/result.json similarity index 100% rename from integration-tests/i18n-with-package-en/result.json rename to integration-tests/with-package/result.json diff --git a/integration-tests/i18n-with-package-en/submission.java b/integration-tests/with-package/submission.java similarity index 100% rename from integration-tests/i18n-with-package-en/submission.java rename to integration-tests/with-package/submission.java diff --git a/makefile b/makefile index c4a8acc..7a4cf91 100644 --- a/makefile +++ b/makefile @@ -25,11 +25,6 @@ dist/judge.jar: $(CLASSES) # Phonies # # ============================================================================ # -# something wicked -space := -space += -$(space) := -$(space) += .PHONY: jar jar: dist/judge.jar @@ -47,4 +42,4 @@ clean: rm -rf build/ mkdir build/ rm -rf dist/ - mkdir dist/ + mkdir dist/ \ No newline at end of file diff --git a/run b/run index d5923c7..95897ab 100755 --- a/run +++ b/run @@ -19,10 +19,6 @@ judge="$(jq -r '.judge' "$config")" workdir="$(jq -r '.workdir' "$config")" filename="$(jq -r '.filename' "$config")" -# Natural language of the user -natural_language="$(jq -r '.natural_language' "$config")" -. "$judge"/i18n/"$natural_language" - # memory limit with some margin memory_limit="$(jq -r '.memory_limit' "$config")" memory_limit="$(( memory_limit * 9 / 10000 ))" @@ -47,16 +43,16 @@ explain_compilation_error() { case "$1" in *"should be declared in a file named"*) # Wrong class name. - printf "$i18n_wrong_class_name\n" "${filename%.java}" + printf "Your class should be called \"%s\".\n" "${filename%.java}" ;; *"cannot find symbol"*) # Cannot find symbol - forgotten import. class_name="$(echo "$1" | sed -n '/symbol: *class/s/.*class \(\S\+\)\s*.*/\1/p')" - [ -z "$class_name" ] || printf "$i18n_forgot_import\n" "$class_name" + [ -z "$class_name" ] || printf "You are trying to use an unknown class \"%s\". Might you have forgotten an import?\n" "$class_name" ;; *"assign a value to final variable"*) # Assignment to final variable. - printf "$i18n_assign_to_final\n" "$(echo "$1" | grep -o "final variable \S\+" | sed "s/final variable //" | sed "s/\\\n//")" + printf "Variables declared to be \"final\" can be assigned to only once. Remove the keyword \"final\" from the variable \"%s\" to solve this.\n" "$(echo "$1" | grep -o "final variable \S\+" | sed "s/final variable //" | sed "s/\\\n//")" ;; esac } @@ -64,7 +60,7 @@ explain_compilation_error() { explain_compilation_warning() { case "$1" in *"found raw type"*) - echo "$i18n_raw_type" + echo "Generic classes should be passed a type." ;; esac } @@ -72,7 +68,7 @@ explain_compilation_warning() { parse_compilation_error_staff() { # arg1: 1 compiler log dodona start-context - dodona start-testcase -f plain -d "$i18n_compilation_error" + dodona start-testcase -f plain -d "Compilation error" dodona append-message -f code -p staff -d "$1" dodona close-testcase -A dodona close-context @@ -90,12 +86,12 @@ parse_compilation_error_student() { # warning type='warning' explanation="$(explain_compilation_warning "$1")" - testcase_msg="$i18n_compilation_warning" + testcase_msg="Compilation warning" else # error type='error' explanation="$(explain_compilation_error "$1")" - testcase_msg="$i18n_compilation_error" + testcase_msg="Compilation error" fi # Start the case, add the explanation message if it exists, add compilation output as message @@ -178,18 +174,18 @@ compilation_failed() { # Build the compilation counts message. case "$compile_error_count" in 0) described_error_count="" ;; - 1) described_error_count="1 $i18n_error" ;; - *) described_error_count="$compile_error_count $i18n_errors" ;; + 1) described_error_count="1 error" ;; + *) described_error_count="$compile_error_count errors" ;; esac case "$compile_warning_count" in 0) described_warning_count="" ;; - 1) described_warning_count="1 $i18n_warning" ;; - *) described_warning_count="$compile_error_count $i18n_warnings" ;; + 1) described_warning_count="1 warning" ;; + *) described_warning_count="$compile_error_count warnings" ;; esac [ "$compile_error_count" -ne 0 -a "$compile_warning_count" -ne 0 ] \ - && described_both_count="$described_error_count $i18n_and $described_warning_count" \ + && described_both_count="$described_error_count and $described_warning_count" \ || described_both_count="$described_error_count$described_warning_count" dodona append-message -f callout -d "$(printf "$callout" "$described_both_count")" @@ -213,7 +209,7 @@ worklibs="$([ -d "$workdir" ] && find "$workdir" -name '*.jar' | xargs echo | tr # Compiling the workdir given code if ! find . -name '*.java' | xargs --no-run-if-empty javac -cp ".:${worklibs}:${testlibs}" -d . -sourcepath . > "$compilation" 2>&1; then - compilation_failed "$compilation" "$i18n_workdir_compilation_message" "$i18n_workdir_compilation_summary" 'staff' 0 + compilation_failed "$compilation" "Something went wrong while compiling the base code for this exercise. Contact your teacher." "Error in base code" 'staff' 0 fi # Create the Input.java class, containing the submitted code @@ -225,7 +221,7 @@ sed -i '1,5{s/^package [a-zA-Z0-9_.]*;//}' "$filename" # Compiling the user code [ "$allow_compilation_warnings" = 'true' ] || compile_opt='-Werror' if ! javac -cp ".:${worklibs}" -Xlint:all $compile_opt "$filename" > "$compilation" 2>&1; then - compilation_failed "$compilation" "$i18n_user_compilation_message" "%s" 'student' 1 + compilation_failed "$compilation" "Your code cannot be compiled and therefore not tested. The compiler reported %s." "%s" 'student' 1 fi # Verify the student submitted the requested class @@ -235,13 +231,13 @@ public class Import { } HERE if ! javac -cp . -d "$importclass" "$importclass/Import.java" >/dev/null 2>&1; then - dodona start-context -f plain -d "$(printf "$i18n_class_not_submitted" "${filename%.java}")" + dodona start-context -f plain -d "$(printf "Your submission does not contain a class \"%s\", so your code could not be tested." "${filename%.java}")" dodona close-context -A if grep -q '^package' "$filename"; then - dodona append-message -f callout -d "$i18n_default_package" + dodona append-message -f callout -d "Are you sure you put your submitted class in the default package? Your class will be put in the default package if you remove the package statement." fi dodona close-tab - dodona close-judgement -A -e 'compilation error' -h "$i18n_compilation_error" + dodona close-judgement -A -e 'compilation error' -h "Compilation error" exit 0 fi @@ -252,7 +248,7 @@ jar -cf "judge.jar" -C /tmp/build . # Compiling the tests if ! find "$resources" -name '*.java' | xargs javac -Xdiags:verbose -cp ".:${resources}:${worklibs}:${testlibs}:judge.jar" -d . -sourcepath "$resources" > "$compilation" 2>&1; then - compilation_failed "$compilation" "$i18n_test_compilation_message" "$i18n_test_compilation_summary" 'student' 0 + compilation_failed "$compilation" "Something went wrong while compiling the tests for this exercise." "Error in the tests" 'student' 0 fi # Everything is compiled @@ -261,4 +257,4 @@ dodona close-tab # Running the tests java -Djava.security.manager=allow -Xss32M -Xmx"${memory_limit}k" -cp ".:${worklibs}:${testlibs}:judge.jar:${resources}/properties" -Ddodona.language="${natural_language}" -Ddodona.output_cutoff="${generated_output_cutoff}" dodona.junit.JUnitJSON -dodona close-judgement +dodona close-judgement \ No newline at end of file diff --git a/src/dodona/i18n/I18nTabTitle.java b/src/dodona/i18n/I18nTabTitle.java deleted file mode 100644 index 1436fdb..0000000 --- a/src/dodona/i18n/I18nTabTitle.java +++ /dev/null @@ -1,23 +0,0 @@ -package dodona.i18n; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -/** - * Sets the title of the tab, which is shown on Dodona on top with a badge - * displaying the amount of failed test cases. The value should be a key that is - * configured in both of the following files: - *

- * evaluation/properties/descriptions.en.properties - * evaluation/properties/descriptions.nl.properties - *

- */ -@Retention(RetentionPolicy.RUNTIME) -public @interface I18nTabTitle { - /** - * The resource key of the description to display. - * - * @return the resource key - */ - String value(); -} diff --git a/src/dodona/i18n/I18nTestDescription.java b/src/dodona/i18n/I18nTestDescription.java deleted file mode 100644 index c395001..0000000 --- a/src/dodona/i18n/I18nTestDescription.java +++ /dev/null @@ -1,25 +0,0 @@ -package dodona.i18n; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -/** - * Sets the description of the test case, which is shown on Dodona as the header - * of the test case. The value should be a key that is configured in both of the - * following files: - *

- * evaluation/properties/descriptions.en.properties - * evaluation/properties/descriptions.nl.properties - *

- * Roughly the equivalent of JUnit 5's @DisplayName-annotation, but with i18n - * support. - */ -@Retention(RetentionPolicy.RUNTIME) -public @interface I18nTestDescription { - /** - * The resource key of the description to display. - * - * @return the resource key - */ - String value(); -} diff --git a/src/dodona/i18n/Language.java b/src/dodona/i18n/Language.java deleted file mode 100644 index 2549ff1..0000000 --- a/src/dodona/i18n/Language.java +++ /dev/null @@ -1,48 +0,0 @@ -package dodona.i18n; - -import java.util.Arrays; -import java.util.Objects; - -import static dodona.junit.JUnitJSON.PROPERTY_LANGUAGE; - -/** - * Supported languages on Dodona. - */ -public enum Language { - DUTCH("nl"), - ENGLISH("en"); - - private final String identifier; - - /** - * Language constructor. - * - * @param identifier 2-letter identifier of the language - */ - Language(final String identifier) { - this.identifier = identifier; - } - - /** - * Gets the current language. Defaults to English if an unknown language was - * passed. - * - * @return the current active language - */ - public static Language current() { - final String fromProperties = System.getProperty(PROPERTY_LANGUAGE); - return Arrays.stream(Language.values()) - .filter(lang -> Objects.equals(fromProperties, lang.identifier)) - .findAny() - .orElse(ENGLISH); - } - - /** - * Gets the 2-letter identifier. - * - * @return the identifier - */ - public String getIdentifier() { - return this.identifier; - } -} diff --git a/src/dodona/junit/JSONListener.java b/src/dodona/junit/JSONListener.java index 26951d3..9ef4b0a 100644 --- a/src/dodona/junit/JSONListener.java +++ b/src/dodona/junit/JSONListener.java @@ -11,9 +11,6 @@ import dodona.feedback.StartTab; import dodona.feedback.StartTestcase; import dodona.feedback.Status; -import dodona.i18n.I18nTabTitle; -import dodona.i18n.I18nTestDescription; -import dodona.i18n.Language; import dodona.json.Json; import org.junit.runner.Description; import org.junit.runner.Result; @@ -21,20 +18,16 @@ import org.junit.runner.notification.RunListener; import org.junit.runners.model.TestTimedOutException; -import java.io.InputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.PropertyResourceBundle; -import java.util.ResourceBundle; public class JSONListener extends RunListener { private static final int STACKSIZE = 50; - private final ResourceBundle descriptions; private final PrintStream writer; private final Json json; @@ -44,42 +37,26 @@ public JSONListener() { } public JSONListener(PrintStream writer) { - this.descriptions = getBundleIfExists("descriptions", Language.current()); this.writer = writer; this.json = new Json(); } - /** - * Gets the given resource bundle for the current language, if it exists. - * - * @param base the base name of the resource bundle - * @return the bundle if it exists, or null otherwise - */ - private static ResourceBundle getBundleIfExists(final String base, final Language language) { - try { - final String bundleName = String.format("%s.%s.properties", base, language.getIdentifier()); - final InputStream bundleStream = JSONListener.class.getClassLoader().getResourceAsStream(bundleName); - return new PropertyResourceBundle(bundleStream); - } catch (final Exception exception) { - return null; - } - } - private void write(Object src) { writer.print(json.asString(src)); } /* COMPLETE RUN */ - public void beforeExecution() {} + public void beforeExecution() { + } - public void afterExecution() {} + public void afterExecution() { + } public void beforeTab(Description description) { - final String title = this.getI18nTabTitle(description) - .orElseGet(() -> this.getTabTitle(description) - .orElse(TabTitle.DEFAULT)); + final String title = this.getTabTitle(description) + .orElse(TabTitle.DEFAULT); final Permission permission = this.getTabPermission(description) - .orElse(TabPermission.DEFAULT); + .orElse(TabPermission.DEFAULT); write(new StartTab(title, permission)); } @@ -89,7 +66,7 @@ public void afterTab() { public void beforeTest(Description description) { final String title = this.getDescription(description); - if(depth < 3) { + if (depth < 3) { // An exception got thrown outside a tab because the test class is incorrect write(new StartTab("Loading tests")); } @@ -97,46 +74,50 @@ public void beforeTest(Description description) { } public void aftertest(Failure failure) { - if(failure == null) { + if (failure == null) { write(new CloseContext(true)); } else { Throwable thrown = failure.getException(); List feedback = new ArrayList<>(); - if(thrown instanceof AnnotatedThrowable) { + if (thrown instanceof AnnotatedThrowable) { feedback = ((AnnotatedThrowable) thrown).getFeedback(); thrown = thrown.getCause(); } - if(thrown instanceof TestCarryingThrowable) { + if (thrown instanceof TestCarryingThrowable) { write(new StartTestcase(Message.plain(""))); write(((TestCarryingThrowable) thrown).getStartTest()); ((TestCarryingThrowable) thrown).getMessages().stream().map(AppendMessage::new).forEach(this::write); write(((TestCarryingThrowable) thrown).getCloseTest()); - } else if(thrown instanceof AssertionError) { + } else if (thrown instanceof AssertionError) { write(new EscalateStatus(Status.WRONG, "Fout")); write(new StartTestcase(Message.code(thrown.getMessage() == null ? "" : thrown.getMessage()))); } else { Throwable deepest = thrown; - while(deepest.getCause() != null) deepest = deepest.getCause(); + while (deepest.getCause() != null) + deepest = deepest.getCause(); write(new StartTestcase(Message.code(deepest.toString()))); if (thrown instanceof TestTimedOutException) { write(new EscalateStatus(Status.TIME_LIMIT_EXCEEDED, "Tijdslimiet overschreden")); } else { write(new EscalateStatus(Status.RUNTIME_ERROR, "Uitvoeringsfout")); } - while(thrown != null) { + while (thrown != null) { StringBuilder message = new StringBuilder(); message.append("Caused by " + thrown); StackTraceElement[] stacktrace = thrown.getStackTrace(); boolean leftDefaultPackage = false; - for(int i = 0; i < stacktrace.length && i < STACKSIZE; i++) { + for (int i = 0; i < stacktrace.length && i < STACKSIZE; i++) { // student code in default package boolean inDefaultPackage = stacktrace[i].getClassName().indexOf('.') < 0; - if(leftDefaultPackage && !inDefaultPackage) break; - if(inDefaultPackage) leftDefaultPackage = true; + if (leftDefaultPackage && !inDefaultPackage) + break; + if (inDefaultPackage) + leftDefaultPackage = true; message.append("\n at " + stacktrace[i].toString()); } - if(stacktrace.length >= STACKSIZE) message.append("\n ..."); + if (stacktrace.length >= STACKSIZE) + message.append("\n ..."); write(new AppendMessage(Message.code(message.toString()))); thrown = thrown.getCause(); } @@ -147,7 +128,7 @@ public void aftertest(Failure failure) { write(new CloseTestcase(false)); write(new CloseContext(false)); - if(depth < 3) { + if (depth < 3) { write(new CloseTab()); } } @@ -160,39 +141,8 @@ public void aftertest(Failure failure) { * @return the human-friendly version */ private String getDescription(final Description desc) { - return getI18nTestDescription(desc) - .orElseGet(() -> getTestDescription(desc) - .orElse(desc.getDisplayName())); - } - - /** - * Parse a @I18nTabTitle annotation. - * - * @param desc the description - * @return the value of the I18nTabTitle annotation if available - */ - private Optional getI18nTabTitle(final Description desc) { - return Optional.ofNullable(this.descriptions).flatMap(bundle -> - Optional.ofNullable(desc.getAnnotation(I18nTabTitle.class)) - .map(I18nTabTitle::value) - .filter(bundle::containsKey) - .map(bundle::getString) - ); - } - - /** - * Parse a @I18nTestDescription annotation. - * - * @param desc the description - * @return the value of the I18nTestDescription annotation if available - */ - private Optional getI18nTestDescription(final Description desc) { - return Optional.ofNullable(this.descriptions).flatMap(bundle -> - Optional.ofNullable(desc.getAnnotation(I18nTestDescription.class)) - .map(I18nTestDescription::value) - .filter(bundle::containsKey) - .map(bundle::getString) - ); + return getTestDescription(desc) + .orElse(desc.getDisplayName()); } /** @@ -223,8 +173,8 @@ private Optional getTabPermission(final Description desc) { */ private static Optional getTestDescription(final Description desc) { return Optional - .ofNullable(desc.getAnnotation(TestDescription.class)) - .map(TestDescription::value); + .ofNullable(desc.getAnnotation(TestDescription.class)) + .map(TestDescription::value); } /* Ugly internals */ @@ -240,12 +190,14 @@ public void testRunFinished(Result result) throws Exception { } public void testSuiteStarted(Description description) throws Exception { - if(depth++ != 2) return; + if (depth++ != 2) + return; beforeTab(description); } public void testSuiteFinished(Description description) throws Exception { - if(--depth != 2) return; + if (--depth != 2) + return; afterTab(); } @@ -268,13 +220,14 @@ public void testFailure(Failure failure) throws Exception { public void testAssumptionFailure(Failure failure) { StringWriter stackCollector = new StringWriter(); stackCollector.append("testAssumptionFailure in " + - failure.getTestHeader() + ": " + - failure.getException().getMessage() + "\n"); + failure.getTestHeader() + ": " + + failure.getException().getMessage() + "\n"); failure.getException().printStackTrace(new PrintWriter(stackCollector)); write(new AppendMessage(Message.internalError(stackCollector.toString()))); } - public void testIgnored(Description description) throws Exception {} + public void testIgnored(Description description) throws Exception { + } } diff --git a/src/dodona/junit/JUnitJSON.java b/src/dodona/junit/JUnitJSON.java index c3e6618..352e184 100644 --- a/src/dodona/junit/JUnitJSON.java +++ b/src/dodona/junit/JUnitJSON.java @@ -12,15 +12,15 @@ import dodona.json.Json; public class JUnitJSON { - public static final String PROPERTY_LANGUAGE = "dodona.language"; public static final String PROPERTY_OUTPUT_CUTOFF = "dodona.output_cutoff"; public static void main(String... args) { Class testSuite = null; try { testSuite = Class.forName("TestSuite", true, currentThread().getContextClassLoader()); - } catch(ClassNotFoundException e) { - System.out.println(new Json().asString(new AppendMessage(Message.internalError("TestSuite class not found.")))); + } catch (ClassNotFoundException e) { + System.out.println( + new Json().asString(new AppendMessage(Message.internalError("TestSuite class not found.")))); System.exit(1); } @@ -30,7 +30,7 @@ public static void main(String... args) { System.setSecurityManager(sm); JUnitCore core = new JUnitCore(); core.addListener(new JSONListener()); - core.run(new Class[]{ testSuite }); + core.run(new Class[] { testSuite }); System.setSecurityManager(sm.getPrevious()); } @@ -41,15 +41,18 @@ public NoExitSecurityManager(SecurityManager previous) { this.previous = Optional.ofNullable(previous); } - @Override public void checkPermission(Permission perm) { + @Override + public void checkPermission(Permission perm) { previous.ifPresent(sm -> sm.checkPermission(perm)); } - @Override public void checkPermission(Permission perm, Object context) { + @Override + public void checkPermission(Permission perm, Object context) { previous.ifPresent(sm -> sm.checkPermission(perm, context)); } - @Override public void checkExit(int status) { + @Override + public void checkExit(int status) { super.checkExit(status); throw new ExitException(status); } diff --git a/validate.sh b/validate.sh deleted file mode 100755 index 39f0118..0000000 --- a/validate.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -set -e - -# Usage: ./validate.sh [--overwrite] - -if [ -z "$1" ]; then - echo "Usage: $0 [--overwrite]" - exit 1 -fi - -repo_path="$1" -shift - -# Check if repo path exists -if [ ! -d "$repo_path" ]; then - echo "Error: Directory '$repo_path' does not exist." - exit 1 -fi - -# Determine if overwrite flag is passed -overwrite="" -if [ "$1" = "--overwrite" ]; then - overwrite="--overwrite" -fi - -# Run the integration test runner with the repo path -./integration-tests/run "$repo_path" $overwrite