First Release

.gitignore

@@ -1 +1,2 @@
 /tests/**/*/results.json
+/tests/*/build/

Dockerfile

@@ -1,7 +1,10 @@
-FROM alpine:3.18
+FROM alpine:3.22
 
 # install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+RUN apk add --no-cache jq coreutils bash binutils make npm
+RUN apk add fpc --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+RUN npm install -g tap-parser
 
 WORKDIR /opt/test-runner
 COPY . .

bin/results-generator.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+declare -r test_file="$1"
+declare -r tap_file="$2"
+declare -A test_codes=()
+
+extract_test_codes () {
+    local state='out'
+    local proc_re='^procedure[[:blank:]]+[0-9A-Za-z_]+\.([0-9A-Za-z_]+);$'
+    local end_re='^end;$'
+    local assert_re=$'^[^\n]+\nbegin\n *(TapAssert.*);\nend;$'
+    local name
+    local body
+    while IFS= read -r line; do
+        if [[ "$state" == 'in' ]]; then
+            printf -v body $'%s\n%s' "$body" "$line"
+            if [[ "$line" =~ $end_re ]]; then
+                state='out'
+                shopt -s nocasematch
+                if [[ "$body" =~ $assert_re ]]; then
+                    test_codes["$name"]="${BASH_REMATCH[1]}"
+                else
+                    echo 'parser error'
+                    exit 1
+                fi
+                shopt -u nocasematch
+            fi
+        elif [[ "$line" =~ $proc_re ]]; then
+            state='in'
+            name="${BASH_REMATCH[1]}"
+            body="$line"
+        fi
+    done < "$test_file"
+}
+
+tap_parser() {
+    local json_test_codes={}
+    for key in "${!test_codes[@]}"; do
+        json_test_codes=$(
+            jq -cn \
+               --argjson json "$json_test_codes" \
+               --arg key "$key" \
+               --arg val "${test_codes["$key"]}" \
+               '$json + {$key: $val}'
+            )
+    done
+    local -a tap_content
+    tap_content=$(< "$tap_file")
+    local -r status="$(jq -r '
+      map(select(.[0] == "assert")) as $asserts |
+      if ($asserts | length) > 0 and all($asserts[] | .[1].ok) then
+        "pass"
+      else
+        "fail"
+      end
+    ' <<< "${tap_content[0]}"
+    )"
+
+    if [[ "$status" != "pass" ]] && \
+           jq -e \
+              '.[] |
+              select(.[0] == "plan" and .[1].comment == "no tests found") |
+              length > 0' <<< "${tap_content[0]}" &>/dev/null;
+    then
+        jq -r '
+        {
+          "version": 3,
+          "status" : "error",
+          "message": (map(select(.[0] == "extra") | .[1]) | join("")[0:65535])
+        }' <<< "${tap_content[0]}"
+    else
+        local -i i=0
+        local extra=''
+        local -a json_arrays
+        while IFS= read -r line; do
+            if jq -e '.[0] == "extra"' <<< "$line" &>/dev/null; then
+                extra+=$(jq -r '.[1]' <<< "$line")
+                extra+=$'\n'
+            elif jq -e '.[0] == "assert"' <<< "$line" &>/dev/null; then
+                if (( ${#extra} > 500 )); then
+                    extra="${extra:0:451}[Output was truncated. Please limit to 500 chars]"
+                fi
+                (( i++ ))
+                json_arrays+=("$(
+                  jq -r --arg extra "$extra" \
+                    '[.[0],
+                     (.[1] +
+                     { "output":
+                       if $extra == "" then null else $extra end })]' \
+                      <<< "$line"
+                  )")
+                extra=''
+            else
+                json_arrays+=("$line")
+            fi
+        done < <(jq -c '.[]' <<< "${tap_content[0]}")
+        printf '%s\n' "${json_arrays[@]}" |
+        jq --slurp \
+           --arg status "$status" \
+           --argjson test_codes "$json_test_codes" \
+           '
+        {
+          "version": 3,
+          "status" : $status,
+          "message": null
+        } + {
+        "tests": [
+            .[] | select(.[0] == "assert") | .[1] |
+            if .name == "Please implement your solution." then
+              { "name": .name, status: "error", test_code: "", message: .name }
+            elif .ok == true then
+              { "name": .name, status: "pass" }
+            else
+              {
+                "name": .diag.message,
+                "status": .diag.severity,
+                "output": .output,
+                "test_code": $test_codes[.name],
+                "message": "GOT:"      + (.diag.data.got|tostring) + "\n" +
+                           "EXPECTED:" + (.diag.data.expect|tostring),
+              }
+            end
+          ]
+        }
+        '
+    fi
+}
+
+extract_test_codes
+tap_parser

bin/run-tests.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 
 # Synopsis:
-# Test the test runner by running it against a predefined set of solutions 
+# Test the test runner by running it against a predefined set of solutions
 # with an expected output.
 
 # Output:
@@ -21,17 +21,25 @@ for test_dir in tests/*; do
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
     # OPTIONAL: Normalize the results file
-    # If the results.json file contains information that changes between 
-    # different test runs (e.g. timing information or paths), you should normalize
-    # the results file to allow the diff comparison below to work as expected
+    # If the results.json file contains information that changes between
+    # different test runs (e.g. timing information or paths), you should
+    # normalize the results file to allow the diff comparison below to work
+    # as expected
 
     file="results.json"
     expected_file="expected_${file}"
+    sed -E -i \
+        -e "s~${test_dir_path}~/solution~g" \
+        -e "s~${test_dir}~/solution~g" \
+        -e 's~/[[:alnum:][:punct:]]+/bin/ppcx64~/usr/bin/ppcx64~' \
+        "${test_dir_path}/${file}"
     echo "${test_dir_name}: comparing ${file} to ${expected_file}"
 
-    if ! diff "${test_dir_path}/${file}" "${test_dir_path}/${expected_file}"; then
+    actual_file="${test_dir_path}/${file}"
+    expected_file="${test_dir_path}/${expected_file}"
+    if ! diff  "$actual_file" "$expected_file"; then
         exit_code=1
     fi
 done
 
-exit ${exit_code}
+exit "${exit_code}"

bin/run.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 
 # Synopsis:
 # Run the test runner on a solution.
@@ -21,40 +21,24 @@ if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
     exit 1
 fi
 
-slug="$1"
-solution_dir=$(realpath "${2%/}")
-output_dir=$(realpath "${3%/}")
-results_file="${output_dir}/results.json"
+declare -r bin_dir="$(dirname $(realpath "$0"))"
+declare -r slug="$1"
+declare -r solution_dir="$(realpath "${2%/}")"
+declare -r output_dir="$(realpath "${3%/}")"
+declare -r tap_file="${output_dir}/tap.json"
+declare -r results_file="${output_dir}/results.json"
+declare -r test_file="${solution_dir}/TestCases.pas"
 
 # Create the output directory if it doesn't exist
-mkdir -p "${output_dir}"
+mkdir -p "$output_dir"
 
 echo "${slug}: testing..."
 
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
-
-# Write the results.json file based on the exit code of the command that was 
-# just executed that tested the implementation file
-if [ $? -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
-else
-    # OPTIONAL: Sanitize the output
-    # In some cases, the test output might be overly verbose, in which case stripping
-    # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
-
-    # OPTIONAL: Manually add colors to the output to help scanning the output for errors
-    # If the test output does not contain colors to help identify failing (or passing)
-    # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
-    #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
-    #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
-
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
-fi
+# Run the tests and generete results
+cd "$solution_dir" || exit 1
+
+make test=all 2>&1 | tap-parser -j 0 > "$tap_file"
+
+"${bin_dir}/results-generator.sh" "$test_file" "$tap_file" > "$results_file"
 
 echo "${slug}: done"

tests/all-fail/AllFail.pas

@@ -0,0 +1,20 @@
+unit AllFail;
+
+{$mode ObjFPC}{$H+}
+
+interface
+
+function abbreviate(const phrase: string) : string;
+
+implementation
+
+uses SysUtils;
+
+function abbreviate(const phrase: string) : string;
+begin
+
+  result := phrase;
+
+end;
+
+end.

tests/all-fail/Makefile

@@ -0,0 +1,15 @@
+SHELL       = /bin/bash
+MAKEFLAGS  += --no-print-directory
+DESTDIR     = build
+EXECUTABLE  = $(DESTDIR)/test
+COMMAND     = fpc -l- -v0 -Sehnw -Fu./lib test.pas -FE"./$(DESTDIR)"
+
+.ONESHELL:
+
+test:
+	@mkdir -p "./$(DESTDIR)"
+	@cp -r ./lib "./$(DESTDIR)"
+	@$(COMMAND) && ./$(EXECUTABLE) $(test)
+
+clean:
+	@rm -fr "./$(DESTDIR)"

tests/all-fail/TestCases.pas

@@ -0,0 +1,89 @@
+unit TestCases;
+
+{$mode ObjFPC}{$H+}
+
+interface
+
+uses Classes, SysUtils, FPCUnit, TestRegistry, FPCUnitTestUtils;
+
+type
+  AllFailTest = class(TTestCase)
+  published
+    procedure basic;
+    procedure lowercase_words;
+    procedure punctuation;
+    procedure all_caps_word;
+    procedure punctuation_without_whitespace;
+    procedure very_long_abbreviation;
+    procedure consecutive_delimiters;
+    procedure apostrophes;
+    procedure underscore_emphasis;
+  end;
+
+implementation
+
+uses AllFail;
+
+// 1e22cceb-c5e4-4562-9afe-aef07ad1eaf4
+procedure AllFailTest.basic;
+begin
+  TapAssertTrue(
+    Self,
+    'basic',
+    'PNG',
+    AllFail.abbreviate('Portable Network Graphics')
+  );
+end;
+
+// 79ae3889-a5c0-4b01-baf0-232d31180c08
+procedure AllFailTest.lowercase_words;
+begin
+  TapAssertTrue(Self, 'lowercase words', 'ROR', AllFail.abbreviate('Ruby on Rails'));
+end;
+
+// ec7000a7-3931-4a17-890e-33ca2073a548
+procedure AllFailTest.punctuation;
+begin
+  TapAssertTrue(Self, 'punctuation', 'FIFO', AllFail.abbreviate('First In, First Out'));
+end;
+
+// 32dd261c-0c92-469a-9c5c-b192e94a63b0
+procedure AllFailTest.all_caps_word;
+begin
+  TapAssertTrue(Self, 'all caps word', 'GIMP', AllFail.abbreviate('GNU Image Manipulation Program'));
+end;
+
+// ae2ac9fa-a606-4d05-8244-3bcc4659c1d4
+procedure AllFailTest.punctuation_without_whitespace;
+begin
+  TapAssertTrue(Self, 'punctuation without whitespace', 'CMOS', AllFail.abbreviate('Complementary metal-oxide semiconductor'));
+end;
+
+// 0e4b1e7c-1a6d-48fb-81a7-bf65eb9e69f9
+procedure AllFailTest.very_long_abbreviation;
+begin
+  TapAssertTrue(Self, 'very long abbreviation', 'ROTFLSHTMDCOALM', AllFail.abbreviate('Rolling On The Floor Laughing So Hard That My Dogs Came Over And Licked Me'));
+end;
+
+// 6a078f49-c68d-4b7b-89af-33a1a98c28cc
+procedure AllFailTest.consecutive_delimiters;
+begin
+  TapAssertTrue(Self, 'consecutive delimiters', 'SIMUFTA', AllFail.abbreviate('Something - I made up from thin air'));
+end;
+
+// 5118b4b1-4572-434c-8d57-5b762e57973e
+procedure AllFailTest.apostrophes;
+begin
+  TapAssertTrue(Self, 'apostrophes', 'HC', AllFail.abbreviate('Halley''s Comet'));
+end;
+
+// adc12eab-ec2d-414f-b48c-66a4fc06cdef
+procedure AllFailTest.underscore_emphasis;
+begin
+  TapAssertTrue(Self, 'underscore emphasis', 'TRNT', AllFail.abbreviate('The Road _Not_ Taken'));
+end;
+
+initialization
+RegisterTest(AllFailTest);
+
+end.