First Release

Dockerfile

@@ -1,7 +1,10 @@
-FROM alpine:3.18
+FROM alpine:3.22
 
 # install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+RUN apk add --no-cache jq coreutils bash binutils make npm
+RUN apk add fpc --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+RUN npm install -g tap-parser
 
 WORKDIR /opt/test-runner
 COPY . .

bin/results-generator.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+
+declare -r test_file="$1"
+declare -r tap_file="$2"
+declare -A test_codes=()
+
+extract_test_codes () {
+    local -r lf=$'\n'
+    local test_name
+    local state='out'
+    local proc_re='^procedure[[:blank:]]+[0-9A-Za-z_]+\.([0-9A-Za-z_]+);$'
+    local end_re='^end;$'
+    local assert_re='^[^\n]+\nbegin\n *(TapAssert.*);\nend;$'
+    local name
+    local body
+    while IFS= read -r line; do
+        if [[ "$state" == 'in' ]]; then
+            printf -v body $'%s\n%s' "$body" "$line"
+            if [[ "$line" =~ $end_re ]]; then
+                state='out'
+                shopt -s nocasematch
+                if [[ "$body" =~ ${assert_re//\\n/${lf}} ]]; then
+                    test_codes["$name"]="${BASH_REMATCH[1]//${lf}/\\n}"
+                else
+                    echo 'parser error'
+                    exit 1
+                fi
+                shopt -u nocasematch
+            fi
+        elif [[ "$line" =~ $proc_re ]]; then
+            state='in'
+            name="${BASH_REMATCH[1]}"
+            body="$line"
+        fi
+    done < "$test_file"
+}
+
+tap_parser() {
+    local -r json_test_codes="$(
+      for key in ${!test_codes[*]}; do
+        printf '"%s"' "$key"
+        printf '"%s"' "${test_codes[$key]}"
+      done |
+      jq -n 'reduce inputs as $i ({}; . + { ($i): input })'
+    )"
+    local -a tap_content
+    mapfile -d '' tap_content < "$tap_file"
+    local -r status="$(jq -r '
+      [
+        .[] |
+        select(.[0] == "assert")
+      ] as $asserts |
+      if ($asserts | length) > 0 and
+        ($asserts | length) ==
+        ([$asserts[] | select(.[1].ok == true)] | length)
+      then
+        "pass"
+      else
+        "fail"
+      end
+    ' <<< "${tap_content[0]}"
+    )"
+
+    if [[ "$status" != "pass" ]] && \
+           jq -e \
+              '.[] |
+              select(.[0] == "plan" and .[1].comment == "no tests found") |
+              length > 0' <<< "${tap_content[0]}" &>/dev/null;
+    then
+        jq -r '
+        {
+        "version": 3,
+        "status" : "error",
+        "message": ([.[] | select(.[0] == "extra") | .[1]] | join("")[0:65535]),
+        }' <<< "${tap_content[0]}"
+    else
+        local -i i=0
+        local extra=''
+        local -a json_arrays
+        while IFS= read -r line; do
+            if jq -e '.[0] == "extra"' <<< "$line" &>/dev/null; then
+                printf -v extra '%s%s\n' "$extra" "$(jq -r '.[1]' <<< "$line")"
+                continue
+            fi
+            if jq -e '.[0] == "assert"' <<< "$line" &>/dev/null; then
+                if (( ${#extra} > 500 )); then
+                    extra='Output was truncated. Please limit to 500 chars'
+                fi
+                (( i++ ))
+                json_arrays+=("$(
+                  jq -r --arg extra "$extra" \
+                    '[.[0],
+                     (.[1] +
+                     { "output":
+                       if $extra == "" then null else $extra end })]' \
+                      <<< "$line"
+                  )")
+                extra=''
+            else
+                json_arrays+=("$line")
+            fi
+        done < <(jq -r '.[] | tojson' <<< "${tap_content[0]}")
+        local json_string
+        printf -v json_string '%s,' "${json_arrays[@]}"
+        json_string="${json_string%,}"
+        printf -v json_string '[%s]' "$json_string"
+        tap_content[0]="$(jq -r . <<< "$json_string")"
+
+        jq -r \
+           --arg status "$status" \
+           --argjson test_codes "$json_test_codes" \
+           '
+    {
+    "version": 3,
+    "status" : $status,
+    "message": null,
+    } + {
+    "tests": [
+      .[] | select(.[0] == "assert") | .[1] |
+      if .name == "Please implement your solution." then
+        { "name": .name, status: "error", test_code: "", message: .name }
+      elif .ok == true then
+        { "name": .name, status: "pass" }
+      else
+        {
+          "name": .diag.message,
+          "status": .diag.severity,
+          "output": .output,
+          "test_code": $test_codes[.name],
+          "message": "GOT:"      + (.diag.data.got|tostring) + "\n" +
+                     "EXPECTED:" + (.diag.data.expect|tostring),
+        }
+      end
+    ]
+  }
+  ' <<< "${tap_content[0]}"
+    fi
+}
+
+extract_test_codes
+tap_parser

bin/run-tests.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 
 # Synopsis:
-# Test the test runner by running it against a predefined set of solutions 
+# Test the test runner by running it against a predefined set of solutions
 # with an expected output.
 
 # Output:
@@ -21,17 +21,32 @@ for test_dir in tests/*; do
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
     # OPTIONAL: Normalize the results file
-    # If the results.json file contains information that changes between 
-    # different test runs (e.g. timing information or paths), you should normalize
-    # the results file to allow the diff comparison below to work as expected
+    # If the results.json file contains information that changes between
+    # different test runs (e.g. timing information or paths), you should
+    # normalize the results file to allow the diff comparison below to work
+    # as expected
 
     file="results.json"
     expected_file="expected_${file}"
+    sed -E -i \
+        -e "s~${test_dir_path}~/solution~g" \
+        -e "s~${test_dir}~/solution~g" \
+        -e 's~/[[:alnum:][:punct:]]+/bin/ppcx64~/usr/bin/ppcx64~' \
+        "${test_dir_path}/${file}"
     echo "${test_dir_name}: comparing ${file} to ${expected_file}"
 
-    if ! diff "${test_dir_path}/${file}" "${test_dir_path}/${expected_file}"; then
+    actual_file="${test_dir_path}/${file}"
+    expected_file="${test_dir_path}/${expected_file}"
+    if ! diff  "$actual_file" "$expected_file"; then
         exit_code=1
     fi
 done
 
+find ./tests \
+    -maxdepth 2 \
+    -type f \
+    -name '*.json' \
+    -not -name expected_results.json \
+    -exec rm {} +
+
 exit ${exit_code}

bin/run.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 
 # Synopsis:
 # Run the test runner on a solution.
@@ -21,40 +21,24 @@ if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
     exit 1
 fi
 
-slug="$1"
-solution_dir=$(realpath "${2%/}")
-output_dir=$(realpath "${3%/}")
-results_file="${output_dir}/results.json"
+declare -r bin_dir="$(dirname $(realpath "$0"))"
+declare -r slug="$1"
+declare -r solution_dir="$(realpath "${2%/}")"
+declare -r output_dir="$(realpath "${3%/}")"
+declare -r tap_file="${output_dir}/tap.json"
+declare -r results_file="${output_dir}/results.json"
+declare -r test_file="${solution_dir}/TestCases.pas"
 
 # Create the output directory if it doesn't exist
-mkdir -p "${output_dir}"
+mkdir -p "$output_dir"
 
 echo "${slug}: testing..."
 
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
-
-# Write the results.json file based on the exit code of the command that was 
-# just executed that tested the implementation file
-if [ $? -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
-else
-    # OPTIONAL: Sanitize the output
-    # In some cases, the test output might be overly verbose, in which case stripping
-    # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
-
-    # OPTIONAL: Manually add colors to the output to help scanning the output for errors
-    # If the test output does not contain colors to help identify failing (or passing)
-    # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
-    #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
-    #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
-
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
-fi
+# Run the tests and generete results
+cd "$solution_dir"
+pushd "$solution_dir" >/dev/null || exit
+make test=all 2>&1 | tap-parser -j 0 > "$tap_file"
+popd >/dev/null || exit
+"${bin_dir}/results-generator.sh" "$test_file" "$tap_file" > "$results_file"
 
 echo "${slug}: done"

tests/all-fail/AllFail.pas

@@ -0,0 +1,20 @@
+unit AllFail;
+
+{$mode ObjFPC}{$H+}
+
+interface
+
+function abbreviate(const phrase: string) : string;
+
+implementation
+
+uses SysUtils;
+
+function abbreviate(const phrase: string) : string;
+begin
+
+  result := phrase;
+
+end;
+
+end.

tests/all-fail/Makefile

@@ -0,0 +1,17 @@
+SHELL       = /bin/bash
+MAKEFLAGS  += --no-print-directory
+DESTDIR     = build
+EXECUTABLE  = $(DESTDIR)/test
+COMMAND     = fpc -l- -v0 -Sehnw -Fu./lib test.pas -FE"./$(DESTDIR)"
+
+.ONESHELL:
+
+test:
+	@$(MAKE) clean
+	@mkdir -p "./$(DESTDIR)"
+	@cp -r ./lib "./$(DESTDIR)"
+	@$(COMMAND) && ./$(EXECUTABLE) $(test) || $(MAKE) clean
+	@$(MAKE) clean
+
+clean:
+	@rm -fr "./$(DESTDIR)"

tests/all-fail/TestCases.pas

@@ -0,0 +1,89 @@
+unit TestCases;
+
+{$mode ObjFPC}{$H+}
+
+interface
+
+uses Classes, SysUtils, FPCUnit, TestRegistry, FPCUnitTestUtils;
+
+type
+  AllFailTest = class(TTestCase)
+  published
+    procedure basic;
+    procedure lowercase_words;
+    procedure punctuation;
+    procedure all_caps_word;
+    procedure punctuation_without_whitespace;
+    procedure very_long_abbreviation;
+    procedure consecutive_delimiters;
+    procedure apostrophes;
+    procedure underscore_emphasis;
+  end;
+
+implementation
+
+uses AllFail;
+
+// 1e22cceb-c5e4-4562-9afe-aef07ad1eaf4
+procedure AllFailTest.basic;
+begin
+  TapAssertTrue(
+    Self,
+    'basic',
+    'PNG',
+    AllFail.abbreviate('Portable Network Graphics')
+  );
+end;
+
+// 79ae3889-a5c0-4b01-baf0-232d31180c08
+procedure AllFailTest.lowercase_words;
+begin
+  TapAssertTrue(Self, 'lowercase words', 'ROR', AllFail.abbreviate('Ruby on Rails'));
+end;
+
+// ec7000a7-3931-4a17-890e-33ca2073a548
+procedure AllFailTest.punctuation;
+begin
+  TapAssertTrue(Self, 'punctuation', 'FIFO', AllFail.abbreviate('First In, First Out'));
+end;
+
+// 32dd261c-0c92-469a-9c5c-b192e94a63b0
+procedure AllFailTest.all_caps_word;
+begin
+  TapAssertTrue(Self, 'all caps word', 'GIMP', AllFail.abbreviate('GNU Image Manipulation Program'));
+end;
+
+// ae2ac9fa-a606-4d05-8244-3bcc4659c1d4
+procedure AllFailTest.punctuation_without_whitespace;
+begin
+  TapAssertTrue(Self, 'punctuation without whitespace', 'CMOS', AllFail.abbreviate('Complementary metal-oxide semiconductor'));
+end;
+
+// 0e4b1e7c-1a6d-48fb-81a7-bf65eb9e69f9
+procedure AllFailTest.very_long_abbreviation;
+begin
+  TapAssertTrue(Self, 'very long abbreviation', 'ROTFLSHTMDCOALM', AllFail.abbreviate('Rolling On The Floor Laughing So Hard That My Dogs Came Over And Licked Me'));
+end;
+
+// 6a078f49-c68d-4b7b-89af-33a1a98c28cc
+procedure AllFailTest.consecutive_delimiters;
+begin
+  TapAssertTrue(Self, 'consecutive delimiters', 'SIMUFTA', AllFail.abbreviate('Something - I made up from thin air'));
+end;
+
+// 5118b4b1-4572-434c-8d57-5b762e57973e
+procedure AllFailTest.apostrophes;
+begin
+  TapAssertTrue(Self, 'apostrophes', 'HC', AllFail.abbreviate('Halley''s Comet'));
+end;
+
+// adc12eab-ec2d-414f-b48c-66a4fc06cdef
+procedure AllFailTest.underscore_emphasis;
+begin
+  TapAssertTrue(Self, 'underscore emphasis', 'TRNT', AllFail.abbreviate('The Road _Not_ Taken'));
+end;
+
+initialization
+RegisterTest(AllFailTest);
+
+end.