Skip to content

Commit b26aad0

Browse files
jimmyttyIsaacGglennj
authored
First Release (#4)
* * first-release * Update bin/results-generator.sh Co-authored-by: Isaac Good <IsaacG@users.noreply.github.com> * Update bin/results-generator.sh Co-authored-by: Isaac Good <IsaacG@users.noreply.github.com> * Update bin/results-generator.sh Co-authored-by: Isaac Good <IsaacG@users.noreply.github.com> * Update bin/run-tests.sh Co-authored-by: Isaac Good <IsaacG@users.noreply.github.com> * Update bin/run-tests.sh Co-authored-by: Isaac Good <IsaacG@users.noreply.github.com> * * indentation * * changing condition to jq all() syntax * * changing pushd/popd to subprocess * Update bin/results-generator.sh Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com> * Update bin/results-generator.sh Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com> * * make sure tap-file is created * * don't infuriate students! * * improving associative array to json conversion * * don't cleanup buld directory * * clean target * * regex shell expansion * Update bin/run.sh Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com> * Update bin/results-generator.sh Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com> * * append a linebreak * * ansi-c regex * * buildir * * add tap.json * * remove tojson * Update bin/results-generator.sh Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com> * Update bin/results-generator.sh Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com> * Update bin/results-generator.sh Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com> * Update bin/results-generator.sh Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com> --------- Co-authored-by: Isaac Good <IsaacG@users.noreply.github.com> Co-authored-by: Glenn Jackman <glenn.jackman@gmail.com>
1 parent 611650d commit b26aad0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+6297
-50
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
/tests/**/*/results.json
2+
/tests/*/build/

‎Dockerfile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
FROM alpine:3.18
1+
FROM alpine:3.22
22

33
# install packages required to run the tests
4-
RUN apk add --no-cache jq coreutils
4+
RUN apk add --no-cache jq coreutils bash binutils make npm
5+
RUN apk add fpc --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
6+
7+
RUN npm install -g tap-parser
58

69
WORKDIR /opt/test-runner
710
COPY . .

‎bin/results-generator.sh‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
#!/usr/bin/env bash
2+
3+
declare -r test_file="$1"
4+
declare -r tap_file="$2"
5+
declare -A test_codes=()
6+
7+
extract_test_codes () {
8+
local state='out'
9+
local proc_re='^procedure[[:blank:]]+[0-9A-Za-z_]+\.([0-9A-Za-z_]+);$'
10+
local end_re='^end;$'
11+
local assert_re=$'^[^\n]+\nbegin\n *(TapAssert.*);\nend;$'
12+
local name
13+
local body
14+
while IFS= read -r line; do
15+
if [[ "$state" == 'in' ]]; then
16+
printf -v body $'%s\n%s' "$body" "$line"
17+
if [[ "$line" =~ $end_re ]]; then
18+
state='out'
19+
shopt -s nocasematch
20+
if [[ "$body" =~ $assert_re ]]; then
21+
test_codes["$name"]="${BASH_REMATCH[1]}"
22+
else
23+
echo 'parser error'
24+
exit 1
25+
fi
26+
shopt -u nocasematch
27+
fi
28+
elif [[ "$line" =~ $proc_re ]]; then
29+
state='in'
30+
name="${BASH_REMATCH[1]}"
31+
body="$line"
32+
fi
33+
done < "$test_file"
34+
}
35+
36+
tap_parser() {
37+
local json_test_codes={}
38+
for key in "${!test_codes[@]}"; do
39+
json_test_codes=$(
40+
jq -cn \
41+
--argjson json "$json_test_codes" \
42+
--arg key "$key" \
43+
--arg val "${test_codes["$key"]}" \
44+
'$json + {$key: $val}'
45+
)
46+
done
47+
local -a tap_content
48+
tap_content=$(< "$tap_file")
49+
local -r status="$(jq -r '
50+
map(select(.[0] == "assert")) as $asserts |
51+
if ($asserts | length) > 0 and all($asserts[] | .[1].ok) then
52+
"pass"
53+
else
54+
"fail"
55+
end
56+
' <<< "${tap_content[0]}"
57+
)"
58+
59+
if [[ "$status" != "pass" ]] && \
60+
jq -e \
61+
'.[] |
62+
select(.[0] == "plan" and .[1].comment == "no tests found") |
63+
length > 0' <<< "${tap_content[0]}" &>/dev/null;
64+
then
65+
jq -r '
66+
{
67+
"version": 3,
68+
"status" : "error",
69+
"message": (map(select(.[0] == "extra") | .[1]) | join("")[0:65535])
70+
}' <<< "${tap_content[0]}"
71+
else
72+
local -i i=0
73+
local extra=''
74+
local -a json_arrays
75+
while IFS= read -r line; do
76+
if jq -e '.[0] == "extra"' <<< "$line" &>/dev/null; then
77+
extra+=$(jq -r '.[1]' <<< "$line")
78+
extra+=$'\n'
79+
elif jq -e '.[0] == "assert"' <<< "$line" &>/dev/null; then
80+
if (( ${#extra} > 500 )); then
81+
extra="${extra:0:451}[Output was truncated. Please limit to 500 chars]"
82+
fi
83+
(( i++ ))
84+
json_arrays+=("$(
85+
jq -r --arg extra "$extra" \
86+
'[.[0],
87+
(.[1] +
88+
{ "output":
89+
if $extra == "" then null else $extra end })]' \
90+
<<< "$line"
91+
)")
92+
extra=''
93+
else
94+
json_arrays+=("$line")
95+
fi
96+
done < <(jq -c '.[]' <<< "${tap_content[0]}")
97+
printf '%s\n' "${json_arrays[@]}" |
98+
jq --slurp \
99+
--arg status "$status" \
100+
--argjson test_codes "$json_test_codes" \
101+
'
102+
{
103+
"version": 3,
104+
"status" : $status,
105+
"message": null
106+
} + {
107+
"tests": [
108+
.[] | select(.[0] == "assert") | .[1] |
109+
if .name == "Please implement your solution." then
110+
{ "name": .name, status: "error", test_code: "", message: .name }
111+
elif .ok == true then
112+
{ "name": .name, status: "pass" }
113+
else
114+
{
115+
"name": .diag.message,
116+
"status": .diag.severity,
117+
"output": .output,
118+
"test_code": $test_codes[.name],
119+
"message": "GOT:" + (.diag.data.got|tostring) + "\n" +
120+
"EXPECTED:" + (.diag.data.expect|tostring),
121+
}
122+
end
123+
]
124+
}
125+
'
126+
fi
127+
}
128+
129+
extract_test_codes
130+
tap_parser

‎bin/run-tests.sh‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env sh
22

33
# Synopsis:
4-
# Test the test runner by running it against a predefined set of solutions
4+
# Test the test runner by running it against a predefined set of solutions
55
# with an expected output.
66

77
# Output:
@@ -21,17 +21,25 @@ for test_dir in tests/*; do
2121
bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
2222

2323
# OPTIONAL: Normalize the results file
24-
# If the results.json file contains information that changes between
25-
# different test runs (e.g. timing information or paths), you should normalize
26-
# the results file to allow the diff comparison below to work as expected
24+
# If the results.json file contains information that changes between
25+
# different test runs (e.g. timing information or paths), you should
26+
# normalize the results file to allow the diff comparison below to work
27+
# as expected
2728

2829
file="results.json"
2930
expected_file="expected_${file}"
31+
sed -E -i \
32+
-e "s~${test_dir_path}~/solution~g" \
33+
-e "s~${test_dir}~/solution~g" \
34+
-e 's~/[[:alnum:][:punct:]]+/bin/ppcx64~/usr/bin/ppcx64~' \
35+
"${test_dir_path}/${file}"
3036
echo "${test_dir_name}: comparing ${file} to ${expected_file}"
3137

32-
if ! diff "${test_dir_path}/${file}" "${test_dir_path}/${expected_file}"; then
38+
actual_file="${test_dir_path}/${file}"
39+
expected_file="${test_dir_path}/${expected_file}"
40+
if ! diff "$actual_file" "$expected_file"; then
3341
exit_code=1
3442
fi
3543
done
3644

37-
exit ${exit_code}
45+
exit "${exit_code}"

‎bin/run.sh‎

Lines changed: 15 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/usr/bin/env sh
1+
#!/usr/bin/env bash
22

33
# Synopsis:
44
# Run the test runner on a solution.
@@ -21,40 +21,24 @@ if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
2121
exit 1
2222
fi
2323

24-
slug="$1"
25-
solution_dir=$(realpath "${2%/}")
26-
output_dir=$(realpath "${3%/}")
27-
results_file="${output_dir}/results.json"
24+
declare -r bin_dir="$(dirname $(realpath "$0"))"
25+
declare -r slug="$1"
26+
declare -r solution_dir="$(realpath "${2%/}")"
27+
declare -r output_dir="$(realpath "${3%/}")"
28+
declare -r tap_file="${output_dir}/tap.json"
29+
declare -r results_file="${output_dir}/results.json"
30+
declare -r test_file="${solution_dir}/TestCases.pas"
2831

2932
# Create the output directory if it doesn't exist
30-
mkdir -p "${output_dir}"
33+
mkdir -p "$output_dir"
3134

3235
echo "${slug}: testing..."
3336

34-
# Run the tests for the provided implementation file and redirect stdout and
35-
# stderr to capture it
36-
test_output=$(false)
37-
# TODO: substitute "false" with the actual command to run the test:
38-
# test_output=$(command_to_run_tests 2>&1)
39-
40-
# Write the results.json file based on the exit code of the command that was
41-
# just executed that tested the implementation file
42-
if [ $? -eq 0 ]; then
43-
jq -n '{version: 1, status: "pass"}' > ${results_file}
44-
else
45-
# OPTIONAL: Sanitize the output
46-
# In some cases, the test output might be overly verbose, in which case stripping
47-
# the unneeded information can be very helpful to the student
48-
# sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
49-
50-
# OPTIONAL: Manually add colors to the output to help scanning the output for errors
51-
# If the test output does not contain colors to help identify failing (or passing)
52-
# tests, it can be helpful to manually add colors to the output
53-
# colorized_test_output=$(echo "${test_output}" \
54-
# | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
55-
# | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
56-
57-
jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
58-
fi
37+
# Run the tests and generete results
38+
cd "$solution_dir" || exit 1
39+
40+
make test=all 2>&1 | tap-parser -j 0 > "$tap_file"
41+
42+
"${bin_dir}/results-generator.sh" "$test_file" "$tap_file" > "$results_file"
5943

6044
echo "${slug}: done"

‎tests/all-fail/AllFail.pas‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
unit AllFail;
2+
3+
{$mode ObjFPC}{$H+}
4+
5+
interface
6+
7+
function abbreviate(const phrase: string) : string;
8+
9+
implementation
10+
11+
uses SysUtils;
12+
13+
function abbreviate(const phrase: string) : string;
14+
begin
15+
16+
result := phrase;
17+
18+
end;
19+
20+
end.

‎tests/all-fail/Makefile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
SHELL = /bin/bash
2+
MAKEFLAGS += --no-print-directory
3+
DESTDIR = build
4+
EXECUTABLE = $(DESTDIR)/test
5+
COMMAND = fpc -l- -v0 -Sehnw -Fu./lib test.pas -FE"./$(DESTDIR)"
6+
7+
.ONESHELL:
8+
9+
test:
10+
@mkdir -p "./$(DESTDIR)"
11+
@cp -r ./lib "./$(DESTDIR)"
12+
@$(COMMAND) && ./$(EXECUTABLE) $(test)
13+
14+
clean:
15+
@rm -fr "./$(DESTDIR)"

‎tests/all-fail/TestCases.pas‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
unit TestCases;
2+
3+
{$mode ObjFPC}{$H+}
4+
5+
interface
6+
7+
uses Classes, SysUtils, FPCUnit, TestRegistry, FPCUnitTestUtils;
8+
9+
type
10+
AllFailTest = class(TTestCase)
11+
published
12+
procedure basic;
13+
procedure lowercase_words;
14+
procedure punctuation;
15+
procedure all_caps_word;
16+
procedure punctuation_without_whitespace;
17+
procedure very_long_abbreviation;
18+
procedure consecutive_delimiters;
19+
procedure apostrophes;
20+
procedure underscore_emphasis;
21+
end;
22+
23+
implementation
24+
25+
uses AllFail;
26+
27+
// 1e22cceb-c5e4-4562-9afe-aef07ad1eaf4
28+
procedure AllFailTest.basic;
29+
begin
30+
TapAssertTrue(
31+
Self,
32+
'basic',
33+
'PNG',
34+
AllFail.abbreviate('Portable Network Graphics')
35+
);
36+
end;
37+
38+
// 79ae3889-a5c0-4b01-baf0-232d31180c08
39+
procedure AllFailTest.lowercase_words;
40+
begin
41+
TapAssertTrue(Self, 'lowercase words', 'ROR', AllFail.abbreviate('Ruby on Rails'));
42+
end;
43+
44+
// ec7000a7-3931-4a17-890e-33ca2073a548
45+
procedure AllFailTest.punctuation;
46+
begin
47+
TapAssertTrue(Self, 'punctuation', 'FIFO', AllFail.abbreviate('First In, First Out'));
48+
end;
49+
50+
// 32dd261c-0c92-469a-9c5c-b192e94a63b0
51+
procedure AllFailTest.all_caps_word;
52+
begin
53+
TapAssertTrue(Self, 'all caps word', 'GIMP', AllFail.abbreviate('GNU Image Manipulation Program'));
54+
end;
55+
56+
// ae2ac9fa-a606-4d05-8244-3bcc4659c1d4
57+
procedure AllFailTest.punctuation_without_whitespace;
58+
begin
59+
TapAssertTrue(Self, 'punctuation without whitespace', 'CMOS', AllFail.abbreviate('Complementary metal-oxide semiconductor'));
60+
end;
61+
62+
// 0e4b1e7c-1a6d-48fb-81a7-bf65eb9e69f9
63+
procedure AllFailTest.very_long_abbreviation;
64+
begin
65+
TapAssertTrue(Self, 'very long abbreviation', 'ROTFLSHTMDCOALM', AllFail.abbreviate('Rolling On The Floor Laughing So Hard That My Dogs Came Over And Licked Me'));
66+
end;
67+
68+
// 6a078f49-c68d-4b7b-89af-33a1a98c28cc
69+
procedure AllFailTest.consecutive_delimiters;
70+
begin
71+
TapAssertTrue(Self, 'consecutive delimiters', 'SIMUFTA', AllFail.abbreviate('Something - I made up from thin air'));
72+
end;
73+
74+
// 5118b4b1-4572-434c-8d57-5b762e57973e
75+
procedure AllFailTest.apostrophes;
76+
begin
77+
TapAssertTrue(Self, 'apostrophes', 'HC', AllFail.abbreviate('Halley''s Comet'));
78+
end;
79+
80+
// adc12eab-ec2d-414f-b48c-66a4fc06cdef
81+
procedure AllFailTest.underscore_emphasis;
82+
begin
83+
TapAssertTrue(Self, 'underscore emphasis', 'TRNT', AllFail.abbreviate('The Road _Not_ Taken'));
84+
end;
85+
86+
initialization
87+
RegisterTest(AllFailTest);
88+
89+
end.

0 commit comments

Comments
 (0)