Detect extra test by group names (#3)

* Detect extra test by group names

* Update readme

Author: cyrilwanner

README.md

@@ -8,13 +8,16 @@ This package provides Jest helpers for usage in a skills competition environment
 
 - [Installation](#installation)
 - [Usage](#usage)
+  - [Grouping](#grouping)
+  - [Extra tests](#extra-tests)
 - [License](#license)
 
 ## Installation
 
 **Requirements:**
 
 - Node `22` or greater
+- Jest `30` or greater
 
 To install this package, run the following command:
 
@@ -26,8 +29,8 @@ It is suggested to add the following npm scripts:
 
 ```json
 "scripts": {
-"test": "jest",
-"test:json": "cross-env SKILLS17_JSON=true jest"
+  "test": "jest",
+  "test:json": "cross-env SKILLS17_JSON=true jest"
 },
 ```
 
@@ -61,6 +64,66 @@ folder of your task, next to the `package.json` file.
 See the [`@skills17/task-config`](https://github.com/skills17/task-config#configuration) package for a detailed
 description of all available properties in the `config.yaml` file.
 
+### Grouping
+
+A core concept is test groups. You usually don't want to test everything for one criterion in one
+test function but instead split it into multiple ones for a cleaner test class and a better overview.
+
+In JS, tests are grouped by a test name prefix defined in the `config.yaml` file.
+
+All `describe`s are concatenated with the actual test names before evaluation.
+
+For example, the following test will have the name `Countries > Overview > lists all countries`:
+
+```typescript
+describe('Countries', () => {
+  describe('Overview', () => {
+    it('lists all countries', () => {
+      // ...
+    });
+  });
+});
+```
+
+To catch and group all tests within the `Overview` description, the group matcher can be set to
+`Countries > Overview > .+` for example. Each of the tests within that group will now award 1 point
+to the group.
+
+### Extra tests
+
+To prevent cheating, extra tests can be used.
+They are not available to the competitors and should test the same things as the normal tests do,
+but with different values.
+
+For example, if your normal test contains a check to search the list of all countries by 'Sw*', copy
+the test into an extra test and change the search string to 'Ca*'.
+Since the competitors will not know the extra test, it would detect statically returned values that
+were returned to satisfy the 'Sw*' tests instead of actually implement the search logic.
+
+Extra tests are detected by their `describe`, which should equal `'Extra'` or `'extra'`. That means
+that you can wrap your test in an additional extra `describe` like shown below. The other
+`describe`s and test names should equal the ones from the normal tests. If they don't, a warning
+will be displayed.
+
+```typescript
+describe('Extra', () => {    // <-- only this describe has to be added
+  describe('Countries', () => {
+    it('lists all countries', () => {
+      // ...
+    });
+  });
+});
+```
+
+It usually makes sense to move the extra tests in a separate folder, so the folder can be deleted
+before the tasks are distributed to the competitors.
+Nothing else needs to be done or configured.
+
+If an extra test fails while the corresponding normal test passes, a warning will be displayed that
+a manual review of that test is required since it detected possible cheating.
+The penalty then has to be decided manually from case to case, the points visible in the output
+assumed that the test passed and there was no cheating.
+
 ## License
 
 [MIT](https://github.com/skills17/jest-helpers/blob/master/LICENSE)

src/skills17-reporter.ts

@@ -43,10 +43,11 @@ export default class Skills17Reporter implements Reporter {
 	}
 
 	onTestResult(test: Test, testResult: TestResult, _aggregatedResults: AggregatedResult): void {
-		const isExtra = test.path.includes('/extra/');
+		const isExtraPath = test.path.includes('/extra/');
 		for (const result of testResult.testResults) {
+			const isExtra = isExtraPath || result.ancestorTitles.map(title => title.toLowerCase()).includes('extra');
 			this.testRun?.recordTest(
-				`${result.ancestorTitles.join(' > ')} > ${result.title}`,
+				`${result.ancestorTitles.filter(title => title.toLowerCase() !== 'extra').join(' > ')} > ${result.title}`,
 				result.title,
 				isExtra,
 				result.status === 'passed',

tests/integration/extra-tests-fail/config.yaml

@@ -0,0 +1,4 @@
+# yaml-language-server: $schema=https://schema.skills17.ch/task-config/v3/config.schema.json
+id: basic-functionality
+groups:
+  - match: example.+

tests/integration/extra-tests-fail/expected.json

@@ -0,0 +1,21 @@
+{
+  "testResults": [
+    {
+      "group": "example.+",
+      "points": 1,
+      "maxPoints": 1,
+      "strategy": "add",
+      "manualCheck": true,
+      "tests": [
+        {
+          "name": "adds 1 + 2 to equal 3",
+          "points": 1,
+          "maxPoints": 1,
+          "successful": true,
+          "required": false,
+          "manualCheck": true
+        }
+      ]
+    }
+  ]
+}

tests/integration/extra-tests-fail/expected.txt

@@ -0,0 +1,9 @@
+------------       RESULT       ------------
+
+Summary:
+  example.+: 1/1 point [manual check required]
+    ? adds 1 + 2 to equal 3 please check manually for static return values and/or logical errors
+
+Total: 1/1 point
+
+Info: The detailed test and error information is visible above the result summary.

tests/integration/extra-tests-fail/jest.config.ts

@@ -0,0 +1,14 @@
+import process from 'node:process';
+import {type JestConfigWithTsJest} from 'ts-jest';
+
+const jsonOnlyReport = Boolean(process.env.SKILLS17_JSON);
+
+const config: JestConfigWithTsJest = {
+	clearMocks: true,
+	reporters: jsonOnlyReport
+		? [['../../../lib/skills17-reporter', {json: jsonOnlyReport}]]
+		: ['../../../lib/skills17-reporter'],
+	testEnvironment: 'node',
+};
+
+export default config;

tests/integration/extra-tests-fail/src/example.js

@@ -0,0 +1,3 @@
+export default function sum(a, b) {
+	return a + b;
+}

tests/integration/extra-tests-fail/src/example.test.js

@@ -0,0 +1,17 @@
+import sum from './example.js';
+
+/* eslint-disable no-undef */
+describe('example', () => {
+	test('adds 1 + 2 to equal 3', () => {
+		expect(sum(1, 2)).toBe(3);
+	});
+});
+
+describe('extra', () => {
+	describe('example', () => {
+		test('adds 1 + 2 to equal 3', () => {
+			expect(sum(0, 0)).toBe(1);
+		});
+	});
+});
+/* eslint-enable no-undef */