Tests for the test suite (#468)

* Add first tests from process examples

* Add tests and docs

* Add more tests

* More tests, clean-up

* Tests for arrays

* Remove optional tests in favor of returns or throws

* EditorConfig + fix invalid files

* Add id and experimental flag to test files

* Add tests for comparison processes

* Add tests for reducers etc.

* Add further tests

* title converted to comments, fix parameter name of exp in tests

* Fix test for tan

* Add tests for array_apply, more NaN tests for comparisons, add required flag for sub-processes

* add array_apply and array_filter tests, fix several math tests

* update readme

* Add first datacube test

* Fix apply test

* Add profile levels

* Fix test in apply

* Add tests for reduce_dimension and apply_dimension

* Add more tests

* Fix various test cases

* Special no-data handling, fix tests

* Add filter_bands and filter_temporal

* Add mask, merge_cubes, refactor datacube object

* Mark experimental processes

* Add additional test for apply_dimension

* Add tests for apply_kernel

* Add tests for aggregate_temporal

* Add tests for aggregate_temporal_period, fix aggregate_temporal

* Add tests for filter_bbox, filter_spatial and mask_polygon

* Use nodata type in tests properly

* Add changelog entry

* Fix invalid datetimes

* Apply suggestions from code review

Co-authored-by: Stefaan Lippens <[email protected]>

* Add comment

* Add consistency checks for test files and fix issues

* Remove list of processes from README

* Update tests/README.md

Co-authored-by: Stefaan Lippens <[email protected]>

* Update tests/schema/schema.json

Co-authored-by: Stefaan Lippens <[email protected]>

* Update tests/README.md

* Update tests

---------

Co-authored-by: Stefaan Lippens <[email protected]>

Author: m-mohr

.editorconfig

@@ -7,3 +7,11 @@ indent_style = spaces
 indent_size = 4
 insert_final_newline = true
 trim_trailing_whitespace = true
+
+[*.json5]
+charset = utf-8
+end_of_line = crlf
+indent_style = spaces
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Implementation guide for implementing OGC API - Processes in openEO
+- Unit Tests (see folder `tests`, moved specification tests and CI tools to `dev`)
 - `date_difference`: Allow `week` as a unit [#506](https://github.com/Open-EO/openeo-processes/issues/506)
 - `export_collection`: New process
 - `export_workspace`: New process

absolute.json

@@ -28,12 +28,6 @@
         }
     },
     "examples": [
-        {
-            "arguments": {
-                "x": 0
-            },
-            "returns": 0
-        },
         {
             "arguments": {
                 "x": 3.5
@@ -45,12 +39,6 @@
                 "x": -0.4
             },
             "returns": 0.4
-        },
-        {
-            "arguments": {
-                "x": -3.5
-            },
-            "returns": 3.5
         }
     ],
     "links": [

apply_kernel.json

@@ -1,7 +1,7 @@
 {
     "id": "apply_kernel",
     "summary": "Apply a spatial convolution with a kernel",
-    "description": "Applies a 2D convolution (i.e. a focal operation with a weighted kernel) on the horizontal spatial dimensions (axes `x` and `y`) of a raster data cube.\n\nEach value in the kernel is multiplied with the corresponding pixel value and all products are summed up afterwards. The sum is then multiplied with the factor.\n\nThe process can't handle non-numerical or infinite numerical values in the data cube. Boolean values are converted to integers (`false` = 0, `true` = 1), but all other non-numerical or infinite values are replaced with zeroes by default (see parameter `replace_invalid`).\n\nFor cases requiring more generic focal operations or non-numerical values, see ``apply_neighborhood()``.",
+    "description": "Applies a 2D convolution (i.e. a focal operation with a weighted kernel) on the horizontal spatial dimensions (axes `x` and `y`) of a raster data cube.\n\nEach value in the kernel is multiplied with the corresponding pixel value and all products are summed up afterwards. The sum is then multiplied with the factor.\n\nThe process can't handle non-numerical or infinite numerical values in the data cube. Boolean values are converted to integers (`false` = 0, `true` = 1), but all other non-numerical, NaN, no-data, or infinite values are replaced with zeroes by default (see parameter `replace_invalid`).\n\nFor cases requiring more generic focal operations or non-numerical values, see ``apply_neighborhood()``.",
     "categories": [
         "cubes",
         "math > image filter"
@@ -70,7 +70,7 @@
         },
         {
             "name": "replace_invalid",
-            "description": "This parameter specifies the value to replace non-numerical or infinite numerical values with. By default, those values are replaced with zeroes.",
+            "description": "This parameter specifies the value to replace non-numerical, NaN, no-data, or infinite numerical values with. By default, those values are replaced with zeroes.",
             "schema": {
                 "type": "number"
             },

dev/check-tests.js

@@ -0,0 +1,68 @@
+// Ensure that each test is valid
+
+const fs = require('fs');
+const path = require('path');
+const JSON5 = require('json5');
+const ajv = require('ajv');
+
+const testsDir = path.join(__dirname, '../tests');
+
+const tests = fs.readdirSync(testsDir).filter(file => file.endsWith('.json5'));
+
+const schemaPath = path.join(testsDir, 'schema', 'schema.json');
+const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+const schema = JSON.parse(schemaContent);
+const validate = new ajv().compile(schema);
+
+const results = {};
+for (const testFile of tests) {
+    const testPath = path.join(testsDir, testFile);
+
+    let testData;
+    // Ensure we can load the test file as JSON5
+    try {
+        testData = JSON5.parse(fs.readFileSync(testPath, 'utf8'));
+    } catch (error) {
+        results[testFile] = `Invalid JSON5: ${error.message}`;
+        continue;
+    }
+
+    // Ensure the file is valid against the schema
+    if (!validate(testData)) {
+        results[testFile] = `Schema validation failed: ${validate.errors.map(err => err.message).join(', ')}`;
+        continue;
+    }
+
+    // Make sure the id is the same as filename without extension
+    const expectedId = path.basename(testFile, '.json5');
+    if (testData.id !== expectedId) {
+        results[testFile] = `ID mismatch: expected ${expectedId}, got ${testData.id}`;
+        continue;
+    }
+
+    // Check if experimental is set to the same value as in the process itself
+    let processFile = path.join(__dirname, '../', expectedId + '.json');
+    if (!fs.existsSync(processFile)) {
+        processFile = path.join(__dirname, '../proposals/', expectedId + '.json');
+    }
+    if (fs.existsSync(processFile)) {
+        const processData = JSON.parse(fs.readFileSync(processFile, 'utf8'));
+        const expected = processData.experimental || false;
+        const actual = testData.experimental || false;
+        if (expected !== actual) {
+            results[testFile] = `Experimental flag mismatch: expected ${expected}, got ${actual}`;
+            continue;
+        }
+    }
+}
+
+if (Object.keys(results).length > 0) {
+    console.error('The following test files have issues:');
+    for (const [file, error] of Object.entries(results)) {
+        console.error(`- ${file}: ${error}`);
+    }
+    process.exit(1);
+}
+else {
+    console.log('All test files are valid and match the expected schema.');
+}

dev/has-tests.js

@@ -0,0 +1,44 @@
+// Ensure that each process has a corresponding file in the tests directory.
+// It can be empty, but it must exist to ensure people made that decision consciously.
+
+const fs = require('fs');
+const path = require('path');
+
+const processesDir = path.join(__dirname, '../');
+const proposalsDir = path.join(__dirname, '../proposals');
+const testsDir = path.join(__dirname, '../tests');
+
+const processes = [
+    ...fs.readdirSync(processesDir),
+    ...fs.readdirSync(proposalsDir),
+]
+    .filter(file => file.endsWith('.json'))
+    .map(file => path.basename(file, '.json'));;
+const tests = fs.readdirSync(testsDir)
+    .filter(file => file.endsWith('.json5'))
+    .map(file => path.basename(file, '.json5'));
+
+// Check which tests are missing for the processes
+const missingTests = processes.filter(process => !tests.includes(process));
+
+if (missingTests.length > 0) {
+    console.error('The following processes are missing tests:');
+    missingTests.forEach(process => console.error(`- ${process}`));
+}
+
+// Check whether there are tests for non-existing processes
+const extraTests = tests.filter(test => !processes.includes(test));
+if (extraTests.length > 0) {
+    console.error('\nThe following tests exist without a corresponding process:');
+    extraTests.forEach(test => console.error(`- ${test}`));
+}
+
+// todo: add check that json5 files are valid
+
+if (missingTests.length === 0 && extraTests.length === 0) {
+    console.log('All processes have corresponding tests and vice versa.');
+    process.exit(0);
+}
+else {
+    process.exit(1);
+}

dev/package.json

@@ -23,8 +23,15 @@
     "http-server": "^14.1.1"
   },
   "scripts": {
-    "test": "openeo-processes-lint testConfig.json",
+    "check-tests": "node check-tests.js",
+    "has-tests": "node has-tests.js",
+    "lint": "openeo-processes-lint testConfig.json",
+    "test": "npm run has-tests && npm run check-tests && npm run lint",
     "generate": "concat-json-files \"../{*,proposals/*}.json\" -t \"processes.json\"",
     "start": "npm run generate && http-server -p 9876 -o docs.html -c-1"
+  },
+  "dependencies": {
+    "ajv": "^8.17.1",
+    "json5": "^2.2.3"
   }
 }

tests/README.md

@@ -0,0 +1,119 @@
+# Tests
+
+This folder contains test cases for the openEO processes.
+
+## Assumptions
+
+The test cases assume a couple of things as they are an abstraction and not bound to specific implementations:
+- The JSON Schema type `number` explicitly includes the values `+Infinity`, `-Infinity` and `NaN`.
+- The input and output values for no-data values are `null` by default unless otherwise specified by a runner.
+- Input that is not valid according to the process schema must be rejected upfront, without any process evaluation attempt. Consequently, cases with schema mismatches are not covered in these tests. For example, the absolute process only tests against the data types `number` and `null`. There are no tests for a boolean or string input.
+- Numerical data types such as uint8 don't matter, i.e. tests don't check for overflows etc. This suite can't provide such tests as the underlying data type is not known.
+- If not otherwise specified for numbers, a precision of 10 decimals is checked so return values should have at least 11 decimals.
+
+## Test Files
+
+To allow for more data types (e.g. infinity and nan for numbers), all the files are encoded in **JSON5** instead of JSON.
+
+The test files have the following schema: [schema/schema.json](./schema/schema.json)
+
+### No-data values
+
+No-data values have a special encoding in tests (see below).
+The encoding is replaced with `null` unless otherwise specified by the runners.
+
+```json5
+{
+  "type": "nodata"
+}
+```
+
+### Datetimes
+
+Datetimes as strings have a varying precision, especially regarding the milliseconds.
+Also, sometimes timezones are differently handled.
+
+Datetimes in return values should be encoded in compliance with RFC 3339 so that the results can be compared better:
+
+```json5
+{
+  "type": "datetime",
+  "value": "2020-01-01T00:00:00Z"
+}
+```
+
+### External references
+
+Arguments and return values can point to external files, e.g.
+
+```json5
+{
+  "$ref": "https://host.example/datacube.json"
+}
+```
+
+The test suite can currently only load JSON and JSON5 files.
+
+### Labeled arrays
+
+Labeled arrays can't be represented in JSON5 and will be provided as an object instead.
+
+```json5
+{
+  "type": "labeled-array",
+  "data": [
+    {
+      "key": "B01",
+      "value": 1.23
+    },
+    {
+      "key": "B02",
+      "value": 0.98
+    }
+    // ...
+  ]
+}
+```
+
+### Datacubes
+
+Datacubes can't be represented in JSON5 and will be provided as an object instead.
+Vector datacubes are currently not supported.
+
+```json5
+{
+  "type": "datacube",
+  "data": [
+    // multi-dimensional array
+    // can be set to `null` if the data values are irrelevant for the test.
+  ],
+  "nodata": [
+    NaN
+  ],
+  "order": ["bands", "t", "y", "x"],
+  "dimensions": {
+    // similar to the STAC datacube extension
+    // properties: type, axis (if type = spatial), values, and reference_system (optional)
+    "bands": {
+      "type": "bands",
+      "values": ["blue","green","red","nir"]
+    },
+    "t": {
+      "type": "temporal",
+      "values": ["2020-06-01T00:00:00Z","2020-06-03T00:00:00Z","2020-06-06T00:00:00Z"]
+    },
+    "y": {
+      "type": "spatial",
+      "axis": "y",
+      "values": [5757495.0,5757485.0,5757475.0,5757465.0],
+      "reference_system": "EPSG:25832"
+    },
+    "x": {
+      "type": "spatial",
+      "axis": "x",
+      "values": [404835.0,404845.0,404855.0,404865.0,404875.0],
+      "reference_system": "EPSG:25832"
+    }
+  }
+}
+```

tests/absolute.json5

@@ -0,0 +1,60 @@
+{
+  "id": "absolute",
+  "level": "L1",
+  "tests": [
+    {
+      "arguments": {
+        "x": 0
+      },
+      "returns": 0
+    },
+    {
+      "arguments": {
+        "x": 1
+      },
+      "returns": 1
+    },
+    {
+      "arguments": {
+        "x": -1
+      },
+      "returns": 1
+    },
+    {
+      "arguments": {
+        "x": 2.5
+      },
+      "returns": 2.5
+    },
+    {
+      "arguments": {
+        "x": -2.5
+      },
+      "returns": 2.5
+    },
+    {
+      "arguments": {
+        "x": NaN
+      },
+      "returns": NaN
+    },
+    {
+      "arguments": {
+        "x": Infinity
+      },
+      "returns": Infinity
+    },
+    {
+      "arguments": {
+        "x": -Infinity
+      },
+      "returns": Infinity
+    },
+    {
+      "arguments": {
+        "x": {"type": "nodata"}
+      },
+      "returns": {"type": "nodata"}
+    }
+  ]
+}