Fix validation of declaration files with filters (#1191)

Ndpnt · web-flow · commit 6816e1efa24b · 2025-09-16T17:31:10.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [patch]
+
+> Development of this release was supported by the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs.
+
+### Fixed
+
+- Fix validation of declaration files with filters
+
 ## 7.2.1 - 2025-09-11
 
 _Full changeset and discussions: [#1189](https://github.com/OpenTermsArchive/engine/pull/1189)._
diff --git a/scripts/declarations/validate/definitions.js b/scripts/declarations/validate/definitions.js
@@ -38,7 +38,7 @@ const definitions = {
         {
           type: 'object',
           description: 'Filter function with parameters. The key is the filter function name, the value is the parameters.',
-          additionalProperties: false,
+          additionalProperties: true,
           minProperties: 1,
           maxProperties: 1,
         },
diff --git a/scripts/declarations/validate/fixtures/invalid-history/invalid-date.json b/scripts/declarations/validate/fixtures/invalid-history/invalid-date.json
@@ -0,0 +1,9 @@
+{
+  "Privacy Policy": [
+    {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content",
+      "validUntil": "2023-12-31"
+    }
+  ]
+}
diff --git a/scripts/declarations/validate/fixtures/invalid-history/missing-valid-until.json b/scripts/declarations/validate/fixtures/invalid-history/missing-valid-until.json
@@ -0,0 +1,8 @@
+{
+  "Privacy Policy": [
+    {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  ]
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/additional-properties.json b/scripts/declarations/validate/fixtures/invalid/additional-properties.json
@@ -0,0 +1,10 @@
+{
+  "name": "Service with additional properties",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  },
+  "unexpectedProperty": "value"
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/invalid-filter-object.json b/scripts/declarations/validate/fixtures/invalid/invalid-filter-object.json
@@ -0,0 +1,17 @@
+{
+  "name": "Service with invalid filter object",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content",
+      "filter": [
+        {
+          "filterName": {
+            "param": "value"
+          },
+          "unexpectedAdditionalProperty": "value"
+        }
+      ]
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/invalid-terms-type.json b/scripts/declarations/validate/fixtures/invalid/invalid-terms-type.json
@@ -0,0 +1,9 @@
+{
+  "name": "Service with invalid terms type",
+  "terms": {
+    "Invalid Terms Type": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/invalid-url.json b/scripts/declarations/validate/fixtures/invalid/invalid-url.json
@@ -0,0 +1,9 @@
+{
+  "name": "Service with invalid URL",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "not-a-valid-url",
+      "select": "div.content"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/missing-fetch.json b/scripts/declarations/validate/fixtures/invalid/missing-fetch.json
@@ -0,0 +1,8 @@
+{
+  "name": "Service without fetch",
+  "terms": {
+    "Privacy Policy": {
+      "select": "div.content"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/missing-name.json b/scripts/declarations/validate/fixtures/invalid/missing-name.json
@@ -0,0 +1,8 @@
+{
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/missing-select.json b/scripts/declarations/validate/fixtures/invalid/missing-select.json
@@ -0,0 +1,8 @@
+{
+  "name": "Service without select",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/missing-terms.json b/scripts/declarations/validate/fixtures/invalid/missing-terms.json
@@ -0,0 +1,3 @@
+{
+  "name": "Service without terms"
+}
diff --git a/scripts/declarations/validate/fixtures/invalid/non-pdf-url-for-pdf.json b/scripts/declarations/validate/fixtures/invalid/non-pdf-url-for-pdf.json
@@ -0,0 +1,8 @@
+{
+  "name": "Service with non-PDF URL for PDF document",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy.html"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/valid-history/pdf-history.json b/scripts/declarations/validate/fixtures/valid-history/pdf-history.json
@@ -0,0 +1,8 @@
+{
+  "Privacy Policy": [
+    {
+      "fetch": "https://example.com/privacy.pdf",
+      "validUntil": "2023-12-31T23:59:59Z"
+    }
+  ]
+}
diff --git a/scripts/declarations/validate/fixtures/valid-history/simple-history.json b/scripts/declarations/validate/fixtures/valid-history/simple-history.json
@@ -0,0 +1,9 @@
+{
+  "Privacy Policy": [
+    {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content",
+      "validUntil": "2023-12-31T23:59:59Z"
+    }
+  ]
+}
diff --git a/scripts/declarations/validate/fixtures/valid/complex-service.json b/scripts/declarations/validate/fixtures/valid/complex-service.json
@@ -0,0 +1,30 @@
+{
+  "name": "Complex service",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": [
+        "div.content",
+        ".privacy-section"
+      ],
+      "filter": [
+        "removeLinks",
+        {
+          "customTransform": {
+            "options": {
+              "cleanup": true
+            }
+          }
+        }
+      ],
+      "remove": [
+        ".ads",
+        ".tracking"
+      ],
+      "executeClientScripts": true
+    },
+    "Terms of Service": {
+      "fetch": "https://example.com/terms.pdf"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/valid/minimal-service.json b/scripts/declarations/validate/fixtures/valid/minimal-service.json
@@ -0,0 +1,9 @@
+{
+  "name": "Minimal service",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/valid/multiple-source-documents.json b/scripts/declarations/validate/fixtures/valid/multiple-source-documents.json
@@ -0,0 +1,24 @@
+{
+  "name": "Multiple source documents service",
+  "terms": {
+    "Privacy Policy": {
+      "combine": [
+        {
+          "fetch": "https://example.com/privacy1",
+          "select": "div.content",
+          "filter": [
+            "removeLinks"
+          ]
+        },
+        {
+          "fetch": "https://example.com/privacy2",
+          "select": ".privacy-section"
+        }
+      ],
+      "select": "body",
+      "filter": [
+        "trimWhitespace"
+      ]
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/valid/pdf-document.json b/scripts/declarations/validate/fixtures/valid/pdf-document.json
@@ -0,0 +1,8 @@
+{
+  "name": "Service with PDF terms",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy.pdf"
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/valid/range-selectors.json b/scripts/declarations/validate/fixtures/valid/range-selectors.json
@@ -0,0 +1,19 @@
+{
+  "name": "Service with range selectors",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": [
+        "div.content",
+        {
+          "startBefore": "h1",
+          "endAfter": "footer"
+        },
+        {
+          "startAfter": "h2",
+          "endBefore": ".end-marker"
+        }
+      ]
+    }
+  }
+}
diff --git a/scripts/declarations/validate/fixtures/valid/service-with-filters.json b/scripts/declarations/validate/fixtures/valid/service-with-filters.json
@@ -0,0 +1,22 @@
+{
+  "name": "Service with filters",
+  "terms": {
+    "Privacy Policy": {
+      "fetch": "https://example.com/privacy",
+      "select": "div.content",
+      "filter": [
+        "removeLinks",
+        {
+          "customFilter": {
+            "param": "value"
+          }
+        },
+        {
+          "anotherFilter": {
+            "setting": true
+          }
+        }
+      ]
+    }
+  }
+}
diff --git a/scripts/declarations/validate/schemas.test.js b/scripts/declarations/validate/schemas.test.js
@@ -0,0 +1,84 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import Ajv from 'ajv';
+import addFormats from 'ajv-formats';
+import { expect } from 'chai';
+
+import serviceHistorySchema from './service.history.schema.js';
+import serviceSchema from './service.schema.js';
+
+describe('Schema validation', () => {
+  let validator;
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+  const fixturesDir = path.join(__dirname, 'fixtures');
+
+  before(() => {
+    validator = new Ajv({ allErrors: true });
+    addFormats(validator);
+  });
+
+  const loadFixture = filePath => {
+    const content = fs.readFileSync(filePath, 'utf8');
+
+    return JSON.parse(content);
+  };
+
+  const getJsonFiles = dir => {
+    const files = fs.readdirSync(dir);
+
+    return files.filter(file => file.endsWith('.json'));
+  };
+
+  const validateDeclaration = (schema, declaration, file, shouldBeValid) => {
+    const valid = validator.validate(schema, declaration);
+
+    if (!valid && shouldBeValid) {
+      console.error(`Validation errors for ${file}:`, validator.errors);
+    }
+
+    return valid;
+  };
+
+  const createValidationTests = (schema, directory, shouldBeValid) => {
+    const files = getJsonFiles(directory);
+    const action = shouldBeValid ? 'validates' : 'rejects';
+
+    files.forEach(file => {
+      it(`${action} ${file}`, () => {
+        const filePath = path.join(directory, file);
+        const declaration = loadFixture(filePath);
+        const valid = validateDeclaration(schema, declaration, file, shouldBeValid);
+
+        expect(valid, `${file} should be ${shouldBeValid ? 'valid' : 'invalid'}`).to.equal(shouldBeValid);
+      });
+    });
+  };
+
+  describe('Service schema', () => {
+    const validDir = path.join(fixturesDir, 'valid');
+    const invalidDir = path.join(fixturesDir, 'invalid');
+
+    describe('with valid declarations', () => {
+      createValidationTests(serviceSchema, validDir, true);
+    });
+
+    describe('with invalid declarations', () => {
+      createValidationTests(serviceSchema, invalidDir, false);
+    });
+  });
+
+  describe('Service history schema', () => {
+    const validHistoryDir = path.join(fixturesDir, 'valid-history');
+    const invalidHistoryDir = path.join(fixturesDir, 'invalid-history');
+
+    describe('with valid history declarations', () => {
+      createValidationTests(serviceHistorySchema, validHistoryDir, true);
+    });
+
+    describe('with invalid history declarations', () => {
+      createValidationTests(serviceHistorySchema, invalidHistoryDir, false);
+    });
+  });
+});

Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,7 @@ const definitions = {`
`38`	`38`	`{`
`39`	`39`	`type: 'object',`
`40`	`40`	`description: 'Filter function with parameters. The key is the filter function name, the value is the parameters.',`
`41`		`- additionalProperties: false,`
	`41`	`+ additionalProperties: true,`
`42`	`42`	`minProperties: 1,`
`43`	`43`	`maxProperties: 1,`
`44`	`44`	`},`