Add support for parameterized filters in service declarations and add removeQueryParams built-in filter (#1178)

Author: Ndpnt

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [minor]
+
+> Development of this release was supported by the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs.
+
+### Added
+
+- Add support for parameters in filters; see more in the [filters documentation](https://docs.opentermsarchive.org/terms/how-to/apply-filters/)
+- Add `removeQueryParams` built-in filter to remove query parameters from links and images; see more in the [built-in filters documentation](https://docs.opentermsarchive.org/terms/reference/built-in-filters/)
+
 ## 7.1.0 - 2025-09-10
 
 _Full changeset and discussions: [#1188](https://github.com/OpenTermsArchive/engine/pull/1188)._

README.md

@@ -4,6 +4,12 @@ This codebase is a Node.js module enabling downloading, archiving and publishing
 
 For documentation, visit [docs.opentermsarchive.org](https://docs.opentermsarchive.org/)
 
+## Testing
+
+Use `npm test` to run all tests.
+
+Use `npm run test:only <file.test.js>... [--watch]` to run specific test files. The `--watch` option enables running those tests each time a file changes.
+
 - - -
 
 ## Contribute

package-lock.json

@@ -45,6 +45,7 @@
     "start:api": "node bin/ota.js serve",
     "start:scheduler": "npm start -- --schedule",
     "test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" \"./scripts/**/*.test.js\" --exit",
+    "test:only": "cross-env NODE_ENV=test mocha --recursive",
     "posttest": "npm run lint",
     "test:debug": "npm run test -- --inspect-brk --exit"
   },
@@ -122,6 +123,6 @@
     "@opentermsarchive/terms-types": "^2.0.0"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=16.0.0 < 23.0.0"
   }
 }

package.json

@@ -29,9 +29,20 @@ const definitions = {
   filters: {
     type: 'array',
     items: {
-      type: 'string',
-      pattern: '^.+$',
-      description: 'Filter function name',
+      oneOf: [
+        {
+          type: 'string',
+          pattern: '^.+$',
+          description: 'Filter function name',
+        },
+        {
+          type: 'object',
+          description: 'Filter function with parameters. The key is the filter function name, the value is the parameters.',
+          additionalProperties: false,
+          minProperties: 1,
+          maxProperties: 1,
+        },
+      ],
     },
   },
   validUntil: {

scripts/declarations/validate/definitions.js

@@ -7,6 +7,7 @@ import { expect } from 'chai';
 import config from 'config';
 import jsonSourceMap from 'json-source-map';
 
+import * as exposedFilters from '../../../src/archivist/extract/exposedFilters.js';
 import extract from '../../../src/archivist/extract/index.js';
 import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from '../../../src/archivist/fetcher/index.js';
 import * as services from '../../../src/archivist/services/index.js';
@@ -76,6 +77,24 @@ export default async options => {
           });
         }
 
+        it('filters do not use reserved names', async () => {
+          const filtersFilePath = path.join(declarationsPath, `${serviceId}.filters.js`);
+
+          if (!fsApi.existsSync(filtersFilePath)) {
+            return; // Skip if no filters file exists
+          }
+
+          const serviceFilters = await services.loadServiceFilters(serviceId);
+          const reservedFilterNames = Object.keys(exposedFilters);
+          const serviceFilterNames = Object.keys(serviceFilters);
+
+          const conflictingNames = serviceFilterNames.filter(name => reservedFilterNames.includes(name));
+
+          if (conflictingNames.length) {
+            throw new Error(`Service filter file "${serviceId}.filters.js" declares filters with names used by built-in filters: "${conflictingNames.join('", "')}". Rename these filters to avoid a collision.`);
+          }
+        });
+
         if (!schemaOnly && service) {
           service.getTermsTypes()
             .filter(termsType => {

scripts/declarations/validate/index.mocha.js

@@ -0,0 +1,75 @@
+import jsdom from 'jsdom';
+
+export default function createWebPageDOM(content, location) {
+  const { document } = new jsdom.JSDOM(content, {
+    url: location,
+    virtualConsole: new jsdom.VirtualConsole(),
+  }).window;
+
+  return Object.assign(document, {
+    select(contentSelectors) {
+      const result = document.createDocumentFragment();
+      let hasContent = false;
+
+      [].concat(contentSelectors).forEach(selector => {
+        if (typeof selector === 'object') {
+          const rangeSelection = this.selectRange(selector);
+          const clonedContent = rangeSelection.cloneContents();
+
+          if (clonedContent.hasChildNodes()) {
+            result.appendChild(clonedContent);
+            hasContent = true;
+          }
+        } else {
+          const elements = document.querySelectorAll(selector);
+
+          if (elements.length > 0) {
+            elements.forEach(element => result.appendChild(element.cloneNode(true)));
+            hasContent = true;
+          }
+        }
+      });
+
+      return hasContent ? result : null;
+    },
+
+    remove(insignificantContentSelectors) {
+      const rangeSelections = [];
+      const nodes = [];
+
+      [].concat(insignificantContentSelectors).forEach(selector => {
+        if (typeof selector === 'object') {
+          rangeSelections.push(this.selectRange(selector));
+        } else {
+          nodes.push(...document.querySelectorAll(selector));
+        }
+      });
+
+      nodes.forEach(node => node.remove());
+      rangeSelections.forEach(rangeSelection => rangeSelection.deleteContents());
+
+      return this;
+    },
+
+    selectRange(rangeSelector) {
+      const { startBefore, startAfter, endBefore, endAfter } = rangeSelector;
+
+      const selection = document.createRange();
+      const startNode = document.querySelector(startBefore || startAfter);
+      const endNode = document.querySelector(endBefore || endAfter);
+
+      if (!startNode) {
+        throw new Error(`The "start" selector has no match in document in: ${JSON.stringify(rangeSelector)}`);
+      }
+
+      if (!endNode) {
+        throw new Error(`The "end" selector has no match in document in: ${JSON.stringify(rangeSelector)}`);
+      }
+
+      selection[startBefore ? 'setStartBefore' : 'setStartAfter'](startNode);
+      selection[endBefore ? 'setEndBefore' : 'setEndAfter'](endNode);
+
+      return selection;
+    },
+  });
+}

src/archivist/extract/dom.js

@@ -0,0 +1,207 @@
+import { expect } from 'chai';
+
+import createWebPageDOM from './dom.js';
+
+describe('createWebPageDOM', () => {
+  const sampleHTML = `
+    <!DOCTYPE html>
+    <html>
+      <head>
+        <title>Test Document</title>
+      </head>
+      <body>
+        <header id="header">
+          <h1>Main Title</h1>
+          <nav class="navigation">
+            <a href="/home">Home</a>
+            <a href="/about">About</a>
+          </nav>
+        </header>
+        <main>
+          <article id="content">
+            <p class="introduction">Introduction paragraph</p>
+            <p class="central">Central paragraph</p>
+            <p class="conclusion">Conclusion paragraph</p>
+          </article>
+          <aside class="sidebar">
+            <div class="widget">Widget content</div>
+          </aside>
+        </main>
+        <footer id="footer">
+          <p>Footer content</p>
+        </footer>
+      </body>
+    </html>
+  `;
+  const location = 'https://example.com/test';
+  let document;
+
+  before(() => {
+    document = createWebPageDOM(sampleHTML, location);
+  });
+
+  it('creates a DOM document from HTML content', () => {
+    expect(document.documentElement.tagName).to.equal('HTML');
+  });
+
+  it('sets the document location', () => {
+    expect(document.location.href).to.equal(location);
+  });
+
+  it('provides access to the DOM API', () => {
+    const title = document.querySelector('title');
+
+    expect(title.textContent).to.equal('Test Document');
+  });
+
+  describe('#select', () => {
+    it('returns elements using CSS selectors', () => {
+      const fragment = document.select('p.introduction');
+      const paragraph = fragment.querySelector('p');
+
+      expect(paragraph.textContent).to.equal('Introduction paragraph');
+    });
+
+    it('returns multiple elements using CSS selectors', () => {
+      const fragment = document.select('p');
+      const paragraphs = fragment.querySelectorAll('p');
+
+      expect(paragraphs.length).to.equal(4);
+    });
+
+    it('returns elements using an array of CSS selectors', () => {
+      const fragment = document.select([ 'h1', '.introduction' ]);
+      const heading = fragment.querySelector('h1');
+      const paragraph = fragment.querySelector('p');
+
+      expect(heading.textContent).to.equal('Main Title');
+      expect(paragraph.textContent).to.equal('Introduction paragraph');
+    });
+
+    it('returns content using a range selector object', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endBefore: '.conclusion',
+      };
+      const fragment = document.select(rangeSelector);
+      const paragraph = fragment.querySelector('p');
+
+      expect(paragraph.textContent).to.equal('Central paragraph');
+    });
+
+    it('returns null when the selector matches no element', () => {
+      const result = document.select('.nonexistent');
+
+      expect(result).to.be.null;
+    });
+  });
+
+  describe('#remove', () => {
+    let testDocument;
+
+    beforeEach(() => {
+      testDocument = createWebPageDOM(sampleHTML, location);
+    });
+
+    it('removes elements using CSS selectors', () => {
+      testDocument.remove('.sidebar');
+      const sidebar = testDocument.querySelector('.sidebar');
+
+      expect(sidebar).to.be.null;
+    });
+
+    it('removes multiple elements using CSS selectors', () => {
+      testDocument.remove('p');
+      const paragraphs = testDocument.querySelectorAll('p');
+
+      expect(paragraphs.length).to.equal(0);
+    });
+
+    it('removes elements using an array of CSS selectors', () => {
+      testDocument.remove([ 'nav', '.widget' ]);
+      const nav = testDocument.querySelector('nav');
+      const widget = testDocument.querySelector('.widget');
+
+      expect(nav).to.be.null;
+      expect(widget).to.be.null;
+    });
+
+    it('removes content using a range selector object', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endBefore: '.conclusion',
+      };
+
+      testDocument.remove(rangeSelector);
+      const bodyParagraph = testDocument.querySelector('.central');
+
+      expect(bodyParagraph).to.be.null;
+    });
+  });
+
+  describe('#selectRange', () => {
+    it('creates a range using startAfter and endBefore', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endBefore: '.conclusion',
+      };
+      const range = document.selectRange(rangeSelector);
+      const fragment = range.cloneContents();
+      const paragraph = fragment.querySelector('p');
+
+      expect(paragraph.textContent).to.equal('Central paragraph');
+    });
+
+    it('creates a range using startBefore and endAfter', () => {
+      const rangeSelector = {
+        startBefore: '.central',
+        endAfter: '.central',
+      };
+      const range = document.selectRange(rangeSelector);
+      const fragment = range.cloneContents();
+      const paragraph = fragment.querySelector('p');
+
+      expect(paragraph.textContent).to.equal('Central paragraph');
+    });
+
+    it('throws a clear error when the startBefore selector has no match', () => {
+      const rangeSelector = {
+        startBefore: '.nonexistent',
+        endBefore: '.conclusion',
+      };
+
+      expect(() => document.selectRange(rangeSelector)).to.throw('"start" selector has no match');
+      expect(() => document.selectRange(rangeSelector)).to.throw(JSON.stringify(rangeSelector));
+    });
+
+    it('throws a clear error when the startAfter selector has no match', () => {
+      const rangeSelector = {
+        startAfter: '.nonexistent',
+        endBefore: '.conclusion',
+      };
+
+      expect(() => document.selectRange(rangeSelector)).to.throw('"start" selector has no match');
+      expect(() => document.selectRange(rangeSelector)).to.throw(JSON.stringify(rangeSelector));
+    });
+
+    it('throws a clear error when the endBefore selector has no match', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endBefore: '.nonexistent',
+      };
+
+      expect(() => document.selectRange(rangeSelector)).to.throw('"end" selector has no match');
+      expect(() => document.selectRange(rangeSelector)).to.throw(JSON.stringify(rangeSelector));
+    });
+
+    it('throws a clear error when the endAfter selector has no match', () => {
+      const rangeSelector = {
+        startAfter: '.introduction',
+        endAfter: '.nonexistent',
+      };
+
+      expect(() => document.selectRange(rangeSelector)).to.throw('"end" selector has no match');
+      expect(() => document.selectRange(rangeSelector)).to.throw(JSON.stringify(rangeSelector));
+    });
+  });
+});