feat: Implement matrix testing for Dataform versions and update CI configuration

Author: max-ostapenko

.github/workflows/ci.yml

@@ -8,6 +8,9 @@ on:
 jobs:
   lint-and-test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        dataform-version: ['2.4.2', '3.0.42']
 
     steps:
       - name: Checkout code
@@ -20,13 +23,13 @@ jobs:
           cache: 'npm'
 
       - name: Install dependencies
-        run: npm ci && cd test-project && npm ci
+        run: npm ci
 
       - name: Run linter
         run: npm run lint
 
-      - name: Run tests
-        run: npm test
+      - name: Run tests (Dataform ${{ matrix.dataform-version }})
+        run: npm test -- ${{ matrix.dataform-version }}
 
 
   dependabot:

README.md

@@ -13,6 +13,32 @@ This package is designed to optimize BigQuery resource usage by automatically as
 * **Automated re-assignement**: Once configured, reservations are applied automatically based on action categorization
 * **Flexible configuration**: Easy adjustment of reservation policies through configuration updates
 
+## Compatibility
+
+This package is tested and compatible with:
+
+* **Dataform v2.4.2** (v2.x series)
+* **Dataform v3.0.42** (v3.x series)
+
+### Testing Across Versions
+
+To verify compatibility locally across all supported Dataform versions:
+
+```bash
+npm test
+```
+
+This command runs the matrix test suite which automatically:
+1. Iterates through all supported Dataform versions (v2 and v3).
+2. Manages configuration file conflicts (e.g., hiding `dataform.json` for v3).
+3. Executes unit tests and integration tests.
+
+For faster iteration on the currently installed version in `test-project`, you can run:
+
+```bash
+npm run test:single
+```
+
 ## Getting Started
 
 ### Initial Setup
@@ -29,28 +55,56 @@ Add the dependency to your `package.json`:
 
 and click **Install Packages** in Dataform UI.
 
-Then, import the package and create a setter function in your global scope under `/includes` directory:
+### Recommended: Automatic Application
+
+The easiest way to integrate this package is to use automatic reservation application. Create a configuration file (e.g., `definitions/_reservations.js`) that will automatically apply reservations to all matching actions:
+
+```javascript
+const { applyAutomaticReservations } = require("@masthead-data/dataform-package");
+
+const RESERVATION_CONFIG = [
+  {
+    tag: 'production',
+    reservation: 'projects/{project}/locations/{location}/reservations/{name}',
+    actions: [
+      'project.dataset.important_table',
+      'project.dataset.critical_view'
+    ]
+  },
+  {
+    tag: 'default',
+    reservation: null,
+    actions: []
+  }
+];
+
+applyAutomaticReservations(RESERVATION_CONFIG);
+```
+
+**Note:** If you have many files in the project we recommend to start the filename with an underscore (e.g., `_reservations.js`) to ensure it runs first in the Dataform queue.
+
+With automatic application, you don't need to add any reservation code to your individual action files — the package handles everything globally.
+
+### Alternative: Manual Application
+
+For more granular control, you can manually apply reservations per file. Create a setter function in your global scope under `/includes` directory:
 
 ```javascript
-const reservations = require("@masthead-data/dataform-package");
+const { createReservationSetter } = require("@masthead-data/dataform-package");
 
 const RESERVATION_CONFIG = [
   ...
 ];
 
-// Option 1: Manual Application (per file)
-const reservation_setter = reservations.createReservationSetter(RESERVATION_CONFIG);
+const reservation_setter = createReservationSetter(RESERVATION_CONFIG);
 
 module.exports = {
-  ...
   reservation_setter
 }
-
-// Option 2: Automatic Application (Global)
-// Place this in a new file, e.g., `definitions/reservations.js`
-reservations.applyAutomaticReservations(RESERVATION_CONFIG);
 ```
 
+Then use `${reservation_setter(ctx)}` in each action file where you want to apply reservations (see usage examples below).
+
 ### Configuration Structure
 
 Configuration object defining reservation policies:
@@ -88,7 +142,9 @@ Configuration arguments:
   * `null`: Use a default reservation
 * **actions**: Array of Dataform action names that are assigned to the reservation
 
-### Usage examples
+### Usage Examples (Manual Application)
+
+**Note:** These examples are only needed if you're using the manual application approach. With automatic application via `applyAutomaticReservations()`, reservations are applied automatically and you don't need to add these calls to your action files.
 
 #### `publish` actions
 
@@ -154,7 +210,36 @@ WHEN NOT MATCHED THEN INSERT (id, value) VALUES (S.id, S.value);
 `);
 ```
 
-Example implementation can be found in [https://github.com/HTTPArchive/dataform](https://github.com/HTTPArchive/dataform).
+## API Reference
+
+### `applyAutomaticReservations(config)`
+
+**Primary Method** - Automatically applies reservation configurations to all actions in your Dataform project.
+
+* **Parameters:**
+  * `config` (Array): Array of reservation configuration objects
+* **Returns:** `void`
+* **Usage:** Call once in a definitions file (e.g., `definitions/_reservations.js`)
+* **Behavior:** Automatically intercepts all `publish()`, `operate()`, and `assert()` calls and applies appropriate reservations based on action names
+
+### `createReservationSetter(config)`
+
+**Secondary Method** - Creates a reservation setter function for manual application per action.
+
+* **Parameters:**
+  * `config` (Array): Array of reservation configuration objects
+* **Returns:** `Function` - A setter function that accepts a Dataform context and returns the appropriate `SET @@reservation` SQL statement
+* **Usage:** Create in an includes file, then call in individual action files using `${reservation_setter(ctx)}`
+* **Use Case:** When you need fine-grained control over which actions get reservations
+
+### `getActionName(ctx)`
+
+**Utility Method** - Extracts the action name from a Dataform context object.
+
+* **Parameters:**
+  * `ctx` (Object): Dataform context object
+* **Returns:** `string|null` - The action name in format `database.schema.name`, or `null` if not found
+* **Usage:** Advanced use cases where you need to programmatically determine action names
 
 ## Under the Hood
 

package.json

@@ -4,7 +4,8 @@
   "description": "Masthead Data package for Dataform to optimize BigQuery resource usage",
   "main": "index.js",
   "scripts": {
-    "test": "jest && cd test-project && npx @dataform/cli compile --json > compiled.json && node ../scripts/verify_compilation.js",
+    "test": "./scripts/test-matrix.sh",
+    "test:single": "jest && cd test-project && npx @dataform/cli compile --json > compiled.json && node ../scripts/verify_compilation.js",
     "test:watch": "jest --watch",
     "lint": "eslint -c .github/linters/eslint.config.mjs .",
     "lint:fix": "eslint -c .github/linters/eslint.config.mjs --fix .",

scripts/test-matrix.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -e
+
+# Use provided arguments as versions, or default to the full matrix
+if [ $# -gt 0 ]; then
+  VERSIONS=("$@")
+else
+  VERSIONS=("2.4.2" "3.0.42")
+fi
+
+# Cleanup function to restore configuration files
+cleanup() {
+  if [ -f "test-project/dataform.json.bak" ]; then
+    echo "Restoring dataform.json"
+    mv "test-project/dataform.json.bak" "test-project/dataform.json"
+  fi
+  if [ -f "test-project/workflow_settings.yaml.bak" ]; then
+    echo "Restoring workflow_settings.yaml"
+    mv "test-project/workflow_settings.yaml.bak" "test-project/workflow_settings.yaml"
+  fi
+}
+
+# Ensure cleanup runs on exit (including failures)
+trap cleanup EXIT INT TERM
+
+echo "Running matrix tests across Dataform versions..."
+
+for VERSION in "${VERSIONS[@]}"; do
+  echo ""
+  echo "========================================="
+  echo "Testing with Dataform v$VERSION"
+  echo "========================================="
+
+  # Configuration management based on version
+  if [[ $VERSION == 3* ]]; then
+    if [ -f "test-project/dataform.json" ]; then
+      echo "Hiding dataform.json for v3 compatibility"
+      mv test-project/dataform.json test-project/dataform.json.bak
+    fi
+  elif [[ $VERSION == 2* ]]; then
+    if [ -f "test-project/workflow_settings.yaml" ]; then
+      echo "Hiding workflow_settings.yaml for v2 compatibility"
+      mv test-project/workflow_settings.yaml test-project/workflow_settings.yaml.bak
+    fi
+  fi
+
+  # Install specific version
+  echo "Installing Dataform @$VERSION..."
+  cd test-project
+  # Use --no-save to avoid cluttering package.json/package-lock.json during matrix tests
+  npm install @dataform/cli@$VERSION @dataform/core@$VERSION --no-save
+  cd ..
+
+  # Run tests using the single version command
+  npm run test:single
+
+  # Restore files after the run so the next version has a clean state
+  cleanup
+
+  echo "✓ Dataform v$VERSION tests passed"
+done
+
+echo ""
+echo "========================================="
+echo "All matrix tests passed!"
+echo "========================================="
+

scripts/verify_compilation.js

@@ -10,7 +10,15 @@ function verify() {
     process.exit(1)
   }
 
-  const compiled = JSON.parse(fs.readFileSync(COMPILED_JSON_PATH, 'utf8'))
+  let fileContent = fs.readFileSync(COMPILED_JSON_PATH, 'utf8')
+
+  // Dataform v2.x outputs a log line before the JSON, skip it
+  const lines = fileContent.split('\n')
+  if (lines[0].startsWith('{\"level\":')) {
+    fileContent = lines.slice(1).join('\n')
+  }
+
+  const compiled = JSON.parse(fileContent)
   let errors = []
 
   const checkTable = (name, expectedPreOps) => {

test-project/.gitignore

@@ -1,2 +1,3 @@
 node_modules/
 compiled.json
+snowflake.log

test-project/dataform.json

@@ -0,0 +1,7 @@
+{
+    "warehouse": "bigquery",
+    "defaultDatabase": "masthead-data",
+    "defaultLocation": "US",
+    "defaultSchema": "test",
+    "assertionSchema": "dataform_assertions"
+}

test-project/definitions/reservations.js test-project/definitions/_reservations.jstest-project/definitions/reservations.js renamed to test-project/definitions/_reservations.js

@@ -1,2 +1 @@
-// Initialize automatic reservations
 reservations.applyAutomaticReservations(reservations.RESERVATION_CONFIG)

test-project/package-lock.json

@@ -1,5 +1,6 @@
 const {
   createReservationSetter,
+
   getActionName
 } = require('../index')