Add ability to include external files in current.sql (#195)

Author: benjie

README.md

@@ -724,6 +724,59 @@ by `graphile-migrate watch` is defined. By default this is in the
 `migrations/current.sql` file, but it might be `migrations/current/*.sql` if
 you're using folder mode.
 
+#### Including external files in the current migration
+
+You can include external files in your `current.sql` to better assist in source
+control. These includes are identified by paths within the `migrations/fixtures`
+folder.
+
+For example. Given the following directory structure:
+
+```
+/- migrate
+ - migrations
+   |
+   - current.sql
+   - fixtures
+     |
+     - functions
+       |
+       - myfunction.sql
+```
+
+and the contents of `myfunction.sql`:
+
+```sql
+create or replace function myfunction(a int, b int)
+returns int as $$
+  select a + b;
+$$ language sql stable;
+```
+
+When you make changes to `myfunction.sql`, include it in your current migration
+by adding `--!include functions/myfunction.sql` to your `current.sql` (or any
+`current/*.sql`). This statement doesn't need to be at the top of the file,
+wherever it is will be replaced by the content of
+`migrations/fixtures/functions/myfunction.sql` when the migration is committed.
+
+```sql
+--!include fixtures/functions/myfunction.sql
+drop policy if exists access_by_numbers on mytable;
+create policy access_by_numbers on mytable for update using (myfunction(4, 2) < 42);
+```
+
+and when the migration is committed or watched, the contents of `myfunction.sql`
+will be included in the result, such that the following SQL is executed:
+
+```sql
+create or replace function myfunction(a int, b int)
+returns int as $$
+  select a + b;
+$$ language sql stable;
+drop policy if exists access_by_numbers on mytable;
+create policy access_by_numbers on mytable for update using (myfunction(4, 2) < 42);
+```
+
 ### Committed migration(s)
 
 The files for migrations that you've committed with `graphile-migrate commit`

__tests__/__snapshots__/include.test.ts.snap

@@ -0,0 +1,9 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`compiles an included file, and won't get stuck in an infinite include loop 1`] = `
+"Circular include detected - '~/migrations/fixtures/foo.sql' is included again! Import statement: \`--!include foo.sql\`; trace:
+  ~/migrations/fixtures/foo.sql
+  ~/migrations/current.sql"
+`;
+
+exports[`disallows calling files outside of the migrations/fixtures folder 1`] = `"Forbidden: cannot include path '~/outsideFolder/foo.sql' because it's not inside '~/migrations/fixtures'"`;

__tests__/commit.test.ts

@@ -1,6 +1,6 @@
 import "./helpers"; // Has side-effects; must come first
 
-import { promises as fsp } from "fs";
+import * as fsp from "fs/promises";
 import mockFs from "mock-fs";
 
 import { commit } from "../src";

__tests__/compile.test.ts

@@ -1,7 +1,8 @@
 import "./helpers";
 
-import { compile } from "../src";
+import * as mockFs from "mock-fs";
 
+import { compile } from "../src";
 let old: string | undefined;
 beforeAll(() => {
   old = process.env.DATABASE_AUTHENTICATOR;
@@ -11,6 +12,10 @@ afterAll(() => {
   process.env.DATABASE_AUTHENTICATOR = old;
 });
 
+afterEach(() => {
+  mockFs.restore();
+});
+
 it("compiles SQL with settings", async () => {
   expect(
     await compile(

__tests__/include.test.ts

@@ -0,0 +1,145 @@
+import "./helpers";
+
+import mockFs from "mock-fs";
+
+import { compileIncludes } from "../src/migration";
+import { ParsedSettings, parseSettings } from "../src/settings";
+
+let old: string | undefined;
+let settings: ParsedSettings;
+beforeAll(async () => {
+  old = process.env.DATABASE_AUTHENTICATOR;
+  process.env.DATABASE_AUTHENTICATOR = "dbauth";
+  settings = await parseSettings({
+    connectionString: "postgres://dbowner:dbpassword@dbhost:1221/dbname",
+    placeholders: {
+      ":DATABASE_AUTHENTICATOR": "!ENV",
+    },
+    migrationsFolder: "migrations",
+  });
+});
+afterAll(() => {
+  process.env.DATABASE_AUTHENTICATOR = old;
+});
+
+afterEach(() => {
+  mockFs.restore();
+});
+
+/** Pretents that our compiled files are 'current.sql' */
+const FAKE_VISITED = new Set([`${process.cwd()}/migrations/current.sql`]);
+
+it("compiles an included file", async () => {
+  mockFs({
+    "migrations/fixtures/foo.sql": "select * from foo;",
+  });
+  expect(
+    await compileIncludes(
+      settings,
+      `\
+--!include foo.sql
+`,
+      FAKE_VISITED,
+    ),
+  ).toEqual(`\
+select * from foo;
+`);
+});
+
+it("compiles multiple included files", async () => {
+  mockFs({
+    "migrations/fixtures/dir1/foo.sql": "select * from foo;",
+    "migrations/fixtures/dir2/bar.sql": "select * from bar;",
+    "migrations/fixtures/dir3/baz.sql": "--!include dir4/qux.sql",
+    "migrations/fixtures/dir4/qux.sql": "select * from qux;",
+  });
+  expect(
+    await compileIncludes(
+      settings,
+      `\
+--!include dir1/foo.sql
+--!include dir2/bar.sql
+--!include dir3/baz.sql
+`,
+      FAKE_VISITED,
+    ),
+  ).toEqual(`\
+select * from foo;
+select * from bar;
+select * from qux;
+`);
+});
+
+it("compiles an included file, and won't get stuck in an infinite include loop", async () => {
+  mockFs({
+    "migrations/fixtures/foo.sql": "select * from foo;\n--!include foo.sql",
+  });
+  const promise = compileIncludes(
+    settings,
+    `\
+--!include foo.sql
+`,
+    FAKE_VISITED,
+  );
+  await expect(promise).rejects.toThrowError(/Circular include/);
+  const message = await promise.catch((e) => e.message);
+  expect(message.replaceAll(process.cwd(), "~")).toMatchSnapshot();
+});
+
+it("disallows calling files outside of the migrations/fixtures folder", async () => {
+  mockFs({
+    "migrations/fixtures/bar.sql": "",
+    "outsideFolder/foo.sql": "select * from foo;",
+  });
+
+  const promise = compileIncludes(
+    settings,
+    `\
+--!include ../../outsideFolder/foo.sql
+`,
+    FAKE_VISITED,
+  );
+  await expect(promise).rejects.toThrowError(/Forbidden: cannot include/);
+  const message = await promise.catch((e) => e.message);
+  expect(message.replaceAll(process.cwd(), "~")).toMatchSnapshot();
+});
+
+it("compiles an included file that contains escapable things", async () => {
+  mockFs({
+    "migrations/fixtures/foo.sql": `\
+begin;
+
+create or replace function current_user_id() returns uuid as $$
+  select nullif(current_setting('user.id', true)::text, '')::uuid;
+$$ language sql stable;
+
+comment on function current_user_id is E'The ID of the current user.';
+
+grant all on function current_user_id to :DATABASE_USER;
+
+commit;
+`,
+  });
+  expect(
+    await compileIncludes(
+      settings,
+      `\
+--!include foo.sql
+`,
+      FAKE_VISITED,
+    ),
+  ).toEqual(`\
+begin;
+
+create or replace function current_user_id() returns uuid as $$
+  select nullif(current_setting('user.id', true)::text, '')::uuid;
+$$ language sql stable;
+
+comment on function current_user_id is E'The ID of the current user.';
+
+grant all on function current_user_id to :DATABASE_USER;
+
+commit;
+
+`);
+});

__tests__/readCurrentMigration.test.ts

@@ -102,3 +102,14 @@ With multiple lines
   const content = await readCurrentMigration(parsedSettings, currentLocation);
   expect(content).toEqual(contentWithSplits);
 });
+
+it("reads from current.sql, and processes included files", async () => {
+  mockFs({
+    "migrations/current.sql": "--!include foo_current.sql",
+    "migrations/fixtures/foo_current.sql": "-- TEST from foo",
+  });
+
+  const currentLocation = await getCurrentMigrationLocation(parsedSettings);
+  const content = await readCurrentMigration(parsedSettings, currentLocation);
+  expect(content).toEqual("-- TEST from foo");
+});

__tests__/uncommit.test.ts

@@ -1,6 +1,6 @@
 import "./helpers"; // Has side-effects; must come first
 
-import { promises as fsp } from "fs";
+import * as fsp from "fs/promises";
 import mockFs from "mock-fs";
 
 import { commit, migrate, uncommit } from "../src";

__tests__/writeCurrentMigration.test.ts

@@ -1,6 +1,6 @@
 import "./helpers"; // Has side-effects; must come first
 
-import { promises as fsp } from "fs";
+import * as fsp from "fs/promises";
 import mockFs from "mock-fs";
 
 import {

package.json

@@ -42,12 +42,12 @@
   "dependencies": {
     "@graphile/logger": "^0.2.0",
     "@types/json5": "^2.2.0",
-    "@types/node": "^20.11.5",
-    "@types/pg": "^8.10.9",
+    "@types/node": "^18",
+    "@types/pg": ">=6 <9",
     "chalk": "^4",
     "chokidar": "^3.5.3",
     "json5": "^2.2.3",
-    "pg": "^8.11.3",
+    "pg": ">=6.5 <9",
     "pg-connection-string": "^2.6.2",
     "pg-minify": "^1.6.3",
     "tslib": "^2.6.2",

scripts/update-docs.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-const { promises: fsp } = require("fs");
+const fsp = require("fs/promises");
 const { spawnSync } = require("child_process");
 
 async function main() {