v2.15.1: Polish Aurora support; add --skip-config option

Author: dimikot

.npmignore

@@ -1,3 +1,5 @@
+pg-id-consts.sql
+
 dist/__tests__
 dist/**/__tests__
 dist/*.tsbuildinfo

README.md

@@ -73,6 +73,8 @@ Custom variables of the tool itself:
 * `MIGRATE_CMD`: default value for `--migrate-cmd` option
 * `WEIGHT_SQL`: default value for `--weight-sql` option
 * `DEACTIVATE_SQL`: default value for `--deactivate-sql` option
+* `PARALLELISM`: default value for `--parallelism` option
+* `MAX_REPLICATION_LAG_SEC`: default value for `--max-replication-lag-sec` option
 
 ## Configuration File: pg-microsharding.config.ts
 
@@ -98,7 +100,7 @@ export default async function(action: "apply" | "undo" | string) {
 }
 ```
 
-The file `pg-microsharding.config.ts` is searched in all parent folders starting from the current working directory when `pg-microsharding` is run (typically you want to have it in the root of your project, near the other configuration files).
+Unless `--skip-config` flag is passed, the file `pg-microsharding.config.ts` is searched in all parent folders starting from the current working directory when `pg-microsharding` is run (typically you want to have it in the root of your project, near the other configuration files).
 
 You can export-default a regular function, an async function, or even a plain constant object.
 
@@ -153,7 +155,8 @@ The tool runs `--migrate-cmd` command right after creating the inactive microsha
 ```bash
 pg-microsharding move \
   --shard=42 --from=host1 --to=host2 \
-  --activate-on-destination=yes
+  --activate-on-destination=yes \
+  --max-replication-lag-sec=20
 ```
 
 Microshards can be moved from one PostgreSQL node to another. There is no need to stop writes while moving microshards: the tool uses PostgreSQL logical replication to stream each microshard table's data, and in the very end, acquires a quick write lock to finalize the move.
@@ -231,6 +234,12 @@ When you run `pg-microsharding factor --factor="*1.2"`, the tool artificially in
 
 The "weight increase factor" is technically stored as a SQL comment on the microshard schema, and it travels along with the microshard when you move it.
 
+### Replication Lag Prevention
+
+The tool tries hard to not affect the replication lag of the destination nodes when moving or rebalancing microshards. It waits until the lag drops below `--max-replication-lag-sec` seconds before running heavy operations (or until the user presses Shift+S to force-continue).
+
+Also, if you want the tool to pause explicitly and wait until the user presses Shift+S before activating the shard on the destination node, you can use the `--wait` option.
+
 ## PostgreSQL Stored Functions API
 
 This is the second part of pg-microsharding tool: a set of stored functions you add to your database.

docs/README.md

@@ -77,6 +77,8 @@ Custom variables of the tool itself:
 * `MIGRATE_CMD`: default value for `--migrate-cmd` option
 * `WEIGHT_SQL`: default value for `--weight-sql` option
 * `DEACTIVATE_SQL`: default value for `--deactivate-sql` option
+* `PARALLELISM`: default value for `--parallelism` option
+* `MAX_REPLICATION_LAG_SEC`: default value for `--max-replication-lag-sec` option
 
 ## Configuration File: pg-microsharding.config.ts
 
@@ -102,7 +104,7 @@ export default async function(action: "apply" | "undo" | string) {
 }
 ```
 
-The file `pg-microsharding.config.ts` is searched in all parent folders starting from the current working directory when `pg-microsharding` is run (typically you want to have it in the root of your project, near the other configuration files).
+Unless `--skip-config` flag is passed, the file `pg-microsharding.config.ts` is searched in all parent folders starting from the current working directory when `pg-microsharding` is run (typically you want to have it in the root of your project, near the other configuration files).
 
 You can export-default a regular function, an async function, or even a plain constant object.
 
@@ -157,7 +159,8 @@ The tool runs `--migrate-cmd` command right after creating the inactive microsha
 ```bash
 pg-microsharding move \
   --shard=42 --from=host1 --to=host2 \
-  --activate-on-destination=yes
+  --activate-on-destination=yes \
+  --max-replication-lag-sec=20
 ```
 
 Microshards can be moved from one PostgreSQL node to another. There is no need to stop writes while moving microshards: the tool uses PostgreSQL logical replication to stream each microshard table's data, and in the very end, acquires a quick write lock to finalize the move.
@@ -235,6 +238,12 @@ When you run `pg-microsharding factor --factor="*1.2"`, the tool artificially in
 
 The "weight increase factor" is technically stored as a SQL comment on the microshard schema, and it travels along with the microshard when you move it.
 
+### Replication Lag Prevention
+
+The tool tries hard to not affect the replication lag of the destination nodes when moving or rebalancing microshards. It waits until the lag drops below `--max-replication-lag-sec` seconds before running heavy operations (or until the user presses Shift+S to force-continue).
+
+Also, if you want the tool to pause explicitly and wait until the user presses Shift+S before activating the shard on the destination node, you can use the `--wait` option.
+
 ## PostgreSQL Stored Functions API
 
 This is the second part of pg-microsharding tool: a set of stored functions you add to your database.

docs/functions/allocate.md

@@ -8,7 +8,7 @@
 
 > **allocate**(`__namedParameters`): `Promise`\<`void`\>
 
-Defined in: [src/api/allocate.ts:11](https://github.com/clickup/pg-microsharding/blob/master/src/api/allocate.ts#L11)
+Defined in: [src/api/allocate.ts:14](https://github.com/clickup/pg-microsharding/blob/master/src/api/allocate.ts#L14)
 
 Ensures that all shards in the range exist on the DSN, then runs a shell
 script (presumably DB migration), and then optionally activates the shards.
@@ -17,8 +17,8 @@ script (presumably DB migration), and then optionally activates the shards.
 
 | Parameter | Type |
 | ------ | ------ |
-| `__namedParameters` | \{ `dsn`: `string`; `from`: `number`; `to`: `number`; `migrateCmd`: `string`; `activate`: `boolean`; \} |
-| `__namedParameters.dsn` | `string` |
+| `__namedParameters` | \{ `dsns`: `string`[]; `from`: `number`; `to`: `number`; `migrateCmd`: `string`; `activate`: `boolean`; \} |
+| `__namedParameters.dsns` | `string`[] |
 | `__namedParameters.from` | `number` |
 | `__namedParameters.to` | `number` |
 | `__namedParameters.migrateCmd` | `string` |

docs/functions/cli.md

@@ -8,7 +8,7 @@
 
 > **cli**(): `void`
 
-Defined in: [src/cli.ts:197](https://github.com/clickup/pg-microsharding/blob/master/src/cli.ts#L197)
+Defined in: [src/cli.ts:207](https://github.com/clickup/pg-microsharding/blob/master/src/cli.ts#L207)
 
 A wrapper around main() to call it from a bin script.
 

docs/functions/main.md

@@ -8,7 +8,7 @@
 
 > **main**(`argsIn`): `Promise`\<`boolean`\>
 
-Defined in: [src/cli.ts:132](https://github.com/clickup/pg-microsharding/blob/master/src/cli.ts#L132)
+Defined in: [src/cli.ts:140](https://github.com/clickup/pg-microsharding/blob/master/src/cli.ts#L140)
 
 Tool main function.
 

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@clickup/pg-microsharding",
   "description": "Microshards support for PostgreSQL",
-  "version": "2.14.2",
+  "version": "2.15.1",
   "license": "MIT",
   "keywords": [
     "postgresql",
@@ -41,7 +41,7 @@
     "@types/lodash": "^4.14.175",
     "@types/minimist": "^1.2.2",
     "@types/node": "^20.4.1",
-    "@types/pg": "^8.6.1",
+    "@types/pg": "^8.11.11",
     "@types/prompts": "^2.4.0",
     "@types/sprintf-js": "^1.1.2",
     "@types/tmp": "^0.2.6",

src/actions/actionAllocate.ts

@@ -35,7 +35,7 @@ export async function actionAllocate(args: Args): Promise<boolean> {
     throw "Please provide --activate=yes or --activate=no";
   }
 
-  await allocate({ dsn: dsns[0], from, to, migrateCmd, activate });
+  await allocate({ dsns, from, to, migrateCmd, activate });
 
   return true;
 }

src/actions/actionList.ts

@@ -31,7 +31,7 @@ export async function actionList(args: Args): Promise<boolean> {
 
   const dsns = await normalizeDsns(args["dsns"] || args["dsn"]);
   if (dsns.length === 0) {
-    throw "Please provide --dsn or --dsns, DB DSNs to remove old schemas from";
+    throw "Please provide --dsn or --dsns, DB DSNs to list the microshards on";
   }
 
   const { islandNosToDsn, islands } = await calcIslandWeights({

src/actions/actionRebalance.ts

@@ -44,6 +44,7 @@ export async function actionRebalance(args: Args): Promise<boolean> {
   const parallelism = Number(args["parallelism"] ?? "") || 3;
   const deactivateSQL = String(args["deactivate-sql"] || "") || undefined;
   const wait = args["wait"];
+  const skipConfig = args["skip-config"];
 
   // Prepare the list of shards except the global shard (we never move it while
   // doing rebalance, it can always be done manually).
@@ -136,6 +137,7 @@ export async function actionRebalance(args: Args): Promise<boolean> {
           : []),
         ...(deactivateSQL ? ["--deactivate-sql", deactivateSQL] : []),
         ...(wait ? ["--wait"] : []),
+        ...(skipConfig ? ["--skip-config"] : []),
       ],
     })),
   );