restructuring and corrections to Replicator flag requirements; make example db and schema names consistent

Author: taroface

src/current/_includes/molt/fetch-data-load-output.md

@@ -33,33 +33,53 @@
 
 	`data extraction` messages are written for each table that is exported to the location in `--bucket-path`:
 
+	<section class="filter-content" markdown="1" data-scope="postgres oracle">
 	~~~ json
 	{"level":"info","table":"migration_schema.employees","time":"2025-02-10T14:28:11-05:00","message":"data extraction phase starting"}
 	~~~
 
 	~~~ json
 	{"level":"info","table":"migration_schema.employees","type":"summary","num_rows":200000,"export_duration_ms":1000,"export_duration":"000h 00m 01s","time":"2025-02-10T14:28:12-05:00","message":"data extraction from source complete"}
 	~~~
+	</section>
+
+	<section class="filter-content" markdown="1" data-scope="mysql">
+	~~~ json
+	{"level":"info","table":"public.employees","time":"2025-02-10T14:28:11-05:00","message":"data extraction phase starting"}
+	~~~
+
+	~~~ json
+	{"level":"info","table":"public.employees","type":"summary","num_rows":200000,"export_duration_ms":1000,"export_duration":"000h 00m 01s","time":"2025-02-10T14:28:12-05:00","message":"data extraction from source complete"}
+	~~~
+	</section>
 
 	`data import` messages are written for each table that is loaded into CockroachDB:
 
+	<section class="filter-content" markdown="1" data-scope="postgres">
 	~~~ json
 	{"level":"info","table":"migration_schema.employees","time":"2025-02-10T14:28:12-05:00","message":"starting data import on target"}
 	~~~
 
-	<section class="filter-content" markdown="1" data-scope="postgres">
 	~~~ json
 	{"level":"info","table":"migration_schema.employees","type":"summary","net_duration_ms":1899.748333,"net_duration":"000h 00m 01s","import_duration_ms":1160.523875,"import_duration":"000h 00m 01s","export_duration_ms":1000,"export_duration":"000h 00m 01s","num_rows":200000,"cdc_cursor":"0/43A1960","time":"2025-02-10T14:28:13-05:00","message":"data import on target for table complete"}
 	~~~
 	</section>
 
 	<section class="filter-content" markdown="1" data-scope="mysql">
 	~~~ json
-	{"level":"info","table":"migration_schema.employees","type":"summary","net_duration_ms":1899.748333,"net_duration":"000h 00m 01s","import_duration_ms":1160.523875,"import_duration":"000h 00m 01s","export_duration_ms":1000,"export_duration":"000h 00m 01s","num_rows":200000,"cdc_cursor":"4c658ae6-e8ad-11ef-8449-0242ac140006:1-29","time":"2025-02-10T14:28:13-05:00","message":"data import on target for table complete"}
+	{"level":"info","table":"public.employees","time":"2025-02-10T14:28:12-05:00","message":"starting data import on target"}
+	~~~
+
+	~~~ json
+	{"level":"info","table":"public.employees","type":"summary","net_duration_ms":1899.748333,"net_duration":"000h 00m 01s","import_duration_ms":1160.523875,"import_duration":"000h 00m 01s","export_duration_ms":1000,"export_duration":"000h 00m 01s","num_rows":200000,"cdc_cursor":"4c658ae6-e8ad-11ef-8449-0242ac140006:1-29","time":"2025-02-10T14:28:13-05:00","message":"data import on target for table complete"}
 	~~~
 	</section>
 
 	<section class="filter-content" markdown="1" data-scope="oracle">
+	~~~ json
+	{"level":"info","table":"migration_schema.employees","time":"2025-02-10T14:28:12-05:00","message":"starting data import on target"}
+	~~~
+
 	~~~ json
 	{"level":"info","table":"migration_schema.employees","type":"summary","net_duration_ms":1899.748333,"net_duration":"000h 00m 01s","import_duration_ms":1160.523875,"import_duration":"000h 00m 01s","export_duration_ms":1000,"export_duration":"000h 00m 01s","num_rows":200000,"cdc_cursor":"2358840","time":"2025-02-10T14:28:13-05:00","message":"data import on target for table complete"}
 	~~~
@@ -75,7 +95,7 @@
 
 	<section class="filter-content" markdown="1" data-scope="mysql">
 	~~~ json
-	{"level":"info","type":"summary","fetch_id":"f5cb422f-4bb4-4bbd-b2ae-08c4d00d1e7c","num_tables":3,"tables":["migration_schema.employees","migration_schema.payments","migration_schema.payments"],"cdc_cursor":"4c658ae6-e8ad-11ef-8449-0242ac140006:1-29","net_duration_ms":6752.847625,"net_duration":"000h 00m 06s","time":"2024-03-18T12:30:37-04:00","message":"fetch complete"}
+	{"level":"info","type":"summary","fetch_id":"f5cb422f-4bb4-4bbd-b2ae-08c4d00d1e7c","num_tables":3,"tables":["public.employees","public.payments","public.payments"],"cdc_cursor":"4c658ae6-e8ad-11ef-8449-0242ac140006:1-29","net_duration_ms":6752.847625,"net_duration":"000h 00m 06s","time":"2024-03-18T12:30:37-04:00","message":"fetch complete"}
 	~~~
 
 	{% if page.name != "migrate-bulk-load.md" %}

src/current/_includes/molt/fetch-schema-table-filtering.md

@@ -1,14 +1,23 @@
-MOLT Fetch can restrict which schemas (or users) and tables are migrated by using the following filter flags:
+Use the following flags to filter the data to be migrated:
 
+<section class="filter-content" markdown="1" data-scope="mysql">
+|      Filter type       |            Flag            |                               Description                                |
+|------------------------|----------------------------|--------------------------------------------------------------------------|
+| Table filter           | `--table-filter`           | POSIX regex matching table names to include across all selected schemas. |
+| Table exclusion filter | `--table-exclusion-filter` | POSIX regex matching table names to exclude across all selected schemas. |
+
+{{site.data.alerts.callout_info}}
+`--schema-filter` does not apply to MySQL sources because MySQL tables belong directly to the database specified in the connection string, not to a separate schema.
+{{site.data.alerts.end}}
+</section>
+
+<section class="filter-content" markdown="1" data-scope="postgres oracle">
 |      Filter type       |            Flag            |                                                                   Description                                                                   |
 |------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------|
 | Schema filter          | `--schema-filter`          | [POSIX regex](https://wikipedia.org/wiki/Regular_expression) matching schema names to include; all matching schemas and their tables are moved. |
 | Table filter           | `--table-filter`           | POSIX regex matching table names to include across all selected schemas.                                                                        |
 | Table exclusion filter | `--table-exclusion-filter` | POSIX regex matching table names to exclude across all selected schemas.                                                                        |
-
-{{site.data.alerts.callout_success}}
-Use `--schema-filter` to migrate only the specified schemas, and refine which tables are moved using `--table-filter` or `--table-exclusion-filter`.
-{{site.data.alerts.end}}
+</section>
 
 <section class="filter-content" markdown="1" data-scope="oracle">
 When migrating from Oracle, you **must** include `--schema-filter` to name an Oracle schema to migrate. This prevents Fetch from attempting to load tables owned by other users. For example:

src/current/_includes/molt/fetch-table-filter-userscript.md

@@ -4,14 +4,15 @@ When loading a subset of tables using `--table-filter`, you **must** provide a T
 
 For example, the following `table_filter.ts` userscript filters change events to the specified source tables:
 
+<section class="filter-content" markdown="1" data-scope="postgres oracle">
 ~~~ ts
 import * as api from "replicator@v1";
 
 // List the source tables (matching source names and casing) to include in replication
 const allowedTables = ["EMPLOYEES", "PAYMENTS", "ORDERS"];
 
 // Update this to your target CockroachDB database and schema name
-api.configureSource("defaultdb.migration_schema", {
+api.configureSource("molt.migration_schema", {
   dispatch: (doc: Document, meta: Document): Record<Table, Document[]> | null => {
     // Replicate only if the table matches one of the allowed tables
     if (allowedTables.includes(meta.table)) {
@@ -33,6 +34,39 @@ api.configureSource("defaultdb.migration_schema", {
   },
 });
 ~~~
+</section>
+
+<section class="filter-content" markdown="1" data-scope="mysql">
+~~~ ts
+import * as api from "replicator@v1";
+
+// List the source tables (matching source names and casing) to include in replication
+const allowedTables = ["EMPLOYEES", "PAYMENTS", "ORDERS"];
+
+// Update this to your target CockroachDB database and schema name
+api.configureSource("molt.public", {
+  dispatch: (doc: Document, meta: Document): Record<Table, Document[]> | null => {
+    // Replicate only if the table matches one of the allowed tables
+    if (allowedTables.includes(meta.table)) {
+      let ret: Record<Table, Document[]> = {};
+      ret[meta.table] = [doc];
+      return ret;
+    }
+    // Ignore all other tables
+    return null;
+  },
+  deletesTo: (doc: Document, meta: Document): Record<Table, Document[]> | null => {
+    // Optionally filter deletes the same way
+    if (allowedTables.includes(meta.table)) {
+      let ret: Record<Table, Document[]> = {};
+      ret[meta.table] = [doc];
+      return ret;
+    }
+    return null;
+  },
+});
+~~~
+</section>
 
 Pass the userscript to MOLT Replicator with the `--userscript` [flag](#replication-flags):
 

src/current/_includes/molt/migration-create-sql-user.md

@@ -14,7 +14,7 @@ Grant database-level privileges for schema creation within the target database:
 GRANT ALL ON DATABASE defaultdb TO crdb_user;
 ~~~
 
-Grant user privileges to create internal MOLT tables like `_molt_fetch_exceptions` in the public schema: 
+Grant user privileges to create internal MOLT tables like `_molt_fetch_exceptions` in the `public` CockroachDB schema: 
 
 {{site.data.alerts.callout_info}}
 Ensure that you are connected to the target database.
@@ -25,16 +25,18 @@ Ensure that you are connected to the target database.
 GRANT CREATE ON SCHEMA public TO crdb_user;
 ~~~
 
-If you manually created the target schema (i.e., [`drop-on-target-and-recreate`](#table-handling-mode) will not be used), grant the following privileges on the schema:
+If you manually defined the target tables (i.e., [`drop-on-target-and-recreate`](#table-handling-mode) will not be used), grant the following privileges on the schema:
 
+<section class="filter-content" markdown="1" data-scope="postgres oracle">
 {% include_cached copy-clipboard.html %}
 ~~~ sql
 GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA migration_schema TO crdb_user;
 ALTER DEFAULT PRIVILEGES IN SCHEMA migration_schema
 GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO crdb_user;
 ~~~
 
-Grant the same privileges for internal MOLT tables:
+Grant the same privileges for internal MOLT tables in the `public` CockroachDB schema:
+</section>
 
 {% include_cached copy-clipboard.html %}
 ~~~ sql
@@ -47,18 +49,27 @@ Depending on the MOLT Fetch [data load mode](#data-load-mode) you will use, gran
 
 #### `IMPORT INTO` privileges
 
-Grant `SELECT`, `INSERT`, and `DROP` (required because the table is taken offline during the `IMPORT INTO`) privileges on all tables in the [target schema](#create-the-target-schema):
+Grant `SELECT`, `INSERT`, and `DROP` (required because the table is taken offline during the `IMPORT INTO`) privileges on all tables being migrated:
 
+<section class="filter-content" markdown="1" data-scope="postgres oracle">
 {% include_cached copy-clipboard.html %}
 ~~~ sql
 GRANT SELECT, INSERT, DROP ON ALL TABLES IN SCHEMA migration_schema TO crdb_user;
 ~~~
+</section>
+
+<section class="filter-content" markdown="1" data-scope="mysql">
+{% include_cached copy-clipboard.html %}
+~~~ sql
+GRANT SELECT, INSERT, DROP ON ALL TABLES IN SCHEMA public TO crdb_user;
+~~~
+</section>
 
 If you plan to use [cloud storage with implicit authentication](#cloud-storage-authentication) for data load, grant the `EXTERNALIOIMPLICITACCESS` [system-level privilege]({% link {{site.current_cloud_version}}/security-reference/authorization.md %}#supported-privileges):
 
 {% include_cached copy-clipboard.html %}
 ~~~ sql
-GRANT EXTERNALIOIMPLICITACCESS TO crdb_user;
+GRANT SYSTEM EXTERNALIOIMPLICITACCESS TO crdb_user;
 ~~~
 
 #### `COPY FROM` privileges

src/current/_includes/molt/migration-prepare-database.md

@@ -45,11 +45,12 @@ Run the `ALTER TABLE` command for each table to replicate.
 CREATE USER 'migration_user'@'%' IDENTIFIED BY 'password';
 ~~~
 
-Grant the user privileges to select only the tables you migrate:
+Grant the user privileges to select the tables you migrate and access GTID information for snapshot consistency:
 
 {% include_cached copy-clipboard.html %}
 ~~~ sql
 GRANT SELECT ON source_database.* TO 'migration_user'@'%';
+GRANT SELECT ON mysql.gtid_executed TO 'migration_user'@'%';
 FLUSH PRIVILEGES;
 ~~~
 
@@ -74,7 +75,7 @@ CREATE USER MIGRATION_USER IDENTIFIED BY 'password';
 When migrating from Oracle Multitenant (PDB/CDB), this should be a [common user](https://docs.oracle.com/database/121/ADMQS/GUID-DA54EBE5-43EF-4B09-B8CC-FAABA335FBB8.htm). Prefix the username with `C##` (e.g., `C##MIGRATION_USER`).
 {{site.data.alerts.end}}
 
-Grant the user privileges to connect, read metadata, and `SELECT` and `FLASHBACK` the tables you plan to migrate. The tables should all reside in a single schema (e.g., `migration_schema`). For details, refer to [Schema and table filtering](#schema-and-table-filtering).
+Grant the user privileges to connect, read metadata, and `SELECT` and `FLASHBACK` the tables you plan to migrate. The tables should all reside in a single schema (for example, `migration_schema`). For details, refer to [Schema and table filtering](#schema-and-table-filtering).
 
 ##### Oracle Multitenant (PDB/CDB) user privileges
 

src/current/_includes/molt/migration-schema-design-practices.md

@@ -1,18 +1,39 @@
-Convert the source schema into a CockroachDB-compatible schema. CockroachDB supports the PostgreSQL wire protocol and is largely [compatible with PostgreSQL syntax]({% link {{ site.current_cloud_version }}/postgresql-compatibility.md %}#features-that-differ-from-postgresql).
+Convert the source table definitions into CockroachDB-compatible equivalents. CockroachDB supports the PostgreSQL wire protocol and is largely [compatible with PostgreSQL syntax]({% link {{ site.current_cloud_version }}/postgresql-compatibility.md %}#features-that-differ-from-postgresql).
 
-- The source and target schemas must **match**. Review [Type mapping]({% link molt/molt-fetch.md %}#type-mapping) to understand which source types can be mapped to CockroachDB types.
+- The source and target table definitions must **match**. Review [Type mapping]({% link molt/molt-fetch.md %}#type-mapping) to understand which source types can be mapped to CockroachDB types.
 
-	For example, a source table defined as `CREATE TABLE migration_schema.tbl (pk INT PRIMARY KEY);` must have a corresponding schema and table in CockroachDB:
+	<section class="filter-content" markdown="1" data-scope="postgres">
+	For example, a PostgreSQL source table defined as `CREATE TABLE migration_schema.tbl (pk INT PRIMARY KEY);` must have a corresponding schema and table in CockroachDB:
 
 	{% include_cached copy-clipboard.html %}
 	~~~ sql
 	CREATE SCHEMA migration_schema;
 	CREATE TABLE migration_schema.tbl (pk INT PRIMARY KEY);
 	~~~
+	</section>
+
+	<section class="filter-content" markdown="1" data-scope="mysql">
+	For MySQL sources, tables belong directly to the database specified in the connection string. A MySQL source table defined as `CREATE TABLE tbl (id INT PRIMARY KEY);` should map to CockroachDB's default `public` schema:
+
+	{% include_cached copy-clipboard.html %}
+	~~~ sql
+	CREATE TABLE tbl (id INT PRIMARY KEY);
+	~~~
+	</section>
+
+	<section class="filter-content" markdown="1" data-scope="oracle">
+	For example, an Oracle source table defined as `CREATE TABLE migration_schema.tbl (pk INT PRIMARY KEY);` must have a corresponding schema and table in CockroachDB:
+
+	{% include_cached copy-clipboard.html %}
+	~~~ sql
+	CREATE SCHEMA migration_schema;
+	CREATE TABLE migration_schema.tbl (pk INT PRIMARY KEY);
+	~~~
+	</section>
 
-	- MOLT Fetch can automatically create a matching CockroachDB schema using the {% if page.name != "migration-strategy.md" %}[`drop-on-target-and-recreate`](#table-handling-mode){% else %}[`drop-on-target-and-recreate`]({% link molt/molt-fetch.md %}#target-table-handling){% endif %} option.
+	- MOLT Fetch can automatically define matching CockroachDB tables using the {% if page.name != "migration-strategy.md" %}[`drop-on-target-and-recreate`](#table-handling-mode){% else %}[`drop-on-target-and-recreate`]({% link molt/molt-fetch.md %}#target-table-handling){% endif %} option.
 
-	- If you create the target schema manually, review how MOLT Fetch handles [type mismatches]({% link molt/molt-fetch.md %}#mismatch-handling). You can use the {% if page.name != "migration-strategy.md" %}[MOLT Schema Conversion Tool](#schema-conversion-tool){% else %}[MOLT Schema Conversion Tool]({% link cockroachcloud/migrations-page.md %}){% endif %} to create a matching schema.
+	- If you define the target tables manually, review how MOLT Fetch handles [type mismatches]({% link molt/molt-fetch.md %}#mismatch-handling). You can use the {% if page.name != "migration-strategy.md" %}[MOLT Schema Conversion Tool](#schema-conversion-tool){% else %}[MOLT Schema Conversion Tool]({% link cockroachcloud/migrations-page.md %}){% endif %} to create matching table definitions.
 
 	<section class="filter-content" markdown="1" data-scope="oracle">
 	- By default, table and column names are case-insensitive in MOLT Fetch. If using the [`--case-sensitive`]({% link molt/molt-fetch.md %}#global-flags) flag, schema, table, and column names must match Oracle's default uppercase identifiers. Use quoted names on the target to preserve case. For example, the following CockroachDB SQL statement will error:

src/current/_includes/molt/migration-stop-replication.md

@@ -4,7 +4,7 @@
 
 1. Wait for replication to drain, which means that all transactions that occurred on the source database have been fully processed and replicated to CockroachDB. There are two ways to determine that replication has fully drained:
 	- When replication is caught up, you will not see new `upserted rows` logs.
-	- If you set up the replication metrics endpoint with `--metricsAddr` in the preceding steps, metrics are available at:
+	- If you set up the replication metrics endpoint with [`--metricsAddr`]({% link molt/replicator-flags.md %}#metrics-addr) in the preceding steps, metrics are available at:
 
 		~~~ 
 		http://{host}:{port}/_/varz

src/current/_includes/molt/molt-docker.md

@@ -68,12 +68,12 @@ When testing locally, specify the host as follows:
     {% if page.name == "molt-replicator.md" %}
     ~~~
     --sourceConn 'postgres://postgres:[email protected]:5432/molt?sslmode=disable'
-    --targetConn "postgres://[email protected]:26257/molt?sslmode=disable"
+    --targetConn "postgres://[email protected]:26257/defaultdb?sslmode=disable"
     ~~~
     {% else %}
     ~~~
     --source 'postgres://postgres:[email protected]:5432/molt?sslmode=disable'
-    --target "postgres://[email protected]:26257/molt?sslmode=disable"
+    --target "postgres://[email protected]:26257/defaultdb?sslmode=disable"
     ~~~
     {% endif %}
 
@@ -82,11 +82,11 @@ When testing locally, specify the host as follows:
     {% if page.name == "molt-replicator.md" %}
     ~~~
     --sourceConn 'postgres://postgres:[email protected]:5432/molt?sslmode=disable'
-    --targetConn "postgres://[email protected]:26257/molt?sslmode=disable"
+    --targetConn "postgres://[email protected]:26257/defaultdb?sslmode=disable"
     ~~~
     {% else %}
     ~~~
     --source 'postgres://postgres:[email protected]:5432/molt?sslmode=disable'
-    --target "postgres://[email protected]:26257/molt?sslmode=disable"
+    --target "postgres://[email protected]:26257/defaultdb?sslmode=disable"
     ~~~
     {% endif %}

src/current/_includes/molt/molt-drop-constraints-indexes.md

@@ -24,5 +24,5 @@ To optimize data load performance, drop all non-`PRIMARY KEY` [constraints]({% l
 Do **not** drop [`PRIMARY KEY`]({% link {{ site.current_cloud_version }}/primary-key.md %}) constraints.
 {{site.data.alerts.end}}
 
-You can [recreate the constraints and indexes after loading the data](#modify-the-cockroachdb-schema).
+You can [recreate the constraints and indexes after loading the data](#add-constraints-and-indexes).
 {% endif %}