tests: Introduce doc example tests

Author: def-

ci/test/pipeline.template.yml

@@ -722,6 +722,18 @@ steps:
         coverage: skip
         sanitizer: skip
 
+      - id: doc-examples
+        label: Documentation Examples Code
+        inputs:
+          - doc/user/data/examples
+        depends_on: build-aarch64
+        timeout_in_minutes: 30
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: doc-examples
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+
   - id: deploy-website
     label: Deploy website
     depends_on: lint-docs

doc/user/data/examples/alter_sink.yml

@@ -7,6 +7,10 @@
     That is, assume you have a Kafka sink `avro_sink` created from `matview_old`
     (See [`CREATE SINK`:Kafka/Redpanda](/sql/create-sink/kafka/) for more
     information):
+  test_setup: |
+    > CREATE MATERIALIZED VIEW matview_old AS SELECT DISTINCT ON (key_col) 1 AS key_col;
+    > CREATE CONNECTION kafka_connection TO KAFKA (BROKER '${testdrive.kafka-addr}', SECURITY PROTOCOL PLAINTEXT);
+    > CREATE CONNECTION csr_connection TO CONFLUENT SCHEMA REGISTRY (URL '${testdrive.schema-registry-url}');
   code: |
     CREATE SINK avro_sink
       FROM matview_old
@@ -29,6 +33,8 @@
     to make progress. See [Valid schema changes](#valid-schema-changes) for
     details.
     {{< /note >}}
+  test_setup: |
+    > CREATE MATERIALIZED VIEW matview_new AS SELECT DISTINCT ON (key_col) 2 AS key_col;
   code: |
     ALTER SINK avro_sink
       SET FROM matview_new
@@ -48,6 +54,8 @@
     For workaround, see [Example: Handle cutover scenarios](#handle-cutover-scenarios)
     {{< /tip >}}
 
+  testable_results: false
+
 - name: "handle-cutover-scenarios-intro"
   description: |
     Because Materialize emits updates from the newly specified relation **only** if
@@ -91,6 +99,7 @@
 - name: "handle-cutover-scenarios-alter-sink-to-transition"
   code: |
     ALTER SINK avro_sink SET FROM transition;
+  testable: false  # TODO: Materialize could not prove that the specified upsert envelope key ("key_col") was a unique key of the underlying relation "materialize.public.transition". There are no known valid unique keys for the underlying relation.
 
 - name: "handle-cutover-scenarios-step-3-intro"
   description: |

doc/user/data/examples/create_source/example_postgres_source.yml

@@ -19,6 +19,18 @@
     PostgreSQL publication is `mz_source` and the connection to PostgreSQL is
     `pg_connection`.
 
+  test_setup: |
+    > CREATE SECRET pgpass AS 'postgres'
+    > CREATE CONNECTION pg_connection TO POSTGRES(HOST postgres, DATABASE postgres, USER postgres, PASSWORD SECRET pgpass);
+    $ postgres-execute connection=postgres://postgres:postgres@postgres
+    ALTER USER postgres WITH replication;
+    DROP SCHEMA IF EXISTS public CASCADE;
+    DROP PUBLICATION IF EXISTS mz_source;
+    CREATE SCHEMA PUBLIC;
+    CREATE TABLE items (a INT);
+    ALTER TABLE items REPLICA IDENTITY FULL;
+    CREATE PUBLICATION mz_source FOR ALL TABLES;
+
   code: |
     /* This example assumes:
     - In the upstream PostgreSQL, you have defined:
@@ -62,3 +74,5 @@
     {{< /note >}}
 
     For more information, see [`CREATE TABLE`](/sql/create-table/).
+
+  testable_results: false

doc/user/data/examples/create_table/example_postgres_table.yml

@@ -48,6 +48,22 @@
 
     {{< /note >}}
 
+  test_setup: |
+    > CREATE SECRET pgpass AS 'postgres'
+    > CREATE CONNECTION pg_connection TO POSTGRES(HOST postgres, DATABASE postgres, USER postgres, PASSWORD SECRET pgpass);
+    $ postgres-execute connection=postgres://postgres:postgres@postgres
+    ALTER USER postgres WITH replication;
+    DROP SCHEMA IF EXISTS public CASCADE;
+    DROP PUBLICATION IF EXISTS mz_source;
+    CREATE SCHEMA PUBLIC;
+    CREATE TABLE items (a INT);
+    ALTER TABLE items REPLICA IDENTITY FULL;
+    CREATE TABLE orders (a INT);
+    ALTER TABLE orders REPLICA IDENTITY FULL;
+    CREATE PUBLICATION mz_source FOR ALL TABLES;
+
+    > CREATE SOURCE pg_source FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source');
+
   code: |
     /* This example assumes:
       - In the upstream PostgreSQL, you have defined:
@@ -86,6 +102,9 @@
     file="shared-content/create-table-from-source-snapshotting.md" >}}
 
     {{< include-md file="shared-content/create-table-if-not-exists-tip.md" >}}
+
+  testable_results: false
+
 - name: "show-tables"
   description: |
     To verify that the table has been created, you can run [`SHOW
@@ -116,6 +135,9 @@
 
     For the list of supported PostgreSQL data types, refer to [supported
     types](/sql/create-table/postgres/#supported-data-types).
+  test_results:
+    - a true integer ""
+    - a true integer ""
 
 - name: "read-from-table"
   description: |

doc/user/data/examples/ingest_data/mysql/create_source_cloud.yml

@@ -3,6 +3,17 @@
     Once you have created the connection, you can use the connection in the
     [`CREATE SOURCE`](/sql/create-source/) command to connect to your MySQL instance and start ingesting
     data:
+  test_setup: |
+    > CREATE SECRET IF NOT EXISTS mysqlpass AS 'p@ssw0rd'
+    > CREATE CONNECTION IF NOT EXISTS mysql_connection TO MYSQL (HOST mysql, USER root, PASSWORD SECRET mysqlpass)
+
+    $ mysql-connect name=mysql url=mysql://root@mysql password=p@ssw0rd
+    $ mysql-execute name=mysql
+    DROP DATABASE IF EXISTS public;
+    CREATE DATABASE public;
+    USE public;
+    CREATE TABLE t (x INT);
+
   code: |
     CREATE SOURCE mz_source
       FROM MYSQL CONNECTION mysql_connection

doc/user/data/examples/ingest_data/postgres/create_connection_ips_cloud.yml

@@ -8,22 +8,38 @@
 
   code: |
     CREATE SECRET pgpass AS '<PASSWORD>';
+  test_replacements:
+    <PASSWORD>: postgres
 
 - name: "create-connection"
   description: |
     Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create a
     connection object with access and authentication details for Materialize to
     use:
 
+  test_setup: |
+    $ postgres-execute connection=postgres://postgres:postgres@postgres
+    DROP USER IF EXISTS materialize;
+    CREATE USER materialize WITH SUPERUSER PASSWORD 'postgres';
+    ALTER USER materialize WITH replication;
+    DROP SCHEMA IF EXISTS public CASCADE;
+    DROP PUBLICATION IF EXISTS mz_source;
+    CREATE SCHEMA PUBLIC;
+    CREATE TABLE items (a INT);
+    ALTER TABLE items REPLICA IDENTITY FULL;
+    CREATE PUBLICATION mz_source FOR ALL TABLES;
   code: |
     CREATE CONNECTION pg_connection TO POSTGRES (
       HOST '<host>',
       PORT 5432,
-      USER 'materialize',
+      USER materialize,
       PASSWORD SECRET pgpass,
       SSL MODE 'require',
       DATABASE '<database>'
     );
+  test_replacements:
+    <host>: postgres
+    <database>: postgres
 
 - name: "create-connection-options-general"
   description: |

doc/user/data/examples/ingest_data/postgres/create_connection_privatelink_cloud.yml

@@ -4,11 +4,15 @@
     client connected to Materialize, use the [`CREATE
     CONNECTION`](/sql/create-connection/#aws-privatelink) command to create an
     AWS PrivateLink connection:
+  test_setup: |
+    $ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
+    ALTER SYSTEM SET max_aws_privatelink_connections = 5
   code: |
     CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK (
       SERVICE NAME 'com.amazonaws.vpce.us-east-1.vpce-svc-0356210a8a432d9e9',
       AVAILABILITY ZONES ('use1-az1', 'use1-az2', 'use1-az4')
     );
+  testable: false  # TODO: Reenable when database-issues#9932 is fixed
 - name: "create-privatelink-connection-options-aurora"
   description: |
     - Replace the `SERVICE NAME` value with the service name you noted
@@ -35,10 +39,13 @@
     To connect to an AWS PrivateLink endpoint service in the **same region** as
     your Materialize environment:
 
+  test_setup: |
+    > DROP CONNECTION privatelink_svc
   code: |
     CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK ( SERVICE
           NAME 'com.amazonaws.vpce.<region_id>.vpce-svc-<endpoint_service_id>',
           AVAILABILITY ZONES ('use1-az1', 'use1-az2', 'use1-az4') );
+  testable: false  # TODO: Reenable when database-issues#9932 is fixed
 
 - name: "create-privatelink-connection-rds-in-region-options"
   description: |
@@ -60,11 +67,14 @@
 
     To connect to an AWS PrivateLink endpoint service in a **different region**
     to the one where your Materialize environment is deployed:
+  test_setup: |
+    > DROP CONNECTION privatelink_svc
   code: |
     CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK ( SERVICE
     NAME 'com.amazonaws.vpce.us-west-1.vpce-svc-<endpoint_service_id>', -- For
     now, the AVAILABILITY ZONES clause **is** required, but will be -- made
     optional in a future release. AVAILABILITY ZONES () );
+  testable: false  # TODO: Reenable when database-issues#9932 is fixed
 
 - name: "create-privatelink-connection-rds-cross-region-options"
   description: |-
@@ -84,6 +94,7 @@
     FROM mz_aws_privatelink_connections plc
     JOIN mz_connections c ON plc.id = c.id
     WHERE c.name = 'privatelink_svc';
+  testable: false  # TODO: Reenable when database-issues#9932 is fixed
 
 
 - name: "get-principal-privatelink-connection-results"
@@ -117,6 +128,8 @@
     VALIDATE CONNECTION privatelink_svc;
   results: |
     If no validation error is returned, move to the next step.
+  testable_results: false
+  testable: false  # TODO: Reenable when database-issues#9932 is fixed
 
 - name: "create-secret"
   description: |
@@ -126,6 +139,7 @@
 
   code: |
     CREATE SECRET pgpass AS '<PASSWORD>';
+  testable: false  # TODO: Reenable when database-issues#9932 is fixed
 
 - name: "create-connection"
   description: |
@@ -142,6 +156,7 @@
       DATABASE '<database>',
       AWS PRIVATELINK privatelink_svc
       );
+  testable: false  # TODO: Reenable when database-issues#9932 is fixed
 
 - name: "create-connection-options-aurora"
   description: |

doc/user/data/examples/ingest_data/postgres/create_connection_ssh_cloud.yml

@@ -11,6 +11,10 @@
         PORT <SSH_BASTION_PORT>,
         USER '<SSH_BASTION_USER>'
     );
+  test_replacements:
+    <SSH_BASTION_HOST>: ssh-bastion-host
+    <SSH_BASTION_PORT>: 22
+    <SSH_BASTION_USER>: mz
 
 - name: "create-ssh-tunnel-connection-options"
   description: |-
@@ -24,8 +28,12 @@
 - name: "get-public-keys-general"
   description: |
     Get Materialize's public keys for the SSH tunnel connection:
+  test_setup: |
+    $ set-regex match=.* replacement=X
   code: |
     SELECT * FROM mz_ssh_tunnel_connections;
+  test_results: |
+    X X X
 
 - name: "get-public-keys-aurora-rds-self-hosted"
   description: |
@@ -40,6 +48,8 @@
         mz_ssh_tunnel_connections USING(id)
     WHERE
         mz_connections.name = 'ssh_connection';
+  test_results: |
+    X X X X
 
 - name: "login-to-ssh-bastion-host"
   description: |
@@ -49,17 +59,21 @@
   code: |
     echo "ssh-ed25519 AAAA...76RH materialize" >> ~/.ssh/authorized_keys
     echo "ssh-ed25519 AAAA...hLYV materialize" >> ~/.ssh/authorized_keys
+  testable: false
 
 - name: "validate-ssh-tunnel-connection"
   description: |
     Back in the SQL client connected to Materialize, validate the SSH tunnel
     connection you created using the [`VALIDATE
     CONNECTION`](/sql/validate-connection) command:
 
+  test_setup: |
+    $ unset-regex
   code: |
     VALIDATE CONNECTION ssh_connection;
   results: |
     If no validation error is returned, move to the next step.
+  testable_results: false
 
 - name: "create-secret"
   description: |
@@ -69,12 +83,25 @@
 
   code: |
     CREATE SECRET pgpass AS '<PASSWORD>';
+  test_replacements:
+    <PASSWORD>: postgres
 
 - name: "create-connection"
   description: |
 
     Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create another connection object, this time with database access and authentication details for Materialize to use:
 
+  test_setup: |
+    $ postgres-execute connection=postgres://postgres:postgres@postgres
+    DROP USER IF EXISTS materialize;
+    CREATE USER materialize WITH SUPERUSER PASSWORD 'postgres';
+    ALTER USER materialize WITH replication;
+    DROP SCHEMA IF EXISTS public CASCADE;
+    DROP PUBLICATION IF EXISTS mz_source;
+    CREATE SCHEMA PUBLIC;
+    CREATE TABLE items (a INT);
+    ALTER TABLE items REPLICA IDENTITY FULL;
+    CREATE PUBLICATION mz_source FOR ALL TABLES;
   code: |
     CREATE CONNECTION pg_connection TO POSTGRES (
       HOST '<host>',
@@ -84,6 +111,9 @@
       DATABASE '<database>',
       SSH TUNNEL ssh_connection
       );
+  test_replacements:
+    <host>: postgres
+    <database>: postgres
 
 - name: "create-connection-options-general"
   description: |