add passing alter_table.md

Author: ptravers

ci/test/pipeline.template.yml

@@ -378,20 +378,68 @@ steps:
       # Too slow on aarch64
       queue: hetzner-x86-64-4cpu-8gb
 
-  - id: sql-server-cdc
-    label: "SQL Server CDC tests"
-    depends_on: build-x86_64
-    timeout_in_minutes: 30
-    inputs: [test/sql-server-cdc]
-    parallelism: 2
-    plugins:
-      - ./ci/plugins/mzcompose:
-          composition: sql-server-cdc
-    agents:
-      # The SQL Server Docker image isn't available on ARM.
-      #
-      # See: <https://github.com/microsoft/mssql-docker/issues/864>
-      queue: hetzner-x86-64-8cpu-16gb
+  - group: "SQL Server tests"
+    key: sql-server-tests
+    steps:
+      - id: sql-server-cdc
+        label: "SQL Server CDC tests"
+        depends_on: build-x86_64
+        timeout_in_minutes: 30
+        inputs: [test/sql-server-cdc]
+        parallelism: 2
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: sql-server-cdc
+        agents:
+          # The SQL Server Docker image isn't available on ARM.
+          #
+          # See: <https://github.com/microsoft/mssql-docker/issues/864>
+          queue: hetzner-x86-64-8cpu-16gb
+
+      - id: sql-server-cdc-old-syntax
+        label: "SQL Server CDC old syntax tests"
+        depends_on: build-x86_64
+        timeout_in_minutes: 30
+        inputs: [test/sql-server-cdc-old-syntax]
+        parallelism: 2
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: sql-server-cdc
+        agents:
+          # The SQL Server Docker image isn't available on ARM.
+          #
+          # See: <https://github.com/microsoft/mssql-docker/issues/864>
+          queue: hetzner-x86-64-8cpu-16gb
+
+      #- id: sql-server-cdc-resumption-old-syntax
+      #  label: "SQL Server resumption old syntax tests"
+      #  depends_on: build-x86_64
+      #  timeout_in_minutes: 30
+      #  inputs: [test/sql-server-resumption-old-syntax]
+      #  parallelism: 2
+      #  plugins:
+      #    - ./ci/plugins/mzcompose:
+      #        composition: sql-server-cdc
+      #  agents:
+      #    # The SQL Server Docker image isn't available on ARM.
+      #    #
+      #    # See: <https://github.com/microsoft/mssql-docker/issues/864>
+      #    queue: hetzner-x86-64-8cpu-16gb
+
+      - id: sql-server-rtr-old-syntax
+        label: "SQL Server RTR old syntax tests"
+        depends_on: build-x86_64
+        timeout_in_minutes: 30
+        inputs: [test/sql-server-rtr-old-syntax]
+        parallelism: 2
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: sql-server-cdc
+        agents:
+          # The SQL Server Docker image isn't available on ARM.
+          #
+          # See: <https://github.com/microsoft/mssql-docker/issues/864>
+          queue: hetzner-x86-64-8cpu-16gb
 
   - group: "Connection tests"
     key: connection-tests

test/sql-server-cdc-old-syntax/alter-source.td

@@ -11,19 +11,12 @@ $ sql-server-connect name=sql-server
 server=tcp:sql-server,1433;IntegratedSecurity=true;TrustServerCertificate=true;User ID=${arg.default-sql-server-user};Password=${arg.default-sql-server-password}
 
 $ sql-server-execute name=sql-server
-IF EXISTS (SELECT 1 FROM sys.databases WHERE name = 'test') BEGIN ALTER DATABASE test SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE test; END;
-CREATE DATABASE test;
 USE test;
-EXEC sys.sp_cdc_enable_db;
-ALTER DATABASE test SET ALLOW_SNAPSHOT_ISOLATION ON;
-
 CREATE TABLE table_a (pk INTEGER PRIMARY KEY, f2 VARCHAR(128));
 INSERT INTO table_a VALUES (1, 'one');
 INSERT INTO table_a VALUES (2, 'two');
 EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'table_a', @role_name = 'SA', @supports_net_changes = 0;
 
-# Check empty publication on ALTER
-
 > CREATE SECRET IF NOT EXISTS sql_server_pass AS '${arg.default-sql-server-password}'
 
 > DROP CONNECTION IF EXISTS sql_server_test_connection CASCADE
@@ -40,6 +33,7 @@ EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'table_a', @
 > CREATE SOURCE mz_source
   FROM SQL SERVER CONNECTION sql_server_test_connection
   FOR ALL TABLES;
+> DROP SOURCE dummy;
 
 > SELECT * FROM table_a;
 1 one
@@ -48,7 +42,7 @@ EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'table_a', @
 $ sql-server-execute name=sql-server
 DROP TABLE table_a;
 
-!ALTER SOURCE mz_source ADD SUBSOURCE table_b;
+! ALTER SOURCE mz_source ADD SUBSOURCE table_b;
 contains:reference to table_b not found in source
 
 # Adding a table with the same name as a running table does not allow you to add
@@ -115,6 +109,8 @@ INSERT INTO table_g VALUES (2, 'two');
   )
   FOR ALL TABLES;
 
+> DROP SOURCE dummy;
+
 > SHOW SUBSOURCES ON mz_source
 mz_source_progress    progress
 table_a               subsource

test/sql-server-cdc-old-syntax/mzcompose.py

@@ -12,26 +12,31 @@
 """
 
 import glob
+import pathlib
 import random
 
-from materialize import MZ_ROOT
-from materialize.mzcompose.composition import Composition, WorkflowArgumentParser
+from materialize import MZ_ROOT, buildkite
+from materialize.mzcompose.composition import (
+    Composition,
+    WorkflowArgumentParser,
+)
 from materialize.mzcompose.services.materialized import Materialized
 from materialize.mzcompose.services.mz import Mz
 from materialize.mzcompose.services.sql_server import SqlServer
 from materialize.mzcompose.services.testdrive import Testdrive
 
 SERVICES = [
     Mz(app_password=""),
-    Materialized(),
+    Materialized(
+        additional_system_parameter_defaults={
+            "log_filter": "mz_storage::source::sql_server=trace,mz_storage::source::sql_server::replication=trace,mz_sql_server_util=debug,info"
+        },
+    ),
     Testdrive(),
     SqlServer(),
 ]
 
 
-#
-# Test that SQL Server ingestion works
-#
 def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     parser.add_argument(
         "filter",
@@ -41,26 +46,35 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     )
     args = parser.parse_args()
 
-    matching_files = []
+    matching_files: list[str] = []
     for filter in args.filter:
         matching_files.extend(
             glob.glob(filter, root_dir=MZ_ROOT / "test" / "sql-server-cdc-old-syntax")
         )
     matching_files = sorted(matching_files)
-    print(f"Filter: {args.filter} Files: {matching_files}")
+    sharded_files: list[str] = buildkite.shard_list(
+        sorted(matching_files), lambda file: file
+    )
+    print(f"Filter: {args.filter} Files: {sharded_files}")
+
+    # Start with a fresh state
+    c.kill("sql-server")
+    c.rm("sql-server")
+    c.kill("materialized")
+    c.rm("materialized")
 
     c.up("materialized", "sql-server")
     seed = random.getrandbits(16)
 
-    c.test_parts(
-        matching_files,
-        lambda file: c.run_testdrive_files(
-            "--no-reset",
+    def run(file: pathlib.Path | str) -> None:
+        c.run_testdrive_files(
             "--max-errors=1",
             f"--seed={seed}",
             f"--var=default-replica-size={Materialized.Size.DEFAULT_SIZE}-{Materialized.Size.DEFAULT_SIZE}",
             f"--var=default-sql-server-user={SqlServer.DEFAULT_USER}",
             f"--var=default-sql-server-password={SqlServer.DEFAULT_SA_PASSWORD}",
+            "setup/setup.td",
             str(file),
-        ),
-    )
+        )
+
+    c.test_parts(sharded_files, run)

test/sql-server-cdc-old-syntax/setup/setup.td

@@ -0,0 +1,58 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+$ sql-server-connect name=sql-server
+server=tcp:sql-server,1433;IntegratedSecurity=true;TrustServerCertificate=true;User ID=${arg.default-sql-server-user};Password=${arg.default-sql-server-password}
+
+$ sql-server-execute name=sql-server split-lines=false
+IF EXISTS (SELECT name FROM sys.databases WHERE name = N'test')
+BEGIN
+    ALTER DATABASE test SET SINGLE_USER WITH ROLLBACK IMMEDIATE;
+    DROP DATABASE test;
+END;
+
+$ sql-server-execute name=sql-server split-lines=false
+USE msdb;
+IF EXISTS (
+    SELECT 1
+    FROM msdb.dbo.sysjobs
+    WHERE name = N'DummyTicker'
+)
+BEGIN
+    EXEC sp_delete_job @job_name = N'DummyTicker';
+END;
+
+$ sql-server-execute name=sql-server
+CREATE DATABASE test COLLATE Latin1_General_100_CI_AI_SC_UTF8;
+USE test;
+
+EXEC sys.sp_cdc_enable_db;
+ALTER DATABASE test SET ALLOW_SNAPSHOT_ISOLATION ON;
+
+CREATE TABLE dummy (t datetime);
+INSERT INTO dummy VALUES (CURRENT_TIMESTAMP);
+EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'dummy', @role_name = 'SA', @supports_net_changes = 0;
+
+$ sql-server-execute name=sql-server split-lines=false
+CREATE OR ALTER PROCEDURE dbo.DummyTicker AS
+BEGIN
+    SET NOCOUNT ON;
+    WHILE 1 = 1
+    BEGIN
+        UPDATE dbo.dummy SET t = CURRENT_TIMESTAMP;
+        WAITFOR DELAY '00:00:01';
+    END
+END;
+
+$ sql-server-execute name=sql-server
+USE msdb;
+EXEC sp_add_job @job_name = N'DummyTicker', @enabled = 1;
+EXEC sp_add_jobstep @job_name = N'DummyTicker', @step_name = N'run', @subsystem = N'TSQL', @database_name = N'test', @command = N'EXEC dbo.DummyTicker;';
+EXEC sp_add_jobserver @job_name = N'DummyTicker';
+EXEC msdb.dbo.sp_start_job N'DummyTicker';