diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f296092130..e93cf1346f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -38,6 +38,9 @@ v2/datastream-mongodb-to-firestore/ @GoogleCloudPlatform/FirestoreMigrationTool # Spanner Bulk migration template v2/sourcedb-to-spanner/ @GoogleCloudPlatform/spanner-migrations-team +# GCS Spanner Validation template +v2/gcs-spanner-dv/ @GoogleCloudPlatform/spanner-migrations-team + # Bigtable Import Export templates v1/src/main/java/com/google/cloud/teleport/bigtable @GoogleCloudPlatform/bigtable-migrations-team v1/src/test/java/com/google/cloud/teleport/bigtable @GoogleCloudPlatform/bigtable-migrations-team diff --git a/.github/codecov.yml b/.github/codecov.yml index f833cc78b7..6f56dbbb6d 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -27,6 +27,7 @@ component_management: - "v2/spanner-custom-shard/**" - "v2/sourcedb-to-spanner/**" - "v2/spanner-to-sourcedb/**" + - "v2/gcs-spanner-dv/**" statuses: - type: project informational: true @@ -58,3 +59,8 @@ component_management: - "v2/sourcedb-to-spanner/**" - "v2/spanner-common/**" - "!v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/constants/**" + - component_id: gcs-spanner-dv + name: gcs-spanner-dv + paths: + - "v2/gcs-spanner-dv/**" + - "v2/spanner-common/**" diff --git a/.github/workflows/java-pr.yml b/.github/workflows/java-pr.yml index 7413e80ba8..b64dc22337 100644 --- a/.github/workflows/java-pr.yml +++ b/.github/workflows/java-pr.yml @@ -38,6 +38,7 @@ on: - '!v2/spanner-migrations-sdk/**' - '!v2/spanner-custom-shard/**' - '!v2/sourcedb-to-spanner/**' + - '!v2/gcs-spanner-dv/**' # Exclude kafka paths from global run (covered in kafka-pr.yml) - '!v2/kafka-to-bigquery/**' - '!v2/kafka-to-gcs/**' diff --git a/.github/workflows/run-it-tests-beam-snapshots.yml b/.github/workflows/run-it-tests-beam-snapshots.yml index 1789e86221..0571f71fa8 100644 --- a/.github/workflows/run-it-tests-beam-snapshots.yml +++ b/.github/workflows/run-it-tests-beam-snapshots.yml @@ -50,8 +50,8 @@ jobs: - name: Calculate Next Beam Snapshot Version id: calculate_beam_snapshot run: | - if [ -n "${{ github.event.inputs.snapshot_version }}" ]; then - BEAM_SNAPSHOT_VERSION="${{ github.event.inputs.snapshot_version }}" + if [ -n "${GITHUB_EVENT_INPUTS_SNAPSHOT_VERSION}" ]; then + BEAM_SNAPSHOT_VERSION="${GITHUB_EVENT_INPUTS_SNAPSHOT_VERSION}" echo "Using provided Beam snapshot version: $BEAM_SNAPSHOT_VERSION" else CURRENT_BEAM_VERSION=$(mvn help:evaluate -Dexpression=beam.version -q -DforceStdout) @@ -69,10 +69,12 @@ jobs: echo "beam_snapshot_version=$BEAM_SNAPSHOT_VERSION" >> $GITHUB_OUTPUT shell: bash + env: + GITHUB_EVENT_INPUTS_SNAPSHOT_VERSION: ${{ github.event.inputs.snapshot_version }} - name: Update pom.xml for Beam Snapshot run: | - SNAPSHOT_VERSION="${{ steps.calculate_beam_snapshot.outputs.beam_snapshot_version }}" + SNAPSHOT_VERSION="${STEPS_CALCULATE_BEAM_SNAPSHOT_OUTPUTS_BEAM_SNAPSHOT_VERSION}" echo "Updating pom.xml to use Beam version: $SNAPSHOT_VERSION" # 1. Update Beam version @@ -98,6 +100,8 @@ jobs: echo "Final pom.xml changes:" git diff pom.xml.bak # Show changes against the original .bak file created by sed shell: bash + env: + STEPS_CALCULATE_BEAM_SNAPSHOT_OUTPUTS_BEAM_SNAPSHOT_VERSION: ${{ steps.calculate_beam_snapshot.outputs.beam_snapshot_version }} - name: Run Build run: ./cicd/run-build diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml index 7bada4c586..233237d5fc 100644 --- a/.github/workflows/spanner-pr.yml +++ b/.github/workflows/spanner-pr.yml @@ -38,6 +38,7 @@ on: - 'v2/gcs-to-sourcedb/**' - 'v2/sourcedb-to-spanner/**' - 'v2/spanner-to-sourcedb/**' + - 'v2/gcs-spanner-dv/**' # Git action files - '.github/workflows/spanner-pr.yml' schedule: diff --git a/v2/gcs-spanner-dv/pom.xml b/v2/gcs-spanner-dv/pom.xml new file mode 100644 index 0000000000..a6e01c6b69 --- /dev/null +++ b/v2/gcs-spanner-dv/pom.xml @@ -0,0 +1,72 @@ + + + + + 4.0.0 + + + com.google.cloud.teleport.v2 + dynamic-templates + 1.0-SNAPSHOT + + + gcs-spanner-dv + + + + com.google.cloud.teleport.v2 + common + ${project.version} + + + com.google.guava + guava + ${guava.version} + + + com.google.cloud + google-cloud-core + + + + + com.google.cloud.teleport + it-google-cloud-platform + ${project.version} + test + + + org.apache.beam + beam-it-jdbc + test + + + com.google.cloud.teleport.v2 + spanner-common + 1.0-SNAPSHOT + compile + + + org.mockito + mockito-inline + LATEST + test + + + diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/constants/GCSSpannerDVConstants.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/constants/GCSSpannerDVConstants.java new file mode 100644 index 0000000000..1f929890eb --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/constants/GCSSpannerDVConstants.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.constants; + +import com.google.cloud.teleport.v2.dto.ComparisonRecord; +import org.apache.beam.sdk.values.TupleTag; + +public class GCSSpannerDVConstants { + + public static final TupleTag SOURCE_TAG = new TupleTag() {}; + public static final TupleTag SPANNER_TAG = new TupleTag() {}; + public static final TupleTag MATCHED_TAG = new TupleTag() {}; + public static final TupleTag MISSING_IN_SPANNER_TAG = + new TupleTag() {}; + public static final TupleTag MISSING_IN_SOURCE_TAG = + new TupleTag() {}; +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/constants/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/constants/package-info.java new file mode 100644 index 0000000000..b3f6d5cd82 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/constants/package-info.java @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.constants; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/package-info.java new file mode 100644 index 0000000000..65f49c97f7 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dofn/package-info.java @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dofn; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/BigQuerySchemas.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/BigQuerySchemas.java new file mode 100644 index 0000000000..f29d8ac419 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/BigQuerySchemas.java @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; + +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.api.services.bigquery.model.TableSchema; +import com.google.common.collect.Lists; + +/** BigQuery schemas for the GCS to Spanner Data Validation pipeline. */ +public final class BigQuerySchemas { + + private BigQuerySchemas() {} + + public static final TableSchema MISMATCHED_RECORDS_SCHEMA = + new TableSchema() + .setFields( + Lists.newArrayList( + new TableFieldSchema() + .setName(MismatchedRecord.RUN_ID_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(MismatchedRecord.SCHEMA_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(MismatchedRecord.TABLE_NAME_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(MismatchedRecord.MISMATCH_TYPE_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(MismatchedRecord.RECORD_KEY_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(MismatchedRecord.SOURCE_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(MismatchedRecord.HASH_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"))); + + public static final TableSchema TABLE_VALIDATION_STATS_SCHEMA = + new TableSchema() + .setFields( + Lists.newArrayList( + new TableFieldSchema() + .setName(TableValidationStats.RUN_ID_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.SCHEMA_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.TABLE_NAME_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.STATUS_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.SOURCE_ROW_COUNT_COLUMN_NAME) + .setType("INTEGER") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.DESTINATION_ROW_COUNT_COLUMN_NAME) + .setType("INTEGER") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.MATCHED_ROW_COUNT_COLUMN_NAME) + .setType("INTEGER") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.MISMATCH_ROW_COUNT_COLUMN_NAME) + .setType("INTEGER") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.START_TIMESTAMP_COLUMN_NAME) + .setType("TIMESTAMP") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(TableValidationStats.END_TIMESTAMP_COLUMN_NAME) + .setType("TIMESTAMP") + .setMode("REQUIRED"))); + + public static final TableSchema VALIDATION_SUMMARY_SCHEMA = + new TableSchema() + .setFields( + Lists.newArrayList( + new TableFieldSchema() + .setName(ValidationSummary.RUN_ID_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(ValidationSummary.SOURCE_DATABASE_COLUMN_NAME) + .setType("STRING") + .setMode("NULLABLE"), + new TableFieldSchema() + .setName(ValidationSummary.DESTINATION_DATABASE_COLUMN_NAME) + .setType("STRING") + .setMode("NULLABLE"), + new TableFieldSchema() + .setName(ValidationSummary.STATUS_COLUMN_NAME) + .setType("STRING") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(ValidationSummary.TOTAL_TABLES_VALIDATED_COLUMN_NAME) + .setType("INTEGER") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(ValidationSummary.TABLES_WITH_MISMATCHES_COLUMN_NAME) + .setType("STRING") + .setMode("NULLABLE"), + new TableFieldSchema() + .setName(ValidationSummary.TOTAL_ROWS_MATCHED_COLUMN_NAME) + .setType("INTEGER") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(ValidationSummary.TOTAL_ROWS_MISMATCHED_COLUMN_NAME) + .setType("INTEGER") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(ValidationSummary.START_TIMESTAMP_COLUMN_NAME) + .setType("TIMESTAMP") + .setMode("REQUIRED"), + new TableFieldSchema() + .setName(ValidationSummary.END_TIMESTAMP_COLUMN_NAME) + .setType("TIMESTAMP") + .setMode("REQUIRED"))); +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/Column.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/Column.java new file mode 100644 index 0000000000..2ca2f7d395 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/Column.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; + +import com.google.auto.value.AutoValue; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; + +@AutoValue +@DefaultSchema(AutoValueSchema.class) +public abstract class Column { + + public abstract String getColName(); + + public abstract String getColValue(); + + public static Builder builder() { + return new AutoValue_Column.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setColName(String colName); + + public abstract Builder setColValue(String colValue); + + public abstract Column build(); + } +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ComparisonRecord.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ComparisonRecord.java new file mode 100644 index 0000000000..dcc79c04bf --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ComparisonRecord.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; + +import com.google.auto.value.AutoValue; +import java.util.List; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; + +@AutoValue +@DefaultSchema(AutoValueSchema.class) +public abstract class ComparisonRecord { + + public abstract String getTableName(); + + public abstract List getPrimaryKeyColumns(); + + public abstract String getHash(); + + public static Builder builder() { + return new AutoValue_ComparisonRecord.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setTableName(String tableName); + + public abstract Builder setPrimaryKeyColumns(List primaryKeyColumns); + + public abstract Builder setHash(String hash); + + public abstract ComparisonRecord build(); + } +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/MismatchedRecord.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/MismatchedRecord.java new file mode 100644 index 0000000000..49dcc1d706 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/MismatchedRecord.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; + +import com.google.auto.value.AutoValue; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; + +/** Represents a row in the MismatchedRecords table in BigQuery. */ +@AutoValue +@DefaultSchema(AutoValueSchema.class) +public abstract class MismatchedRecord { + public static final String RUN_ID_COLUMN_NAME = "run_id"; + public static final String SCHEMA_NAME = "schema_name"; + public static final String TABLE_NAME_COLUMN_NAME = "table_name"; + public static final String MISMATCH_TYPE_COLUMN_NAME = "mismatch_type"; + public static final String RECORD_KEY_COLUMN_NAME = "record_key"; + public static final String SOURCE_COLUMN_NAME = "source"; + public static final String HASH_COLUMN_NAME = "hash"; + + public abstract String getRunId(); + + public abstract String getSchemaName(); + + public abstract String getTableName(); + + public abstract String getMismatchType(); + + public abstract String getRecordKey(); + + public abstract String getSource(); + + public abstract String getHash(); + + public static Builder builder() { + return new AutoValue_MismatchedRecord.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setRunId(String runId); + + public abstract Builder setSchemaName(String schemaName); + + public abstract Builder setTableName(String tableName); + + public abstract Builder setMismatchType(String mismatchType); + + public abstract Builder setRecordKey(String recordKey); + + public abstract Builder setSource(String source); + + public abstract Builder setHash(String hash); + + public abstract MismatchedRecord build(); + } +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/SpannerTableReadConfiguration.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/SpannerTableReadConfiguration.java new file mode 100644 index 0000000000..c5e4c7b1f2 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/SpannerTableReadConfiguration.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; + +import com.google.auto.value.AutoValue; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; + +@AutoValue +@DefaultSchema(AutoValueSchema.class) +public abstract class SpannerTableReadConfiguration { + + public abstract String getTableName(); + + /** Stores the list of columns when reading from Spanner. By default, all columns are read. */ + @Nullable + public abstract List getColumnsToInclude(); + + /** + * Stores the list of columns to exclude when reading from Spanner. By default, all columns are + * read. + */ + @Nullable + public abstract List getColumnsToExclude(); + + /** Allows the user to specify a custom query at a table level for reading from Spanner. */ + @Nullable + public abstract String getCustomQuery(); + + public static Builder builder() { + return new AutoValue_SpannerTableReadConfiguration.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setTableName(String tableName); + + public abstract Builder setColumnsToInclude(List columnsToInclude); + + public abstract Builder setColumnsToExclude(List columnsToExclude); + + public abstract Builder setCustomQuery(String customQuery); + + public abstract SpannerTableReadConfiguration build(); + } +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/TableValidationStats.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/TableValidationStats.java new file mode 100644 index 0000000000..521343a69f --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/TableValidationStats.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; + +import com.google.auto.value.AutoValue; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.joda.time.Instant; + +/** Represents a row in the TableValidationStats table in BigQuery. */ +@AutoValue +@DefaultSchema(AutoValueSchema.class) +public abstract class TableValidationStats { + public static final String RUN_ID_COLUMN_NAME = "run_id"; + public static final String SCHEMA_NAME = "schema_name"; + public static final String TABLE_NAME_COLUMN_NAME = "table_name"; + public static final String STATUS_COLUMN_NAME = "status"; + public static final String SOURCE_ROW_COUNT_COLUMN_NAME = "source_row_count"; + public static final String DESTINATION_ROW_COUNT_COLUMN_NAME = "destination_row_count"; + public static final String MATCHED_ROW_COUNT_COLUMN_NAME = "matched_row_count"; + public static final String MISMATCH_ROW_COUNT_COLUMN_NAME = "mismatch_row_count"; + public static final String START_TIMESTAMP_COLUMN_NAME = "start_timestamp"; + public static final String END_TIMESTAMP_COLUMN_NAME = "end_timestamp"; + + public abstract String getRunId(); + + public abstract String getSchemaName(); + + public abstract String getTableName(); + + public abstract String getStatus(); + + public abstract Long getSourceRowCount(); + + public abstract Long getDestinationRowCount(); + + public abstract Long getMatchedRowCount(); + + public abstract Long getMismatchRowCount(); + + public abstract Instant getStartTimestamp(); + + public abstract Instant getEndTimestamp(); + + public static Builder builder() { + return new AutoValue_TableValidationStats.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setRunId(String runId); + + public abstract Builder setSchemaName(String schemaName); + + public abstract Builder setTableName(String tableName); + + public abstract Builder setStatus(String status); + + public abstract Builder setSourceRowCount(Long sourceRowCount); + + public abstract Builder setDestinationRowCount(Long destinationRowCount); + + public abstract Builder setMatchedRowCount(Long matchedRowCount); + + public abstract Builder setMismatchRowCount(Long mismatchRowCount); + + public abstract Builder setStartTimestamp(Instant startTimestamp); + + public abstract Builder setEndTimestamp(Instant endTimestamp); + + public abstract TableValidationStats build(); + } +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ValidationSummary.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ValidationSummary.java new file mode 100644 index 0000000000..73c060a0a8 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ValidationSummary.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; + +import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.joda.time.Instant; + +/** Represents a row in the ValidationSummary table in BigQuery. */ +@AutoValue +@DefaultSchema(AutoValueSchema.class) +public abstract class ValidationSummary { + + public static final String RUN_ID_COLUMN_NAME = "run_id"; + public static final String SOURCE_DATABASE_COLUMN_NAME = "source_database"; + public static final String DESTINATION_DATABASE_COLUMN_NAME = "destination_database"; + public static final String STATUS_COLUMN_NAME = "status"; + public static final String TOTAL_TABLES_VALIDATED_COLUMN_NAME = "total_tables_validated"; + public static final String TABLES_WITH_MISMATCHES_COLUMN_NAME = "tables_with_mismatches"; + public static final String TOTAL_ROWS_MATCHED_COLUMN_NAME = "total_rows_matched"; + public static final String TOTAL_ROWS_MISMATCHED_COLUMN_NAME = "total_rows_mismatched"; + public static final String START_TIMESTAMP_COLUMN_NAME = "start_timestamp"; + public static final String END_TIMESTAMP_COLUMN_NAME = "end_timestamp"; + + public abstract String getRunId(); + + @Nullable + public abstract String getSourceDatabase(); + + @Nullable + public abstract String getDestinationDatabase(); + + public abstract String getStatus(); + + public abstract Long getTotalTablesValidated(); + + @Nullable + public abstract String getTablesWithMismatches(); + + public abstract Long getTotalRowsMatched(); + + public abstract Long getTotalRowsMismatched(); + + public abstract Instant getStartTimestamp(); + + public abstract Instant getEndTimestamp(); + + public static Builder builder() { + return new AutoValue_ValidationSummary.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setRunId(String runId); + + public abstract Builder setSourceDatabase(String sourceDatabase); + + public abstract Builder setDestinationDatabase(String destinationDatabase); + + public abstract Builder setStatus(String status); + + public abstract Builder setTotalTablesValidated(Long totalTablesValidated); + + public abstract Builder setTablesWithMismatches(String tablesWithMismatches); + + public abstract Builder setTotalRowsMatched(Long totalRowsMatched); + + public abstract Builder setTotalRowsMismatched(Long totalRowsMismatched); + + public abstract Builder setStartTimestamp(Instant startTimestamp); + + public abstract Builder setEndTimestamp(Instant endTimestamp); + + public abstract ValidationSummary build(); + } +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ValidationSummaryAccumulator.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ValidationSummaryAccumulator.java new file mode 100644 index 0000000000..ed441939b3 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/ValidationSummaryAccumulator.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** Accumulator for the custom CombineFn. */ +public class ValidationSummaryAccumulator implements Serializable { + public long totalTables = 0; + public long totalMatched = 0; + public long totalMismatched = 0; + public List tablesWithMismatches = new ArrayList<>(); +} diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/package-info.java new file mode 100644 index 0000000000..5d1f85e77b --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/dto/package-info.java @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.dto; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/fn/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/fn/package-info.java new file mode 100644 index 0000000000..bb468b1c78 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/fn/package-info.java @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.fn; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/mapper/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/mapper/package-info.java new file mode 100644 index 0000000000..c5efed8fac --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/mapper/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** Google Cloud Teleport templates that process data within Google Cloud. */ +/** Mapper classes for Dataflow Templates. */ +package com.google.cloud.teleport.v2.mapper; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/package-info.java new file mode 100644 index 0000000000..f07f6e88e1 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/templates/package-info.java @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** Google Cloud Teleport templates that process data within Google Cloud. */ +package com.google.cloud.teleport.v2.templates; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/package-info.java new file mode 100644 index 0000000000..1386954151 --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/transforms/package-info.java @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.transforms; diff --git a/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/package-info.java b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/package-info.java new file mode 100644 index 0000000000..9ec0e3ed2b --- /dev/null +++ b/v2/gcs-spanner-dv/src/main/java/com/google/cloud/teleport/v2/visitor/package-info.java @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** Google Cloud Teleport templates that process data within Google Cloud. */ +package com.google.cloud.teleport.v2.visitor; diff --git a/v2/pom.xml b/v2/pom.xml index acc9f94a04..e4381e0798 100644 --- a/v2/pom.xml +++ b/v2/pom.xml @@ -706,6 +706,7 @@ elasticsearch-common file-format-conversion gcs-to-sourcedb + gcs-spanner-dv googlecloud-to-elasticsearch googlecloud-to-googlecloud googlecloud-and-jms