Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ v2/datastream-mongodb-to-firestore/ @GoogleCloudPlatform/FirestoreMigrationTool
# Spanner Bulk migration template
v2/sourcedb-to-spanner/ @GoogleCloudPlatform/spanner-migrations-team

# GCS Spanner Validation template
v2/gcs-spanner-dv/ @GoogleCloudPlatform/spanner-migrations-team

# Bigtable Import Export templates
v1/src/main/java/com/google/cloud/teleport/bigtable @GoogleCloudPlatform/bigtable-migrations-team
v1/src/test/java/com/google/cloud/teleport/bigtable @GoogleCloudPlatform/bigtable-migrations-team
Expand Down
6 changes: 6 additions & 0 deletions .github/codecov.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ component_management:
- "v2/spanner-custom-shard/**"
- "v2/sourcedb-to-spanner/**"
- "v2/spanner-to-sourcedb/**"
- "v2/gcs-spanner-dv/**"
statuses:
- type: project
informational: true
Expand Down Expand Up @@ -58,3 +59,8 @@ component_management:
- "v2/sourcedb-to-spanner/**"
- "v2/spanner-common/**"
- "!v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/constants/**"
- component_id: gcs-spanner-dv
name: gcs-spanner-dv
paths:
- "v2/gcs-spanner-dv/**"
- "v2/spanner-common/**"
1 change: 1 addition & 0 deletions .github/workflows/java-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ on:
- '!v2/spanner-migrations-sdk/**'
- '!v2/spanner-custom-shard/**'
- '!v2/sourcedb-to-spanner/**'
- '!v2/gcs-spanner-dv/**'
# Exclude kafka paths from global run (covered in kafka-pr.yml)
- '!v2/kafka-to-bigquery/**'
- '!v2/kafka-to-gcs/**'
Expand Down
10 changes: 7 additions & 3 deletions .github/workflows/run-it-tests-beam-snapshots.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ jobs:
- name: Calculate Next Beam Snapshot Version
id: calculate_beam_snapshot
run: |
if [ -n "${{ github.event.inputs.snapshot_version }}" ]; then
BEAM_SNAPSHOT_VERSION="${{ github.event.inputs.snapshot_version }}"
if [ -n "${GITHUB_EVENT_INPUTS_SNAPSHOT_VERSION}" ]; then
BEAM_SNAPSHOT_VERSION="${GITHUB_EVENT_INPUTS_SNAPSHOT_VERSION}"
echo "Using provided Beam snapshot version: $BEAM_SNAPSHOT_VERSION"
else
CURRENT_BEAM_VERSION=$(mvn help:evaluate -Dexpression=beam.version -q -DforceStdout)
Expand All @@ -69,10 +69,12 @@ jobs:

echo "beam_snapshot_version=$BEAM_SNAPSHOT_VERSION" >> $GITHUB_OUTPUT
shell: bash
env:
GITHUB_EVENT_INPUTS_SNAPSHOT_VERSION: ${{ github.event.inputs.snapshot_version }}

- name: Update pom.xml for Beam Snapshot
run: |
SNAPSHOT_VERSION="${{ steps.calculate_beam_snapshot.outputs.beam_snapshot_version }}"
SNAPSHOT_VERSION="${STEPS_CALCULATE_BEAM_SNAPSHOT_OUTPUTS_BEAM_SNAPSHOT_VERSION}"
echo "Updating pom.xml to use Beam version: $SNAPSHOT_VERSION"

# 1. Update Beam version
Expand All @@ -98,6 +100,8 @@ jobs:
echo "Final pom.xml changes:"
git diff pom.xml.bak # Show changes against the original .bak file created by sed
shell: bash
env:
STEPS_CALCULATE_BEAM_SNAPSHOT_OUTPUTS_BEAM_SNAPSHOT_VERSION: ${{ steps.calculate_beam_snapshot.outputs.beam_snapshot_version }}

- name: Run Build
run: ./cicd/run-build
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/spanner-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ on:
- 'v2/gcs-to-sourcedb/**'
- 'v2/sourcedb-to-spanner/**'
- 'v2/spanner-to-sourcedb/**'
- 'v2/gcs-spanner-dv/**'
# Git action files
- '.github/workflows/spanner-pr.yml'
schedule:
Expand Down
72 changes: 72 additions & 0 deletions v2/gcs-spanner-dv/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
~ Copyright (C) 2026 Google Inc.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not
~ use this file except in compliance with the License. You may obtain a copy of
~ the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ License for the specific language governing permissions and limitations under
~ the License.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.google.cloud.teleport.v2</groupId>
<artifactId>dynamic-templates</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>

<artifactId>gcs-spanner-dv</artifactId>

<dependencies>
<dependency>
<groupId>com.google.cloud.teleport.v2</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-core</artifactId>
</dependency>

<!-- Test Dependencies -->
<dependency>
<groupId>com.google.cloud.teleport</groupId>
<artifactId>it-google-cloud-platform</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-it-jdbc</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.cloud.teleport.v2</groupId>
<artifactId>spanner-common</artifactId>
<version>1.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>LATEST</version> <!-- Use the latest version available -->
<scope>test</scope>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Copyright (C) 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.cloud.teleport.v2.constants;

import com.google.cloud.teleport.v2.dto.ComparisonRecord;
import org.apache.beam.sdk.values.TupleTag;

public class GCSSpannerDVConstants {

public static final TupleTag<ComparisonRecord> SOURCE_TAG = new TupleTag<ComparisonRecord>() {};
public static final TupleTag<ComparisonRecord> SPANNER_TAG = new TupleTag<ComparisonRecord>() {};
public static final TupleTag<ComparisonRecord> MATCHED_TAG = new TupleTag<ComparisonRecord>() {};
public static final TupleTag<ComparisonRecord> MISSING_IN_SPANNER_TAG =
new TupleTag<ComparisonRecord>() {};
public static final TupleTag<ComparisonRecord> MISSING_IN_SOURCE_TAG =
new TupleTag<ComparisonRecord>() {};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Copyright (C) 2026 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.cloud.teleport.v2.constants;
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Copyright (C) 2026 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.cloud.teleport.v2.dofn;
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/*
* Copyright (C) 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.cloud.teleport.v2.dto;

import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.common.collect.Lists;

/** BigQuery schemas for the GCS to Spanner Data Validation pipeline. */
public final class BigQuerySchemas {

private BigQuerySchemas() {}

public static final TableSchema MISMATCHED_RECORDS_SCHEMA =
new TableSchema()
.setFields(
Lists.newArrayList(
new TableFieldSchema()
.setName(MismatchedRecord.RUN_ID_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(MismatchedRecord.SCHEMA_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(MismatchedRecord.TABLE_NAME_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(MismatchedRecord.MISMATCH_TYPE_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(MismatchedRecord.RECORD_KEY_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(MismatchedRecord.SOURCE_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(MismatchedRecord.HASH_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED")));

public static final TableSchema TABLE_VALIDATION_STATS_SCHEMA =
new TableSchema()
.setFields(
Lists.newArrayList(
new TableFieldSchema()
.setName(TableValidationStats.RUN_ID_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.SCHEMA_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.TABLE_NAME_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.STATUS_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.SOURCE_ROW_COUNT_COLUMN_NAME)
.setType("INTEGER")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.DESTINATION_ROW_COUNT_COLUMN_NAME)
.setType("INTEGER")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.MATCHED_ROW_COUNT_COLUMN_NAME)
.setType("INTEGER")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.MISMATCH_ROW_COUNT_COLUMN_NAME)
.setType("INTEGER")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.START_TIMESTAMP_COLUMN_NAME)
.setType("TIMESTAMP")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(TableValidationStats.END_TIMESTAMP_COLUMN_NAME)
.setType("TIMESTAMP")
.setMode("REQUIRED")));

public static final TableSchema VALIDATION_SUMMARY_SCHEMA =
new TableSchema()
.setFields(
Lists.newArrayList(
new TableFieldSchema()
.setName(ValidationSummary.RUN_ID_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(ValidationSummary.SOURCE_DATABASE_COLUMN_NAME)
.setType("STRING")
.setMode("NULLABLE"),
new TableFieldSchema()
.setName(ValidationSummary.DESTINATION_DATABASE_COLUMN_NAME)
.setType("STRING")
.setMode("NULLABLE"),
new TableFieldSchema()
.setName(ValidationSummary.STATUS_COLUMN_NAME)
.setType("STRING")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(ValidationSummary.TOTAL_TABLES_VALIDATED_COLUMN_NAME)
.setType("INTEGER")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(ValidationSummary.TABLES_WITH_MISMATCHES_COLUMN_NAME)
.setType("STRING")
.setMode("NULLABLE"),
new TableFieldSchema()
.setName(ValidationSummary.TOTAL_ROWS_MATCHED_COLUMN_NAME)
.setType("INTEGER")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(ValidationSummary.TOTAL_ROWS_MISMATCHED_COLUMN_NAME)
.setType("INTEGER")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(ValidationSummary.START_TIMESTAMP_COLUMN_NAME)
.setType("TIMESTAMP")
.setMode("REQUIRED"),
new TableFieldSchema()
.setName(ValidationSummary.END_TIMESTAMP_COLUMN_NAME)
.setType("TIMESTAMP")
.setMode("REQUIRED")));
}
Loading
Loading