diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml index 4a486890cc..c02e681d36 100644 --- a/.github/workflows/spanner-pr.yml +++ b/.github/workflows/spanner-pr.yml @@ -54,34 +54,6 @@ env: permissions: write-all jobs: - spotless_check: - name: Spotless - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Spotless - run: | - ./cicd/run-spotless \ - --modules-to-build="SPANNER" - checkstyle_check: - name: Checkstyle - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Checkstyle - run: | - ./cicd/run-checkstyle \ - --modules-to-build="SPANNER" java_build: name: Build timeout-minutes: 60 @@ -98,74 +70,9 @@ jobs: --modules-to-build="SPANNER" - name: Cleanup Java Environment uses: ./.github/actions/cleanup-java-env - java_unit_tests: - name: Unit Tests - needs: [java_build] - timeout-minutes: 60 - runs-on: [self-hosted, it] - steps: - - name: Checkout Code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Unit Tests - run: | - ./cicd/run-unit-tests \ - --modules-to-build="SPANNER" - - name: Upload Unit Tests Report - uses: actions/upload-artifact@v4 - if: always() # always run even if the previous step fails - with: - name: surefire-unit-test-results - path: | - **/surefire-reports/TEST-*.xml - **/surefire-reports/*.html - **/surefire-reports/html/** - retention-days: 1 - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v5.5.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: GoogleCloudPlatform/DataflowTemplates - files: 'target/site/jacoco-aggregate/jacoco.xml' - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - java_integration_smoke_tests_templates: - name: Dataflow Templates Integration Smoke Tests - needs: [spotless_check, checkstyle_check, java_build, java_unit_tests] - timeout-minutes: 60 - # Run on any runner that matches all the specified runs-on values. - runs-on: [self-hosted, it] - steps: - - name: Checkout Code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Integration Smoke Tests - run: | - ./cicd/run-it-smoke-tests \ - --modules-to-build="SPANNER" \ - --it-region="us-central1" \ - --it-project="cloud-teleport-testing" \ - --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ - --it-private-connectivity="datastream-connect-2" - - name: Upload Smoke Tests Report - uses: actions/upload-artifact@v4 - if: always() # always run even if the previous step fails - with: - name: surefire-smoke-test-results - path: | - **/surefire-reports/TEST-*.xml - **/surefire-reports/*.html - **/surefire-reports/html/** - retention-days: 1 - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env java_integration_tests_templates: name: Dataflow Templates Integration Tests - needs: [java_integration_smoke_tests_templates] + needs: [java_build] timeout-minutes: 180 # Run on any runner that matches all the specified runs-on values. runs-on: [self-hosted, it] @@ -178,7 +85,7 @@ jobs: - name: Run Integration Tests run: | ./cicd/run-it-tests \ - --modules-to-build="SPANNER" \ + --it-module="spanner-to-sourcedb" \ --it-region="us-central1" \ --it-project="cloud-teleport-testing" \ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ @@ -190,7 +97,7 @@ jobs: name: surefire-integration-test-results path: | **/surefire-reports/TEST-*.xml - **/surefire-reports/*.html + **/surere-reports/*.html **/surefire-reports/html/** retention-days: 10 - name: Integration Test report on GitHub @@ -210,7 +117,7 @@ jobs: java_load_tests_templates: if: contains(github.event.pull_request.labels.*.name, 'run-load-tests') name: Dataflow Templates Load Tests - needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates] + needs: [java_build, java_integration_tests_templates] timeout-minutes: 600 # Run on any runner that matches all the specified runs-on values. runs-on: [self-hosted, perf] @@ -229,4 +136,4 @@ jobs: --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ --it-private-connectivity="datastream-connect-2" - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env + uses: ./.github/actions/cleanup-java-env \ No newline at end of file diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index 286ca47dce..022ee426c4 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -27,8 +27,16 @@ import ( func main() { flags.RegisterCommonFlags() flags.RegisterItFlags() + + var itModule string + flag.StringVar(&itModule, "it-module", "", "(optional) The single module to run integration tests for") + flag.Parse() + if itModule != "" { + flags.SetModulesToBuild(itModule) + } + // Run mvn install before running integration tests mvnFlags := workflows.NewMavenFlags() err := workflows.MvnCleanInstall().Run( diff --git a/cicd/cmd/run-it-tests/run-it-tests b/cicd/cmd/run-it-tests/run-it-tests new file mode 100755 index 0000000000..250ddd7602 Binary files /dev/null and b/cicd/cmd/run-it-tests/run-it-tests differ diff --git a/cicd/internal/flags/common-flags.go b/cicd/internal/flags/common-flags.go index b901341ee1..c10c326a17 100644 --- a/cicd/internal/flags/common-flags.go +++ b/cicd/internal/flags/common-flags.go @@ -66,6 +66,12 @@ var ( "v2/datastream-to-sql/", }, } + + // New map for short name to full path mapping + shortNameToFullPathMap = map[string]string{ + "spanner-to-sourcedb": "v2/spanner-to-sourcedb/", + // Add other mappings as needed + } ) // Registers all common flags. Must be called before flag.Parse(). @@ -73,9 +79,20 @@ func RegisterCommonFlags() { flag.StringVar(&modulesToBuild, "modules-to-build", ALL, "List of modules to build/run commands against") } +// SetModulesToBuild allows setting the modules to build programmatically. +func SetModulesToBuild(module string) { + modulesToBuild = module +} + // Returns all modules to build. func ModulesToBuild() []string { m := modulesToBuild + + // Check if the module is a short name + if fullPath, ok := shortNameToFullPathMap[m]; ok { + return []string{fullPath} + } + if m == "DEFAULT" { // "DEFAULT" is "ALL" minus other modules defined in moduleMap var s []string diff --git a/cicd/run-it-tests b/cicd/run-it-tests new file mode 100755 index 0000000000..0e8fe8c79b Binary files /dev/null and b/cicd/run-it-tests differ diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbIT.java index 4b881cb357..8289d07859 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbIT.java @@ -46,7 +46,6 @@ import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; import org.junit.AfterClass; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -158,7 +157,7 @@ public static void cleanUp() throws IOException { } @Test - @Ignore("Skipping spannerToSourceDbBasic test") + // @Ignore("Skipping spannerToSourceDbBasic test") public void spannerToSourceDbBasic() throws InterruptedException, IOException { assertThatPipeline(jobInfo).isRunning(); // Write row in Spanner @@ -217,12 +216,20 @@ private void assertRowInMySQL() throws InterruptedException { pipelineOperator() .waitForCondition( createConfig(jobInfo, TEST_TIMEOUT), - () -> jdbcResourceManager.getRowCount(TABLE) == 1); // only one row is inserted + () -> { + long rowCount = jdbcResourceManager.getRowCount(TABLE); + LOG.info("Current row count for table {}: {}", TABLE, rowCount); + return rowCount == 1; + }); // only one row is inserted assertThatResult(result).meetsConditions(); List> rows = jdbcResourceManager.readTable(TABLE); + LOG.info("Retrieved rows from MySQL for table {}: {}", TABLE, rows); assertThat(rows).hasSize(1); + LOG.info("id is : {}", rows.get(0).get("id")); assertThat(rows.get(0).get("id")).isEqualTo(1); + LOG.info("name is : {}", rows.get(0).get("name")); assertThat(rows.get(0).get("name")).isEqualTo("FF"); + LOG.info("from is : {}", rows.get(0).get("from")); assertThat(rows.get(0).get("from")).isEqualTo("AA"); }