diff --git a/it/cassandra/pom.xml b/it/cassandra/pom.xml index 6e45886c3c..e3493d23bf 100644 --- a/it/cassandra/pom.xml +++ b/it/cassandra/pom.xml @@ -56,9 +56,9 @@ - com.datastax.oss + org.apache.cassandra java-driver-core - 4.17.0 + ${cassandra-java-driver-core.version} org.slf4j diff --git a/it/pom.xml b/it/pom.xml index e454933242..3e7698fc9f 100644 --- a/it/pom.xml +++ b/it/pom.xml @@ -32,6 +32,7 @@ 3.6.1 1.21.3 v1b3-rev20250519-2.0.0 + 4.18.1 diff --git a/v2/pom.xml b/v2/pom.xml index 6c7b8d90e2..6897c48e8d 100644 --- a/v2/pom.xml +++ b/v2/pom.xml @@ -45,6 +45,7 @@ com.google.cloud.teleport.v2.spanner.IntegrationTest ../JAVA_LICENSE_HEADER + 4.18.1 @@ -165,7 +166,7 @@ com.google.protobuf protobuf-java - 4.33.2 + 4.33.1 @@ -284,12 +285,12 @@ copy-dependencies package - copy-dependencies + copy-dependencies - ${project.build.directory}/extra_libs - conscrypt-openjdk-uber - true + ${project.build.directory}/extra_libs + conscrypt-openjdk-uber + true @@ -535,7 +536,7 @@ + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> diff --git a/v2/sourcedb-to-spanner/pom.xml b/v2/sourcedb-to-spanner/pom.xml index 435d281342..eb9bda6eb7 100644 --- a/v2/sourcedb-to-spanner/pom.xml +++ b/v2/sourcedb-to-spanner/pom.xml @@ -16,278 +16,274 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~--> - 4.0.0 + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + 4.0.0 - - com.google.cloud.teleport.v2 - dynamic-templates - 1.0-SNAPSHOT - + + com.google.cloud.teleport.v2 + dynamic-templates + 1.0-SNAPSHOT + - sourcedb-to-spanner - - 4.18.1 - - + sourcedb-to-spanner - - - com.google.cloud.teleport - structured-logging - ${project.version} - - - com.google.cloud.teleport.v2 - common - ${project.version} - - - com.google.cloud.teleport.v2 - jdbc-common - ${project.version} - - - com.google.guava - guava - ${guava.version} - - - com.google.cloud - google-cloud-core - - - org.syncope.identityconnectors - framework - 0.4.3 - - - org.syncope.identityconnectors - framework-internal - 0.4.3 - runtime - - - mysql - mysql-connector-java - ${mysql-connector-java.version} - - - org.postgresql - postgresql - ${postgresql.version} - - - - org.apache.cassandra - java-driver-core - ${cassandra-java-driver-core.version} - + + + com.google.cloud.teleport + structured-logging + ${project.version} + + + com.google.cloud.teleport.v2 + common + ${project.version} + + + com.google.cloud.teleport.v2 + jdbc-common + ${project.version} + + + com.google.guava + guava + ${guava.version} + + + com.google.cloud + google-cloud-core + + + org.syncope.identityconnectors + framework + 0.4.3 + + + org.syncope.identityconnectors + framework-internal + 0.4.3 + runtime + + + mysql + mysql-connector-java + ${mysql-connector-java.version} + + + org.postgresql + postgresql + ${postgresql.version} + + + + org.apache.cassandra + java-driver-core + ${cassandra-java-driver-core.version} + - - - - com.codahale.metrics - metrics-core - 3.0.2 - - - org.slf4j - slf4j-api - - - + + + + com.codahale.metrics + metrics-core + 3.0.2 + + + org.slf4j + slf4j-api + + + - - - com.google.cloud.teleport - it-google-cloud-platform - ${project.version} - test - - - org.apache.beam - beam-it-jdbc - test - - - org.apache.beam - beam-it-cassandra - test - - - com.datastax.oss - java-driver-core - - - + + + com.google.cloud.teleport + it-google-cloud-platform + ${project.version} + test + + + org.apache.beam + beam-it-jdbc + test + + + org.apache.beam + beam-it-cassandra + test + + + com.datastax.oss + java-driver-core + + + - - - org.apache.derby - derby - 10.14.2.0 - test - - - org.apache.derby - derbyclient - 10.14.2.0 - test - - - org.apache.derby - derbynet - 10.14.2.0 - test - - - - com.google.truth - truth - 1.4.5 - test - + + + org.apache.derby + derby + 10.14.2.0 + test + + + org.apache.derby + derbyclient + 10.14.2.0 + test + + + org.apache.derby + derbynet + 10.14.2.0 + test + + + + com.google.truth + truth + 1.4.5 + test + - - com.google.cloud.teleport.v2 - spanner-common - 1.0-SNAPSHOT - compile - - - com.google.cloud.teleport - it-jdbc - 1.0-SNAPSHOT - test - - - com.google.cloud.teleport.v2 - datastream-to-spanner - 1.0-SNAPSHOT - compile - - - com.github.nosan - embedded-cassandra - 5.0.2 - test - - - org.apache.beam - beam-sdks-java-io-cassandra - - - com.microsoft.sqlserver - mssql-jdbc - ${mssql-jdbc.version} - test - - - - - org.apache.beam - beam-sdks-java-io-common - 2.64.0 - test - - - - - com.codahale.metrics - metrics-core - 3.0.2 - - - org.slf4j - slf4j-api - - - - - - com.github.stefanbirkner - system-rules - 1.19.0 - test - - - com.datastax.astra - beam-sdks-java-io-astra - 4.18.1 - - - - com.datastax.astra - astra-sdk-devops - 0.6.3 - + + com.google.cloud.teleport.v2 + spanner-common + 1.0-SNAPSHOT + compile + + + com.google.cloud.teleport + it-jdbc + 1.0-SNAPSHOT + test + + + com.google.cloud.teleport.v2 + datastream-to-spanner + 1.0-SNAPSHOT + compile + + + com.github.nosan + embedded-cassandra + 5.0.2 + test + + + org.apache.beam + beam-sdks-java-io-cassandra + + + com.microsoft.sqlserver + mssql-jdbc + ${mssql-jdbc.version} + test + + + + + org.apache.beam + beam-sdks-java-io-common + 2.64.0 + test + + + + + com.codahale.metrics + metrics-core + 3.0.2 + + + org.slf4j + slf4j-api + + + + + + com.github.stefanbirkner + system-rules + 1.19.0 + test + + + com.datastax.astra + beam-sdks-java-io-astra + 4.18.1 + + + + com.datastax.astra + astra-sdk-devops + 0.6.3 + - + - - org.apache.commons - commons-collections4 - 4.1 - compile - - - org.bouncycastle - bcpkix-jdk15on - 1.70 - test - - - com.google.cloud.teleport - it-conditions - test - ${project.version} - - - - - - org.apache.cassandra - java-driver-core - ${cassandra-java-driver-core.version} - + + org.apache.commons + commons-collections4 + 4.1 + compile + + + org.bouncycastle + bcpkix-jdk15on + 1.70 + test + + + com.google.cloud.teleport + it-conditions + test + ${project.version} + - + + + + org.apache.cassandra + java-driver-core + ${cassandra-java-driver-core.version} + + + - - - useRealSpanner - - true - - !activateFailureInjection - - - - - com.google.cloud.teleport.v2 - real-spanner-service - ${project.version} - - - - - failureInjectionTest - - - activateFailureInjection - true - - - - - com.google.cloud.teleport.v2 - failure-injected-spanner-service - ${project.version} - - - - + + + useRealSpanner + + true + + !activateFailureInjection + + + + + com.google.cloud.teleport.v2 + real-spanner-service + ${project.version} + + + + + failureInjectionTest + + + activateFailureInjection + true + + + + + com.google.cloud.teleport.v2 + failure-injected-spanner-service + ${project.version} + + + + diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/CassandraAllDataTypesIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/CassandraAllDataTypesIT.java index a0df2b2657..c63a723e81 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/CassandraAllDataTypesIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/CassandraAllDataTypesIT.java @@ -54,7 +54,6 @@ import org.jline.utils.Log; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -70,8 +69,6 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(SourceDbToSpanner.class) @RunWith(JUnit4.class) -@Ignore( - "This test is disabled currently because it is failing with Unknown Cassandra Option advanced.load-balancing-policy.dc-failover.preferred-remote-dcs") public class CassandraAllDataTypesIT extends SourceDbToSpannerITBase { private static final Logger LOG = LoggerFactory.getLogger(MySQLDataTypesIT.class); @@ -121,6 +118,7 @@ private FlexTemplateDataflowJobResourceManager launchDlqReplay(LaunchInfo bulkJo .addParameter("transformationJarPath", customTransformation.jarPath()) .addParameter("transformationClassName", customTransformation.classPath()) .addParameter("dlqRetryMinutes", DLQ_RETRY_MINUTES.toString()) + .addParameter("workerMachineType", "n2-standard-4") .addEnvironmentVariable( "additionalExperiments", List.of("use_runner_v2", "enable_data_sampling")) .build(); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java index 36a7b2ac1f..01982d093c 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java @@ -297,13 +297,13 @@ private Map getCassandraParameters( String configFile = String.format( """ - datastax-java-driver { - basic.contact-points = ["%s:%d"] - basic.session-keyspace = %s - basic.load-balancing-policy { - local-datacenter = datacenter1 - } - }""", + datastax-java-driver { + basic.contact-points = ["%s:%d"] + basic.session-keyspace = %s + basic.load-balancing-policy { + local-datacenter = datacenter1 + } + }""", cassandraResourceManager.getHost(), cassandraResourceManager.getPort(), cassandraResourceManager.getKeyspaceName()); diff --git a/v2/spanner-common/pom.xml b/v2/spanner-common/pom.xml index 54313b83f8..130fa5dd3d 100644 --- a/v2/spanner-common/pom.xml +++ b/v2/spanner-common/pom.xml @@ -46,18 +46,18 @@ 1.0-SNAPSHOT compile - - com.datastax.oss - java-driver-core - 4.17.0 - compile - - - org.slf4j - slf4j-api - - - + + org.apache.cassandra + java-driver-core + ${cassandra-java-driver-core.version} + compile + + + org.slf4j + slf4j-api + + + org.mockito mockito-inline diff --git a/v2/spanner-to-sourcedb/pom.xml b/v2/spanner-to-sourcedb/pom.xml index 0a598c39c8..cd2fcb5d89 100644 --- a/v2/spanner-to-sourcedb/pom.xml +++ b/v2/spanner-to-sourcedb/pom.xml @@ -69,7 +69,7 @@ 5.0.1 - + com.google.cloud.teleport it-google-cloud-platform @@ -100,9 +100,9 @@ test - com.datastax.oss + org.apache.cassandra java-driver-core - 4.17.0 + ${cassandra-java-driver-core.version} org.slf4j