Skip to content

Commit 33add88

Browse files
Adding Spark 3.5 with Scala 2.13 module (#47492)
* Scala 2.13 skeleton * Update sdk/cosmos/ci.yml Co-authored-by: Copilot <[email protected]> * Update sdk/cosmos/azure-cosmos-spark_3/pom.xml Co-authored-by: Copilot <[email protected]> * Update sdk/cosmos/azure-cosmos-spark_3-5_2-13/CONTRIBUTING.md Co-authored-by: Copilot <[email protected]> * Update sdk/cosmos/azure-cosmos-spark_3-5_2-13/CONTRIBUTING.md Co-authored-by: Copilot <[email protected]> * Update pom.xml * md fixes * Fixing wrong versions * Update pom.xml * Fix versioning powershell script to exclude checks for versions specified via variables * Removing unused external dependencies * Update ci.yml * Update pom.xml * Fixing scala 2.13 incompatibilities * Update pom.xml * Fixing CI tests or spark 3.5 to use Java 17 * Update CHANGELOG.md * Fixing illegal access error when using Java 17 * Making --add-opens conditional on Java version * Update spark.yml * Fixing Spark live tests * Update databricks-jar-install.sh * Update TransientIOErrorsRetryingReadManyIterator.scala * Update CosmosItemIdentityHelper.scala * Update CosmosItemIdentityHelper.scala * Fixing scala 2.13 jsonNode serialization incompatibility * Update CosmosItemIdentityHelper.scala * Update CosmosItemIdentityHelper.scala * Fixing scala 2.13 regression * Fixing Scala 2.13 test failure * Fixing build break * Update SparkE2EChangeFeedITest.scala --------- Co-authored-by: Copilot <[email protected]>
1 parent 5508f56 commit 33add88

40 files changed

+910
-183
lines changed

.vscode/cspell.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
"sdk/cosmos/azure-cosmos-spark_3-4_2-12/**",
106106
"sdk/cosmos/azure-cosmos-spark_3-5/**",
107107
"sdk/cosmos/azure-cosmos-spark_3-5_2-12/**",
108+
"sdk/cosmos/azure-cosmos-spark_3-5_2-13/**",
108109
"sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/**",
109110
"sdk/cosmos/fabric-cosmos-spark-auth_3/**",
110111
"sdk/cosmos/azure-cosmos-encryption/**",

eng/.docsettings.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ known_content_issues:
7878
- ['sdk/cosmos/azure-cosmos-spark_3-3_2-12/README.md', '#3113']
7979
- ['sdk/cosmos/azure-cosmos-spark_3-4_2-12/README.md', '#3113']
8080
- ['sdk/cosmos/azure-cosmos-spark_3-5_2-12/README.md', '#3113']
81+
- ['sdk/cosmos/azure-cosmos-spark_3-5_2-13/README.md', '#3113']
8182
- ['sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/README.md', '#3113']
8283
- ['sdk/cosmos/fabric-cosmos-spark-auth_3/README.md', '#3113']
8384
- ['sdk/cosmos/azure-cosmos-spark_3_2-12/dev/README.md', '#3113']

eng/pipelines/templates/stages/cosmos-emulator-matrix.json

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,18 +71,25 @@
7171
"JavaTestVersion": "1.11",
7272
"AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true"
7373
},
74-
"Spark 3.5 Integration Tests targeting Cosmos Emulator - Java 8'": {
75-
"ProfileFlag": "-Dspark-e2e_3-5",
74+
"Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 8'": {
75+
"ProfileFlag": "-Dspark-e2e_3-5_2-12",
7676
"PROTOCOLS": "[\"Tcp\"]",
7777
"DESIRED_CONSISTENCIES": "[\"Session\"]",
7878
"JavaTestVersion": "1.8",
7979
"AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true"
8080
},
81-
"Spark 3.5 Integration Tests targeting Cosmos Emulator - Java 11'": {
82-
"ProfileFlag": "-Dspark-e2e_3-5",
81+
"Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 17'": {
82+
"ProfileFlag": "-Dspark-e2e_3-5_2-12",
8383
"PROTOCOLS": "[\"Tcp\"]",
8484
"DESIRED_CONSISTENCIES": "[\"Session\"]",
85-
"JavaTestVersion": "1.11",
85+
"JavaTestVersion": "1.17",
86+
"AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true"
87+
},
88+
"Spark 3.5, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": {
89+
"ProfileFlag": "-Dspark-e2e_3-5_2-13",
90+
"PROTOCOLS": "[\"Tcp\"]",
91+
"DESIRED_CONSISTENCIES": "[\"Session\"]",
92+
"JavaTestVersion": "1.17",
8693
"AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true"
8794
},
8895
"Kafka Integration Tests targeting Cosmos Emulator - Java 11": {

eng/versioning/external_dependencies.txt

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,8 +256,11 @@ cosmos-spark_3-3_org.apache.spark:spark-hive_2.12;3.3.0
256256
cosmos-spark_3-4_org.apache.spark:spark-hive_2.12;3.4.0
257257
cosmos-spark_3-5_org.apache.spark:spark-hive_2.12;3.5.0
258258
cosmos_org.scala-lang:scala-library;2.12.19
259-
cosmos_org.scala-lang.modules:scala-java8-compat_2.12;0.8.0
259+
cosmos-scala213_org.scala-lang:scala-library;2.13.17
260+
cosmos_org.scala-lang.modules:scala-java8-compat_2.12;0.9.1
261+
cosmos-scala213_org.scala-lang.modules:scala-java8-compat_2.13;0.9.1
260262
cosmos_io.projectreactor:reactor-scala-extensions_2.12;0.8.0
263+
cosmos-scala213_io.projectreactor:reactor-scala-extensions_2.13;0.8.0
261264
cosmos_commons-io:commons-io;2.4
262265
cosmos_com.microsoft.azure:applicationinsights-core;2.6.4
263266
cosmos_io.micrometer:micrometer-core;1.15.1
@@ -266,9 +269,13 @@ cosmos_io.micrometer:micrometer-registry-graphite;1.15.1
266269

267270
# Cosmos Spark connector tests only
268271
cosmos_org.scalatest:scalatest_2.12;3.2.2
272+
cosmos-scala213_org.scalatest:scalatest_2.13;3.2.2
269273
cosmos_org.scalatest:scalatest-flatspec_2.12;3.2.3
274+
cosmos-scala213_org.scalatest:scalatest-flatspec_2.13;3.2.3
270275
cosmos_org.scalactic:scalactic_2.12;3.2.3
276+
cosmos-scala213_org.scalactic:scalactic_2.13;3.2.3
271277
cosmos_org.scalamock:scalamock_2.12;5.0.0
278+
cosmos-scala213_org.scalamock:scalamock_2.13;5.0.0
272279
cosmos_com.globalmentor:hadoop-bare-naked-local-fs;0.1.0
273280
cosmos_org.mockito:mockito-core;4.8.1
274281

eng/versioning/pom_file_version_scanner.ps1

Lines changed: 93 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -723,31 +723,46 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object {
723723
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: dependency is missing version element for groupId=$($groupId), artifactId=$($artifactId) should be <version></version> <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
724724
continue
725725
}
726-
if ($versionNode.NextSibling -and $versionNode.NextSibling.NodeType -eq "Comment")
726+
727+
if ($versionNode.FirstChild.Value.StartsWith('${'))
727728
{
728-
# unfortunately because there are POM exceptions we need to wildcard the group which may be
729-
# something like <area>_groupId
730-
if ($versionNode.NextSibling.Value.Trim() -notmatch "{x-version-update;(.+)?$($groupId):$($artifactId);\w+}")
729+
# skip version checks when they have been intentionally applied via variables
730+
}
731+
else
732+
{
733+
if ($versionNode.NextSibling -and $versionNode.NextSibling.NodeType -eq "Comment")
731734
{
732-
$hasError = $true
733-
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: dependency version update tag for groupId=$($groupId), artifactId=$($artifactId) should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
735+
# unfortunately because there are POM exceptions we need to wildcard the group which may be
736+
# something like <area>_groupId
737+
if ($versionNode.FirstChild.Value.StartsWith('${'))
738+
{
739+
# skip version checks when they have been intentionally applied via variables
740+
}
741+
else
742+
{
743+
if ($versionNode.NextSibling.Value.Trim() -notmatch "{x-version-update;(.+)?$($groupId):$($artifactId);\w+}")
744+
{
745+
$hasError = $true
746+
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: dependency version update tag for groupId=$($groupId), artifactId=$($artifactId) should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
747+
}
748+
else
749+
{
750+
# verify the version tag and version are correct
751+
$retVal = Test-Dependency-Tag-And-Version $libHash $extDepHash $versionNode.InnerText.Trim() $versionNode.NextSibling.Value $artifactsPerSDHashSet
752+
if ($retVal)
753+
{
754+
$hasError = $true
755+
$potentialLogMessage = Join-With-NewLine $potentialLogMessage $retVal
756+
}
757+
}
758+
}
734759
}
735760
else
736761
{
737-
# verify the version tag and version are correct
738-
$retVal = Test-Dependency-Tag-And-Version $libHash $extDepHash $versionNode.InnerText.Trim() $versionNode.NextSibling.Value $artifactsPerSDHashSet
739-
if ($retVal)
740-
{
741-
$hasError = $true
742-
$potentialLogMessage = Join-With-NewLine $potentialLogMessage $retVal
743-
}
762+
$hasError = $true
763+
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: Missing dependency version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
744764
}
745-
}
746-
else
747-
{
748-
$hasError = $true
749-
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: Missing dependency version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
750-
}
765+
}
751766
}
752767
# Verify every plugin has a group, artifact and version
753768
# Verify every dependency has a group, artifact and version
@@ -882,80 +897,87 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object {
882897
$groupId = $split[0]
883898
$artifactId = $split[1]
884899
$version = $split[2]
885-
# The groupId match has to be able to deal with <area>_ for external dependency exceptions
886-
if (!$includeNode.NextSibling -or $includeNode.NextSibling.NodeType -ne "Comment")
887-
{
888-
$hasError = $true
889-
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: <include> is missing the update tag which should be <!-- {x-include-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
890-
}
891-
elseif ($includeNode.NextSibling.Value.Trim() -notmatch "{x-include-update;(.+)?$($groupId):$($artifactId);(current|dependency|external_dependency)}")
892-
{
893-
$hasError = $true
894-
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: <include> version update tag for $($includeNode.InnerText) should be <!-- {x-include-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
900+
901+
if ($version.StartsWith('[${')) {
902+
# skip version checks when they have been intentionally applied via variables
895903
}
896904
else
897905
{
898-
# verify that the version is formatted correctly
899-
if (!$version.StartsWith("[") -or !$version.EndsWith("]"))
906+
# The groupId match has to be able to deal with <area>_ for external dependency exceptions
907+
if (!$includeNode.NextSibling -or $includeNode.NextSibling.NodeType -ne "Comment")
900908
{
901909
$hasError = $true
902-
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: the version entry '$($version)' for <include> '$($rawIncludeText)' is not formatted correctly. The include version needs to of the form '[<version>]', the braces lock the include to a specific version for these entries. -->"
910+
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: <include> is missing the update tag which should be <!-- {x-include-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
911+
}
912+
elseif ($includeNode.NextSibling.Value.Trim() -notmatch "{x-include-update;(.+)?$($groupId):$($artifactId);(current|dependency|external_dependency)}")
913+
{
914+
$hasError = $true
915+
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: <include> version update tag for $($includeNode.InnerText) should be <!-- {x-include-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
903916
}
904-
# verify the version has the correct value
905917
else
906918
{
907-
$versionWithoutBraces = $version.Substring(1, $version.Length -2)
908-
# the key into the dependency has needs to be created from the tag's group/artifact
909-
# entries in case it's an external dependency entry. Because this has already
910-
# been validated for format, grab the group:artifact
911-
$depKey = $includeNode.NextSibling.Value.Trim().Split(";")[1]
912-
$depType = $includeNode.NextSibling.Value.Trim().Split(";")[2]
913-
$depType = $depType.Substring(0, $depType.IndexOf("}"))
914-
if ($depType -eq $DependencyTypeExternal)
919+
# verify that the version is formatted correctly
920+
if (!$version.StartsWith("[") -or !$version.EndsWith("]"))
915921
{
916-
if ($extDepHash.ContainsKey($depKey))
917-
{
918-
if ($versionWithoutBraces -ne $extDepHash[$depKey].ver)
919-
{
920-
$hasError = $true
921-
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: $($depKey)'s version is '$($versionWithoutBraces)' but the external_dependency version is listed as $($extDepHash[$depKey].ver)"
922-
}
923-
}
924-
else
925-
{
926-
$hasError = $true
927-
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: the groupId:artifactId entry '$($depKey)' for <include> '$($rawIncludeText)' is not a valid external dependency. Please verify the entry exists in the external_dependencies.txt file. -->"
928-
}
922+
$hasError = $true
923+
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: the version entry '$($version)' for <include> '$($rawIncludeText)' is not formatted correctly. The include version needs to of the form '[<version>]', the braces lock the include to a specific version for these entries. -->"
929924
}
925+
# verify the version has the correct value
930926
else
931927
{
932-
# If the tag isn't external_dependency then verify it exists in the library hash
933-
if (!$libHash.ContainsKey($depKey))
934-
{
935-
$hasError = $true
936-
return "Error: $($depKey)'s dependency type is '$($depType)' but the dependency does not exist in any of the version_*.txt files. Should this be an external_dependency? Please ensure the dependency type is correct or the dependency is added to the appropriate file."
937-
938-
}
939-
if ($depType -eq $DependencyTypeDependency)
928+
$versionWithoutBraces = $version.Substring(1, $version.Length -2)
929+
# the key into the dependency has needs to be created from the tag's group/artifact
930+
# entries in case it's an external dependency entry. Because this has already
931+
# been validated for format, grab the group:artifact
932+
$depKey = $includeNode.NextSibling.Value.Trim().Split(";")[1]
933+
$depType = $includeNode.NextSibling.Value.Trim().Split(";")[2]
934+
$depType = $depType.Substring(0, $depType.IndexOf("}"))
935+
if ($depType -eq $DependencyTypeExternal)
940936
{
941-
if ($versionWithoutBraces -ne $libHash[$depKey].depVer)
937+
if ($extDepHash.ContainsKey($depKey))
938+
{
939+
if ($versionWithoutBraces -ne $extDepHash[$depKey].ver)
940+
{
941+
$hasError = $true
942+
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: $($depKey)'s version is '$($versionWithoutBraces)' but the external_dependency version is listed as $($extDepHash[$depKey].ver)"
943+
}
944+
}
945+
else
942946
{
943947
$hasError = $true
944-
return "Error: $($depKey)'s <version> is '$($versionString)' but the dependency version is listed as $($libHash[$depKey].depVer)"
948+
$potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: the groupId:artifactId entry '$($depKey)' for <include> '$($rawIncludeText)' is not a valid external dependency. Please verify the entry exists in the external_dependencies.txt file. -->"
945949
}
946950
}
947-
elseif ($depType -eq $DependencyTypeCurrent)
951+
else
948952
{
949-
# Verify that none of the 'current' dependencies are using a groupId that starts with 'unreleased_' or 'beta_'
950-
if ($depKey.StartsWith('unreleased_') -or $depKey.StartsWith('beta_'))
953+
# If the tag isn't external_dependency then verify it exists in the library hash
954+
if (!$libHash.ContainsKey($depKey))
951955
{
952956
$hasError = $true
953-
return "Error: $($versionUpdateString) is using an unreleased_ or beta_ dependency and trying to set current value. Only dependency versions can be set with an unreleased or beta dependency."
957+
return "Error: $($depKey)'s dependency type is '$($depType)' but the dependency does not exist in any of the version_*.txt files. Should this be an external_dependency? Please ensure the dependency type is correct or the dependency is added to the appropriate file."
958+
954959
}
955-
if ($versionWithoutBraces -ne $libHash[$depKey].curVer)
960+
if ($depType -eq $DependencyTypeDependency)
956961
{
957-
$hasError = $true
958-
return "Error: $($depKey)'s <version> is '$($versionString)' but the current version is listed as $($libHash[$depKey].curVer)"
962+
if ($versionWithoutBraces -ne $libHash[$depKey].depVer)
963+
{
964+
$hasError = $true
965+
return "Error: $($depKey)'s <version> is '$($versionString)' but the dependency version is listed as $($libHash[$depKey].depVer)"
966+
}
967+
}
968+
elseif ($depType -eq $DependencyTypeCurrent)
969+
{
970+
# Verify that none of the 'current' dependencies are using a groupId that starts with 'unreleased_' or 'beta_'
971+
if ($depKey.StartsWith('unreleased_') -or $depKey.StartsWith('beta_'))
972+
{
973+
$hasError = $true
974+
return "Error: $($versionUpdateString) is using an unreleased_ or beta_ dependency and trying to set current value. Only dependency versions can be set with an unreleased or beta dependency."
975+
}
976+
if ($versionWithoutBraces -ne $libHash[$depKey].curVer)
977+
{
978+
$hasError = $true
979+
return "Error: $($depKey)'s <version> is '$($versionString)' but the current version is listed as $($libHash[$depKey].curVer)"
980+
}
959981
}
960982
}
961983
}

0 commit comments

Comments
 (0)