Adding Spark 3.5 with Scala 2.13 module

.vscode/cspell.json

@@ -105,6 +105,7 @@
     "sdk/cosmos/azure-cosmos-spark_3-4_2-12/**",
     "sdk/cosmos/azure-cosmos-spark_3-5/**",
     "sdk/cosmos/azure-cosmos-spark_3-5_2-12/**",
+    "sdk/cosmos/azure-cosmos-spark_3-5_2-13/**",
     "sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/**",
     "sdk/cosmos/fabric-cosmos-spark-auth_3/**",
     "sdk/cosmos/azure-cosmos-encryption/**",

eng/.docsettings.yml

@@ -78,6 +78,7 @@ known_content_issues:
   - ['sdk/cosmos/azure-cosmos-spark_3-3_2-12/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3-4_2-12/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3-5_2-12/README.md', '#3113']
+  - ['sdk/cosmos/azure-cosmos-spark_3-5_2-13/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/README.md', '#3113']
   - ['sdk/cosmos/fabric-cosmos-spark-auth_3/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3_2-12/dev/README.md', '#3113']

eng/pipelines/templates/stages/cosmos-emulator-matrix.json

@@ -71,18 +71,25 @@
         "JavaTestVersion": "1.11",
         "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true"
       },
-      "Spark 3.5 Integration Tests targeting Cosmos Emulator - Java 8'": {
-        "ProfileFlag": "-Dspark-e2e_3-5",
+      "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 8'": {
+        "ProfileFlag": "-Dspark-e2e_3-5_2-12",
         "PROTOCOLS": "[\"Tcp\"]",
         "DESIRED_CONSISTENCIES": "[\"Session\"]",
         "JavaTestVersion": "1.8",
         "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true"
       },
-      "Spark 3.5 Integration Tests targeting Cosmos Emulator - Java 11'": {
-        "ProfileFlag": "-Dspark-e2e_3-5",
+      "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 17'": {
+        "ProfileFlag": "-Dspark-e2e_3-5_2-12",
         "PROTOCOLS": "[\"Tcp\"]",
         "DESIRED_CONSISTENCIES": "[\"Session\"]",
-        "JavaTestVersion": "1.11",
+        "JavaTestVersion": "1.17",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true"
+      },
+      "Spark 3.5, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": {
+        "ProfileFlag": "-Dspark-e2e_3-5_2-13",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.17",
         "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true"
       },
       "Kafka Integration Tests targeting Cosmos Emulator - Java 11": {

eng/versioning/external_dependencies.txt

@@ -256,8 +256,11 @@ cosmos-spark_3-3_org.apache.spark:spark-hive_2.12;3.3.0
 cosmos-spark_3-4_org.apache.spark:spark-hive_2.12;3.4.0
 cosmos-spark_3-5_org.apache.spark:spark-hive_2.12;3.5.0
 cosmos_org.scala-lang:scala-library;2.12.19
-cosmos_org.scala-lang.modules:scala-java8-compat_2.12;0.8.0
+cosmos-scala213_org.scala-lang:scala-library;2.13.17
+cosmos_org.scala-lang.modules:scala-java8-compat_2.12;0.9.1
+cosmos-scala213_org.scala-lang.modules:scala-java8-compat_2.13;0.9.1
 cosmos_io.projectreactor:reactor-scala-extensions_2.12;0.8.0
+cosmos-scala213_io.projectreactor:reactor-scala-extensions_2.13;0.8.0
 cosmos_commons-io:commons-io;2.4
 cosmos_com.microsoft.azure:applicationinsights-core;2.6.4
 cosmos_io.micrometer:micrometer-core;1.15.1
@@ -266,9 +269,13 @@ cosmos_io.micrometer:micrometer-registry-graphite;1.15.1
 
 # Cosmos Spark connector tests only
 cosmos_org.scalatest:scalatest_2.12;3.2.2
+cosmos-scala213_org.scalatest:scalatest_2.13;3.2.2
 cosmos_org.scalatest:scalatest-flatspec_2.12;3.2.3
+cosmos-scala213_org.scalatest:scalatest-flatspec_2.13;3.2.3
 cosmos_org.scalactic:scalactic_2.12;3.2.3
+cosmos-scala213_org.scalactic:scalactic_2.13;3.2.3
 cosmos_org.scalamock:scalamock_2.12;5.0.0
+cosmos-scala213_org.scalamock:scalamock_2.13;5.0.0
 cosmos_com.globalmentor:hadoop-bare-naked-local-fs;0.1.0
 cosmos_org.mockito:mockito-core;4.8.1
 

eng/versioning/pom_file_version_scanner.ps1

@@ -723,31 +723,46 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object {
             $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: dependency is missing version element for groupId=$($groupId), artifactId=$($artifactId) should be <version></version> <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
             continue
         }
-        if ($versionNode.NextSibling -and $versionNode.NextSibling.NodeType -eq "Comment")
+
+        if ($versionNode.FirstChild.Value.StartsWith('${'))
         {
-            # unfortunately because there are POM exceptions we need to wildcard the group which may be
-            # something like <area>_groupId
-            if ($versionNode.NextSibling.Value.Trim() -notmatch "{x-version-update;(.+)?$($groupId):$($artifactId);\w+}")
+            # skip version checks when they have been intentionally applied via variables
+        }
+        else 
+        {
+            if ($versionNode.NextSibling -and $versionNode.NextSibling.NodeType -eq "Comment")
             {
-                $hasError = $true
-                $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: dependency version update tag for groupId=$($groupId), artifactId=$($artifactId) should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
+                # unfortunately because there are POM exceptions we need to wildcard the group which may be
+                # something like <area>_groupId
+                if ($versionNode.FirstChild.Value.StartsWith('${'))
+                {
+                    # skip version checks when they have been intentionally applied via variables
+                }
+                else 
+                {
+                    if ($versionNode.NextSibling.Value.Trim() -notmatch "{x-version-update;(.+)?$($groupId):$($artifactId);\w+}")
+                    {
+                        $hasError = $true
+                        $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: dependency version update tag for groupId=$($groupId), artifactId=$($artifactId) should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
+                    }
+                    else
+                    {
+                        # verify the version tag and version are correct
+                        $retVal = Test-Dependency-Tag-And-Version $libHash $extDepHash $versionNode.InnerText.Trim() $versionNode.NextSibling.Value $artifactsPerSDHashSet
+                        if ($retVal)
+                        {
+                            $hasError = $true
+                            $potentialLogMessage = Join-With-NewLine $potentialLogMessage $retVal
+                        }
+                    }
+                }
             }
             else
             {
-                # verify the version tag and version are correct
-                $retVal = Test-Dependency-Tag-And-Version $libHash $extDepHash $versionNode.InnerText.Trim() $versionNode.NextSibling.Value $artifactsPerSDHashSet
-                if ($retVal)
-                {
-                    $hasError = $true
-                    $potentialLogMessage = Join-With-NewLine $potentialLogMessage $retVal
-                }
+                $hasError = $true
+                $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: Missing dependency version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
             }
-        }
-        else
-        {
-            $hasError = $true
-            $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: Missing dependency version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
-        }
+        }    
     }
     # Verify every plugin has a group, artifact and version
     # Verify every dependency has a group, artifact and version
@@ -882,80 +897,87 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object {
                 $groupId = $split[0]
                 $artifactId = $split[1]
                 $version = $split[2]
-                # The groupId match has to be able to deal with <area>_ for external dependency exceptions
-                if (!$includeNode.NextSibling -or $includeNode.NextSibling.NodeType -ne "Comment")
-                {
-                    $hasError = $true
-                    $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: <include> is missing the update tag which should be <!-- {x-include-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
-                }
-                elseif ($includeNode.NextSibling.Value.Trim() -notmatch "{x-include-update;(.+)?$($groupId):$($artifactId);(current|dependency|external_dependency)}")
-                {
-                    $hasError = $true
-                    $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: <include> version update tag for $($includeNode.InnerText) should be <!-- {x-include-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
+
+                if ($version.StartsWith('[${')) {
+                    # skip version checks when they have been intentionally applied via variables
                 }
                 else
                 {
-                    # verify that the version is formatted correctly
-                    if (!$version.StartsWith("[") -or !$version.EndsWith("]"))
+                    # The groupId match has to be able to deal with <area>_ for external dependency exceptions
+                    if (!$includeNode.NextSibling -or $includeNode.NextSibling.NodeType -ne "Comment")
                     {
                         $hasError = $true
-                        $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: the version entry '$($version)' for <include> '$($rawIncludeText)' is not formatted correctly. The include version needs to of the form '[<version>]', the braces lock the include to a specific version for these entries. -->"
+                        $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: <include> is missing the update tag which should be <!-- {x-include-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
+                    }
+                    elseif ($includeNode.NextSibling.Value.Trim() -notmatch "{x-include-update;(.+)?$($groupId):$($artifactId);(current|dependency|external_dependency)}")
+                    {
+                        $hasError = $true
+                        $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: <include> version update tag for $($includeNode.InnerText) should be <!-- {x-include-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
                     }
-                    # verify the version has the correct value
                     else
                     {
-                        $versionWithoutBraces = $version.Substring(1, $version.Length -2)
-                        # the key into the dependency has needs to be created from the tag's group/artifact
-                        # entries in case it's an external dependency entry. Because this has already
-                        # been validated for format, grab the group:artifact
-                        $depKey = $includeNode.NextSibling.Value.Trim().Split(";")[1]
-                        $depType = $includeNode.NextSibling.Value.Trim().Split(";")[2]
-                        $depType = $depType.Substring(0, $depType.IndexOf("}"))
-                        if ($depType -eq $DependencyTypeExternal)
+                        # verify that the version is formatted correctly
+                        if (!$version.StartsWith("[") -or !$version.EndsWith("]"))
                         {
-                            if ($extDepHash.ContainsKey($depKey))
-                            {
-                                if ($versionWithoutBraces -ne $extDepHash[$depKey].ver)
-                                {
-                                    $hasError = $true
-                                    $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: $($depKey)'s version is '$($versionWithoutBraces)' but the external_dependency version is listed as $($extDepHash[$depKey].ver)"
-                                }
-                            }
-                            else
-                            {
-                                $hasError = $true
-                                $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: the groupId:artifactId entry '$($depKey)' for <include> '$($rawIncludeText)' is not a valid external dependency. Please verify the entry exists in the external_dependencies.txt file. -->"
-                            }
+                            $hasError = $true
+                            $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: the version entry '$($version)' for <include> '$($rawIncludeText)' is not formatted correctly. The include version needs to of the form '[<version>]', the braces lock the include to a specific version for these entries. -->"
                         }
+                        # verify the version has the correct value
                         else
                         {
-                            # If the tag isn't external_dependency then verify it exists in the library hash
-                            if (!$libHash.ContainsKey($depKey))
-                            {
-                                $hasError = $true
-                                return "Error: $($depKey)'s dependency type is '$($depType)' but the dependency does not exist in any of the version_*.txt files. Should this be an external_dependency? Please ensure the dependency type is correct or the dependency is added to the appropriate file."
-
-                            }
-                            if ($depType -eq $DependencyTypeDependency)
+                            $versionWithoutBraces = $version.Substring(1, $version.Length -2)
+                            # the key into the dependency has needs to be created from the tag's group/artifact
+                            # entries in case it's an external dependency entry. Because this has already
+                            # been validated for format, grab the group:artifact
+                            $depKey = $includeNode.NextSibling.Value.Trim().Split(";")[1]
+                            $depType = $includeNode.NextSibling.Value.Trim().Split(";")[2]
+                            $depType = $depType.Substring(0, $depType.IndexOf("}"))
+                            if ($depType -eq $DependencyTypeExternal)
                             {
-                                if ($versionWithoutBraces -ne $libHash[$depKey].depVer)
+                                if ($extDepHash.ContainsKey($depKey))
+                                {
+                                    if ($versionWithoutBraces -ne $extDepHash[$depKey].ver)
+                                    {
+                                        $hasError = $true
+                                        $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: $($depKey)'s version is '$($versionWithoutBraces)' but the external_dependency version is listed as $($extDepHash[$depKey].ver)"
+                                    }
+                                }
+                                else
                                 {
                                     $hasError = $true
-                                    return "Error: $($depKey)'s <version> is '$($versionString)' but the dependency version is listed as $($libHash[$depKey].depVer)"
+                                    $potentialLogMessage = Join-With-NewLine $potentialLogMessage "Error: the groupId:artifactId entry '$($depKey)' for <include> '$($rawIncludeText)' is not a valid external dependency. Please verify the entry exists in the external_dependencies.txt file. -->"
                                 }
                             }
-                            elseif ($depType -eq $DependencyTypeCurrent)
+                            else
                             {
-                                # Verify that none of the 'current' dependencies are using a groupId that starts with 'unreleased_' or 'beta_'
-                                if ($depKey.StartsWith('unreleased_') -or $depKey.StartsWith('beta_'))
+                                # If the tag isn't external_dependency then verify it exists in the library hash
+                                if (!$libHash.ContainsKey($depKey))
                                 {
                                     $hasError = $true
-                                    return "Error: $($versionUpdateString) is using an unreleased_ or beta_ dependency and trying to set current value. Only dependency versions can be set with an unreleased or beta dependency."
+                                    return "Error: $($depKey)'s dependency type is '$($depType)' but the dependency does not exist in any of the version_*.txt files. Should this be an external_dependency? Please ensure the dependency type is correct or the dependency is added to the appropriate file."
+
                                 }
-                                if ($versionWithoutBraces -ne $libHash[$depKey].curVer)
+                                if ($depType -eq $DependencyTypeDependency)
                                 {
-                                    $hasError = $true
-                                    return "Error: $($depKey)'s <version> is '$($versionString)' but the current version is listed as $($libHash[$depKey].curVer)"
+                                    if ($versionWithoutBraces -ne $libHash[$depKey].depVer)
+                                    {
+                                        $hasError = $true
+                                        return "Error: $($depKey)'s <version> is '$($versionString)' but the dependency version is listed as $($libHash[$depKey].depVer)"
+                                    }
+                                }
+                                elseif ($depType -eq $DependencyTypeCurrent)
+                                {
+                                    # Verify that none of the 'current' dependencies are using a groupId that starts with 'unreleased_' or 'beta_'
+                                    if ($depKey.StartsWith('unreleased_') -or $depKey.StartsWith('beta_'))
+                                    {
+                                        $hasError = $true
+                                        return "Error: $($versionUpdateString) is using an unreleased_ or beta_ dependency and trying to set current value. Only dependency versions can be set with an unreleased or beta dependency."
+                                    }
+                                    if ($versionWithoutBraces -ne $libHash[$depKey].curVer)
+                                    {
+                                        $hasError = $true
+                                        return "Error: $($depKey)'s <version> is '$($versionString)' but the current version is listed as $($libHash[$depKey].curVer)"
+                                    }
                                 }
                             }
                         }