Merge branch 'main' into anfog/2022822_security_updates

Author: AFFogarty

azure-pipelines-e2e-tests-template.yml

@@ -24,7 +24,7 @@ stages:
         ${{ else }}:
           pool:
             ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
-              name: NetCore1ESPool-Public
+              name: NetCore-Public
               demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
             ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
               name: NetCore1ESPool-Internal
@@ -39,6 +39,20 @@ stages:
             pwsh: true
             targetType: inline
             script: |
+              $numRetry = 0
+              $maxRetry = 60*10
+
+              while($numRetry -le $maxRetry)
+              {
+                  sudo fuser -v /var/lib/dpkg/lock-frontend 2>&1
+
+                  if ($LASTEXITCODE -ne 0) { Break }
+                  
+                  sleep 1
+                  $numRetry++
+                  echo "Waited $numRetry s for release of dpkg locks"
+              }
+              
               sudo apt update
               sudo apt -y install maven
               mvn -version
@@ -169,4 +183,4 @@ stages:
           env:
             HADOOP_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)hadoop
             SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
-            DOTNET_WORKER_DIR: $(CURRENT_DOTNET_WORKER_DIR)
+            DOTNET_WORKER_DIR: $(CURRENT_DOTNET_WORKER_DIR)

azure-pipelines.yml

@@ -46,7 +46,7 @@ resources:
   repositories:
   - repository: forwardCompatibleRelease
     type: github
-    endpoint: dotnet
+    endpoint: public
     name: dotnet/spark
     ref: refs/tags/v$(forwardCompatibleRelease)
 
@@ -366,8 +366,8 @@ stages:
         backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1)
         forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1)
     - version: '3.2.0'
-      enableForwardCompatibleTests: false
-      enableBackwardCompatibleTests: false
+      enableForwardCompatibleTests: true
+      enableBackwardCompatibleTests: true
       jobOptions:
       - pool: 'Windows'
         testOptions: ""
@@ -378,8 +378,32 @@ stages:
         backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
         forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
     - version: '3.2.1'
-      enableForwardCompatibleTests: false
-      enableBackwardCompatibleTests: false
+      enableForwardCompatibleTests: true
+      enableBackwardCompatibleTests: true
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_2)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_2)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
+    - version: '3.2.2'
+      enableForwardCompatibleTests: true
+      enableBackwardCompatibleTests: true
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_2)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_2)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
+    - version: '3.2.3'
+      enableForwardCompatibleTests: true
+      enableBackwardCompatibleTests: true
       jobOptions:
       - pool: 'Windows'
         testOptions: ""

docs/building/ubuntu-instructions.md

@@ -40,7 +40,7 @@ If you already have all the pre-requisites, skip to the [build](ubuntu-instructi
        ```bash
        mkdir -p ~/bin/maven
        cd ~/bin/maven
-       wget https://www-us.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
+       wget https://dlcdn.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
        tar -xvzf apache-maven-3.6.3-bin.tar.gz
        ln -s apache-maven-3.6.3 current
        export M2_HOME=~/bin/maven/current

src/scala/microsoft-spark-3-2/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala

@@ -40,7 +40,7 @@ import scala.util.Try
 object DotnetRunner extends Logging {
   private val DEBUG_PORT = 5567
   private val supportedSparkMajorMinorVersionPrefix = "3.2"
-  private val supportedSparkVersions = Set[String]("3.2.0", "3.2.1")
+  private val supportedSparkVersions = Set[String]("3.2.0", "3.2.1", "3.2.2", "3.2.3")
 
   val SPARK_VERSION = DotnetUtils.normalizeSparkVersion(spark.SPARK_VERSION)