Add comments and cleanup azure pipeline (#764)

Author: suhsteve

azure-pipelines-e2e-tests-template.yml

@@ -20,10 +20,6 @@ stages:
       - job: Run_${{ replace(option.pool, ' ', '_') }}
         pool: ${{ option.pool }}
 
-        variables:
-          ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-            _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
-
         steps:
         - task: DownloadBuildArtifacts@0
           displayName: Download Build Artifacts
@@ -109,8 +105,6 @@ stages:
             }
           displayName: 'Setup Backward Compatible Microsoft Spark Worker ${{ parameters.backwardCompatibleRelease }}'
           workingDirectory: $(Build.BinariesDirectory)
-          env:
-            SPARK_VERSION: ${{ test.version }}
 
         - task: DotNetCoreCLI@2
           displayName: 'E2E Backward Compatibility Tests'

azure-pipelines.yml

@@ -31,6 +31,8 @@ variables:
   (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestIndexCreateAndDelete)&\
   (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestSignatures)"
 
+  # Filter DataFrameTests.TestDataFrameGroupedMapUdf and DataFrameTests.TestGroupedMapUdf backwardCompatible
+  # tests due to https://github.com/dotnet/spark/pull/711
   backwardCompatibleTestOptions_Windows_3: "--filter \
   (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\
   (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf)"