Sync eng/common directory with azure-sdk-tools for PR 7537 (#33888)

azure-sdk · scbedd · benbp · web-flow · commit 4ecc31b98b4e · 2024-01-17T14:09:56.000-08:00
* Add ConflictedFile to git-helpers.ps1, add git-helpers.tests.ps1 to exercise basic functionality.
* Add `resolve-asset-conflict.ps1` a script that can autoresolve an assets.json file.

---------

Co-authored-by: Scott Beddall (from Dev Box) &lt;scbedd@microsoft.com&gt;
Co-authored-by: Scott Beddall &lt;45376673+scbedd@users.noreply.github.com&gt;
Co-authored-by: Ben Broderick Phillips &lt;bebroder@microsoft.com&gt;
diff --git a/eng/common/scripts/Helpers/git-helpers.ps1 b/eng/common/scripts/Helpers/git-helpers.ps1
@@ -36,3 +36,75 @@ function Get-ChangedFiles {
   }
   return $changedFiles
 }
+
+class ConflictedFile {
+  [string]$LeftSource = ""
+  [string]$RightSource = ""
+  [string]$Content = ""
+  [string]$Path = ""
+  [boolean]$IsConflicted = $false
+
+  ConflictedFile([string]$File = "") {
+    if (!(Test-Path $File)) {
+      throw "File $File does not exist, pass a valid file path to the constructor."
+    }
+
+    # Normally we would use Resolve-Path $file, but git only can handle relative paths using git show <commitsh>:<path>
+    # Therefore, just maintain whatever the path is given to us. Left() and Right() should therefore be called from the same
+    # directory as where we defined the relative path to the target file.
+    $this.Path = $File
+    $this.Content = Get-Content -Raw $File
+
+    $this.ParseContent($this.Content)
+  }
+
+  [array] Left(){
+    if ($this.IsConflicted) {
+      # we are forced to get this line by line and reassemble via join because of how powershell is interacting with
+      # git show --textconv commitsh:path
+      # powershell ignores the newlines with and without --textconv, which results in a json file without original spacing.
+      # by forcefully reading into the array line by line, the whitespace is preserved. we're relying on gits autoconverstion of clrf to lf
+      # to ensure that the line endings are consistent.
+      Write-Host "git show $($this.LeftSource):$($this.Path)"
+      $tempContent = git show ("$($this.LeftSource):$($this.Path)")
+      return $tempContent -split "`r?`n"
+    }
+    else {
+      return $this.Content
+    }
+  }
+
+  [array] Right(){
+    if ($this.IsConflicted) {
+      Write-Host "git show $($this.RightSource):$($this.Path)"
+      $tempContent =  git show ("$($this.RightSource):$($this.Path)")
+      return $tempContent -split "`r?`n"
+    }
+    else {
+      return $this.Content
+    }
+  }
+
+  [void] ParseContent([string]$IncomingContent) {
+    $lines = $IncomingContent -split "`r?`n"
+    $l = @()
+    $r = @()
+    
+    foreach($line in $lines) {
+      if ($line -match "^<<<<<<<\s*(.+)") {
+        $this.IsConflicted = $true
+        $this.LeftSource = $matches[1]
+        continue
+      }
+      elseif ($line -match "^>>>>>>>\s*(.+)") {
+        $this.IsConflicted = $true
+        $this.RightSource = $matches[1]
+        continue
+      }
+
+      if ($this.LeftSource -and $this.RightSource) {
+        break
+      }
+    }
+  }
+}
diff --git a/eng/common/scripts/Helpers/git-helpers.tests.ps1 b/eng/common/scripts/Helpers/git-helpers.tests.ps1
@@ -0,0 +1,57 @@
+# Install-Module -Name Pester -Force -SkipPublisherCheck
+# Invoke-Pester -Passthru path/to/git-helpers.tests.ps1
+BeforeAll {
+    . $PSScriptRoot/git-helpers.ps1
+
+    $RunFolder = "$PSScriptRoot/.testruns"
+
+    if (Test-Path $RunFolder){
+        Remove-Item -Recurse -Force $RunFolder
+    }
+
+    New-Item -ItemType Directory -Path $RunFolder
+}
+
+Describe "git-helpers.ps1 tests"{
+    Context "Test Parse-ConflictedFile" {
+        It "Parses a basic conflicted file" {
+            $content = @'
+{
+  "AssetsRepo": "Azure/azure-sdk-assets-integration",
+  "AssetsRepoPrefixPath": "python",
+  "TagPrefix": "python/storage/azure-storage-blob",
+<<<<<<< HEAD
+  "Tag": "integration/example/storage_feature_addition2"
+=======
+  "Tag": "integration/example/storage_feature_addition1"
+>>>>>>> test-storage-tag-combination
+}
+'@
+            $contentPath = Join-Path $RunFolder "basic_conflict_test.json"
+            Set-Content -Path $contentPath -Value $content
+
+            $resolution = [ConflictedFile]::new($contentPath)
+            $resolution.IsConflicted | Should -Be $true
+            $resolution.LeftSource | Should -Be "HEAD"
+            $resolution.RightSource | Should -Be "test-storage-tag-combination"
+        }
+
+        It "Recognizes when no conflicts are present" {
+            $content = @'
+{
+  "AssetsRepo": "Azure/azure-sdk-assets-integration",
+  "AssetsRepoPrefixPath": "python",
+  "TagPrefix": "python/storage/azure-storage-blob",
+  "Tag": "integration/example/storage_feature_addition1"
+}
+'@
+            $contentPath = Join-Path $RunFolder "no_conflict_test.json"
+            Set-Content -Path $contentPath -Value $content
+
+            $resolution = [ConflictedFile]::new($contentPath)
+            $resolution.IsConflicted | Should -Be $false
+            $resolution.LeftSource | Should -Be ""
+            $resolution.RightSource | Should -Be ""
+        }
+    }
+}
diff --git a/eng/common/testproxy/onboarding/common-asset-functions.ps1 b/eng/common/testproxy/onboarding/common-asset-functions.ps1
@@ -57,6 +57,24 @@ class Version {
   }
 }
 
+Function Resolve-Proxy {
+    $testProxyExe = "test-proxy"
+    # this script requires the presence of the test-proxy on the PATH
+    $proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false
+    $proxyStandalonePresent = Test-Exe-In-Path -ExeToLookFor "Azure.Sdk.Tools.TestProxy" -ExitOnError $false
+
+    if (-not $proxyToolPresent -and -not $proxyStandalonePresent) {
+        Write-Error "This script requires the presence of a test-proxy executable to complete its operations. Exiting."
+        exit 1
+    }
+
+    if (-not $proxyToolPresent) {
+        $testProxyExe = "Azure.Sdk.Tools.TestProxy"
+    }
+
+    return $testProxyExe
+}
+
 Function Test-Exe-In-Path {
   Param([string] $ExeToLookFor, [bool]$ExitOnError = $true)
   if ($null -eq (Get-Command $ExeToLookFor -ErrorAction SilentlyContinue)) {
diff --git a/eng/common/testproxy/scripts/resolve-asset-conflict/README.md b/eng/common/testproxy/scripts/resolve-asset-conflict/README.md
@@ -0,0 +1,62 @@
+# Merge Proxy Tags Script
+
+This script is intended to allow easy resolution of a conflicting `assets.json` file.
+
+In most cases where two branches `X` and `Y` have progressed alongside each other, a simple
+
+`git checkout X && git merge Y` can successfully merge _other_ than the `assets.json` file.
+
+That often will end up looking like this:
+
+```text
+{
+  "AssetsRepo": "Azure/azure-sdk-assets-integration",
+  "AssetsRepoPrefixPath": "python",
+  "TagPrefix": "python/storage/azure-storage-blob",
+<<<<<<< HEAD
+  "Tag": "integration/example/storage_feature_addition2"
+=======
+  "Tag": "integration/example/storage_feature_addition1"
+>>>>>>> test-storage-tag-combination
+}
+```
+
+This script uses `git` to tease out the source and target tags, then merge the incoming tag into the recordings of the base tag.
+
+This script should _only_ be used on an already conflicted `assets.json` file. Otherwise, no action will be executed.
+
+## Usage
+
+### PreReqs
+
+- Must have []`pshell 6+`](https://learn.microsoft.com/powershell/scripting/install/installing-powershell-on-windows)
+- Must have `git` available on your PATH
+- Must have the `test-proxy` available on your PATH
+  - `test-proxy` is honored when the proxy is installed as a `dotnet tool`
+  - `Azure.Sdk.Tools.TestProxy` is honored when the standalone executable is on your PATH
+  - Defaults to `dotnet tool` if both are present on the PATH.
+
+### Call the script
+
+For simplicity when resolving merge-conflicts, invoke the script from the root of the repo. The help instructions from `merge-asset-tags` use paths relative from repo root.
+
+```powershell
+# including context to get into a merge conflict
+cd "path/to/language/repo/root"
+git checkout base-branch
+git merge target-branch
+# auto resolve / merge conflicting tag values
+./eng/common/testproxy/scripts/resolve-asset-conflict/resolve-asset-conflict.ps1 sdk/storage/azure-storage-blob/assets.json
+# user pushes
+test-proxy push -a sdk/storage/azure-storage-blob/assets.json
+```
+
+### Resolving conflicts
+
+When an assets.json merge has conflicts on the **test recordings** side, the `merge-proxy-tags` script will exit with an error describing how to re-invoke the `merge-proxy-tags` script AFTER you resolve the conflicts.
+
+- `cd` into the assets location output by the script
+- resolve the conflict or conflicts
+- add the resolution, and invoke `git cherry-pick --continue`
+
+Afterwards, re-invoke the `merge-proxy-tags` script with arguments given to you in original error. This will leave the assets in a `touched` state that can be `test-proxy push`-ed.
diff --git a/eng/common/testproxy/scripts/resolve-asset-conflict/resolve-asset-conflict.ps1 b/eng/common/testproxy/scripts/resolve-asset-conflict/resolve-asset-conflict.ps1
@@ -0,0 +1,79 @@
+#Requires -Version 6.0
+#Requires -PSEdition Core
+
+<#
+.SYNOPSIS
+Within an assets.json file that has in a conflicted state (specifically on asset tag), merge the two tags that are conflicting and leave the assets.json in a commitable state.
+
+.DESCRIPTION
+USAGE: resolve-asset-conflict.ps1 path/to/target_assets_json
+
+Parses the assets.json file and determines which tags are in conflict. If there are no conflicts, the script exits.
+
+1. Parse the tags (base and target) from conflicting assets.json.
+2. Update the assets.json with the base tag, but remember the target tag.
+3. merge-proxy-tags.ps1 $AssetsJson base_tag target_tag
+
+This script requires that test-proxy or azure.sdk.tools.testproxy should be on the PATH.
+
+.PARAMETER AssetsJson
+The script uses a target assets.json to understand what tags are in conflict. This is the only required parameter.
+#>
+
+param(
+  [Parameter(Position=0)]
+  [string] $AssetsJson
+)
+
+. (Join-Path $PSScriptRoot ".." ".." "onboarding" "common-asset-functions.ps1")
+. (Join-Path $PSScriptRoot ".." ".." ".." "scripts" "Helpers" "git-helpers.ps1")
+
+$TestProxy = Resolve-Proxy
+
+if (!(Test-Path $AssetsJson)) {
+  Write-Error "AssetsJson file does not exist: $AssetsJson"
+  exit 1
+}
+
+# normally we we would Resolve-Path the $AssetsJson, but the git show command only works with relative paths, so we'll just keep that here.
+if (-not $AssetsJson.EndsWith("assets.json")) {
+  Write-Error "This script can only resolve conflicts within an assets.json. The file provided is not an assets.json: $AssetsJson"
+  exit 1
+}
+
+$conflictingAssets = [ConflictedFile]::new($AssetsJson)
+
+if (-not $conflictingAssets.IsConflicted) {
+  Write-Host "No conflicts found in $AssetsJson, nothing to resolve, so there is no second tag to merge. Exiting"
+  exit 0
+}
+
+# this is very dumb, but will properly work!
+try {
+  $BaseAssets = $conflictingAssets.Left() | ConvertFrom-Json
+}
+catch {
+  Write-Error "Failed to convert previous version to valid JSON format."
+  exit 1
+}
+
+try {
+  $TargetAssets = $conflictingAssets.Right() | ConvertFrom-Json
+}
+catch {
+  Write-Error "Failed to convert target assets.json version to valid JSON format."
+  exit 1
+}
+
+Write-Host "Replacing conflicted assets.json with base branch version." -ForegroundColor Green
+Set-Content -Path $AssetsJson -Value $conflictingAssets.Left()
+
+$ScriptPath = Join-Path $PSScriptRoot ".." "tag-merge" "merge-proxy-tags.ps1"
+& $ScriptPath $AssetsJson $BaseAssets.Tag $TargetAssets.Tag
+
+if ($lastexitcode -eq 0) {
+   Write-Host "Successfully auto-merged assets tag '$($TargetASsets.Tag)' into tag '$($BaseAssets.Tag)'. Invoke 'test-proxy push -a $AssetsJson' and commit the resulting assets.json!" -ForegroundColor Green
+}
+else {
+  Write-Host "Conflicts were discovered, resolve the conflicts and invoke the `"merge-proxy-tags.ps1`" as recommended in the line directly above."
+}
diff --git a/eng/common/testproxy/scripts/tag-merge/README.md b/eng/common/testproxy/scripts/tag-merge/README.md
@@ -16,7 +16,7 @@ This script merely allows the abstraction of some of this "combination" work.
 - Must have the `test-proxy` available on your PATH
   - `test-proxy` is honored when the proxy is installed as a `dotnet tool`
   - `Azure.Sdk.Tools.TestProxy` is honored when the standalone executable is on your PATH
-  - Preference for `dotnet tool` if present
+  - Defaults to `dotnet tool` if both are present on the PATH.
 
 ### Call the script
 
diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1
@@ -68,24 +68,6 @@ function Git-Command($CommandString, $WorkingDirectory, $HardExit=$true) {
     return $result.Output
 }
 
-function Resolve-Proxy {
-    $testProxyExe = "test-proxy"
-    # this script requires the presence of the test-proxy on the PATH
-    $proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false
-    $proxyStandalonePresent = Test-Exe-In-Path -ExeToLookFor "Azure.Sdk.Tools.TestProxy" -ExitOnError $false
-
-    if (-not $proxyToolPresent -and -not $proxyStandalonePresent) {
-        Write-Error "This script requires the presence of a test-proxy executable to complete its operations. Exiting."
-        exit 1
-    }
-
-    if (-not $proxyToolPresent) {
-        $testProxyExe = "Azure.Sdk.Tools.TestProxy"
-    }
-
-    return $testProxyExe
-}
-
 function Call-Proxy {
     param(
     [string] $TestProxyExe,
@@ -256,14 +238,17 @@ function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) {
     }
 }
 
-function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){
+function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory, $RelativeAssetsJson){
+    $remainingTagString = $RemainingTags -join " "
     foreach($Tag in $RemainingTags) {
         $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation
         $existingTags = Save-Incomplete-Progress $Tag $MountDirectory
         $cherryPickResult = Git-Command-With-Result "cherry-pick $tagSha" - $AssetsRepoLocation -HardExit $false
 
         if ($cherryPickResult.ExitCode -ne 0) {
-            Write-Host "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-run this script with the same arguments as before." -ForegroundColor Red
+            $error = "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-invoke " +
+            "by `"./eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 $RelativeAssetsJson $remainingTagString`""
+            Write-Host $error -ForegroundColor Red
             exit 1
         }
     }
@@ -294,6 +279,7 @@ if ($PSVersionTable["PSVersion"].Major -lt 6) {
 # resolve the proxy location so that we can invoke it easily, if not present we exit here.
 $proxyExe = Resolve-Proxy
 
+$relativeAssetsJson = $AssetsJson
 $AssetsJson = Resolve-Path $AssetsJson
 
 # figure out where the root of the repo for the passed assets.json is. We need it to properly set the mounting
@@ -313,6 +299,6 @@ $tags = Resolve-Target-Tags $AssetsJson $TargetTags $mountDirectory
 
 Start-Message $AssetsJson $Tags $AssetsRepoLocation $mountDirectory
 
-$CombinedTags = Combine-Tags $Tags $AssetsRepoLocation $mountDirectory
+$CombinedTags = Combine-Tags $Tags $AssetsRepoLocation $mountDirectory $relativeAssetsJson
 
 Finish-Message $AssetsJson $CombinedTags $AssetsRepoLocation $mountDirectory
