Merge branch 'main' into 1180-fix-url-validation

Author: AakashSuresh2003

.github/workflows/CI.yml

@@ -11,8 +11,15 @@ on:
     - cron: "0 7 * * *"
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   build:
+    permissions:
+      contents: read
+      actions: write
+      security-events: write
     strategy:
       fail-fast: false
       matrix:
@@ -85,6 +92,9 @@ jobs:
         if: matrix.runner-os == 'ubuntu-latest'
 
   upload-event-file:
+    permissions:
+      contents: read
+      actions: write
     runs-on: ubuntu-latest
     steps:
       # This is used by the subsequent publish-test-results.yaml
@@ -95,6 +105,9 @@ jobs:
           path: ${{ github.event_path }}
 
   build-for-e2e-test:
+    permissions:
+      contents: read
+      actions: write
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == 'github'
     strategy:
       fail-fast: false
@@ -140,6 +153,10 @@ jobs:
             dist/win-x64/gei-windows-amd64.exe
 
   e2e-test:
+    permissions:
+      contents: read
+      actions: write
+      checks: write
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == 'github'
     needs: [build-for-e2e-test]
     strategy:
@@ -276,6 +293,8 @@ jobs:
         shell: pwsh
 
   publish:
+    permissions:
+      contents: write
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/v')
     needs: [build, e2e-test]
@@ -321,10 +340,12 @@ jobs:
             ./dist/ado2gh.*.win-x64.zip
             ./dist/ado2gh.*.win-x86.zip
             ./dist/ado2gh.*.linux-x64.tar.gz
+            ./dist/ado2gh.*.linux-arm64.tar.gz
             ./dist/ado2gh.*.osx-x64.tar.gz
             ./dist/win-x64/gei-windows-amd64.exe
             ./dist/win-x86/gei-windows-386.exe
             ./dist/linux-x64/gei-linux-amd64
+            ./dist/linux-arm64/gei-linux-arm64
             ./dist/osx-x64/gei-darwin-amd64
 
       - name: Create gh-ado2gh Release
@@ -337,6 +358,7 @@ jobs:
             ./dist/win-x86/ado2gh-windows-386.exe
             ./dist/win-x64/ado2gh-windows-amd64.exe
             ./dist/linux-x64/ado2gh-linux-amd64
+            ./dist/linux-arm64/ado2gh-linux-arm64
             ./dist/osx-x64/ado2gh-darwin-amd64
 
       - name: Create gh-bbs2gh Release
@@ -349,6 +371,7 @@ jobs:
             ./dist/win-x86/bbs2gh-windows-386.exe
             ./dist/win-x64/bbs2gh-windows-amd64.exe
             ./dist/linux-x64/bbs2gh-linux-amd64
+            ./dist/linux-arm64/bbs2gh-linux-arm64
             ./dist/osx-x64/bbs2gh-darwin-amd64
 
       - name: Archive Release Notes

.github/workflows/publish-test-results.yml

@@ -35,10 +35,9 @@ jobs:
 
       - name: Extract PR Number
         run: |
-          eventjson=`cat 'artifacts/Event File/event.json'`
-          prnumber=`echo $(jq -r '.pull_request.number' <<< "$eventjson")`
-          echo "PR_NUMBER=$(echo $prnumber | tr -cd '0-9')" >> $GITHUB_ENV
-      
+          prnumber=$(jq -r '.pull_request.number' < 'artifacts/Event File/event.json')
+          sanitized_prnumber=$(grep -E '^[0-9]+$' <<< "$prnumber")
+          echo "PR_NUMBER=$sanitized_prnumber" >> "$GITHUB_ENV"
       - name: Publish Unit Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
         with:

README.md

@@ -95,6 +95,24 @@ When the CLI is launched, it logs if a newer version of the CLI is available. Yo
 
 When the CLI is launched, it logs a warning if there are any ongoing [GitHub incidents](https://www.githubstatus.com/) that might affect your use of the CLI. You can skip this check by setting the `GEI_SKIP_STATUS_CHECK` environment variable to `true`. 
 
+### Configuring multipart upload chunk size
+
+Set the `GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES` environment variable to change the archive upload part size. Provide the value in mebibytes (MiB); For example:
+
+```powershell
+# Windows PowerShell
+$env:GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES = "10"
+```
+
+```bash
+# macOS/Linux
+export GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES=10
+```
+
+This sets the chunk size to 10 MiB (10,485,760 bytes). The minimum supported value is 5 MiB, and the default remains 100 MiB.
+
+This might be needed to improve upload reliability in environments with proxies or very slow connections.
+
 ## Contributions
 
 See [Contributing](CONTRIBUTING.md) for more info on how to get involved.

RELEASENOTES.md

@@ -1,2 +1,4 @@
-- Added validation to detect and return clear error messages when a URL is provided instead of a name for organization, repository, or enterprise arguments (e.g., `--github-org`, `--github-target-org`, `--source-repo`, `--github-target-enterprise`)
-
+- Added support for linux-arm64 architecture for all CLI binaries (gei, ado2gh, bbs2gh), enabling users to run GEI on ARM-based systems including MacOS with Apple Silicon inside ARC runners
+- Fixed issue where alert migration commands (migrate-code-scanning-alerts, migrate-secret-alerts) required GH_PAT environment variable even when GitHub tokens were provided via command-line arguments (--github-source-pat, --github-target-pat). These commands now work correctly with CLI-only token authentication.
+- Added support for configurable multipart upload chunk size for GitHub-owned storage uploads via `GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES` environment variable (minimum 5 MiB, default 100 MiB) to improve upload reliability in environments with proxies or slow connections
+- Added validation to detect and return clear error messages when a URL is provided instead of a name for organization, repository, or enterprise arguments (e.g., `--github-org`, `--github-target-org`, `--source-repo`, `--github-target-enterprise`)

publish.ps1

@@ -59,6 +59,21 @@ else {
     }
 
     Copy-Item ./dist/linux-x64/ado2gh ./dist/linux-x64/ado2gh-linux-amd64
+
+    # linux-arm64 build
+    dotnet publish src/ado2gh/ado2gh.csproj -c Release -o dist/linux-arm64/ -r linux-arm64 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion
+
+    if ($LASTEXITCODE -ne 0) {
+        exit $LASTEXITCODE
+    }
+
+    tar -cvzf ./dist/ado2gh.$AssemblyVersion.linux-arm64.tar.gz -C ./dist/linux-arm64 ado2gh
+
+    if (Test-Path -Path ./dist/linux-arm64/ado2gh-linux-arm64) {
+        Remove-Item ./dist/linux-arm64/ado2gh-linux-arm64
+    }
+
+    Copy-Item ./dist/linux-arm64/ado2gh ./dist/linux-arm64/ado2gh-linux-arm64
 }
 
 if ((Test-Path env:SKIP_MACOS) -And $env:SKIP_MACOS.ToUpper() -eq "TRUE") {
@@ -126,6 +141,19 @@ else {
     }
 
     Rename-Item ./dist/linux-x64/gei gei-linux-amd64
+
+    # linux-arm64 build
+    dotnet publish src/gei/gei.csproj -c Release -o dist/linux-arm64/ -r linux-arm64 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion
+
+    if ($LASTEXITCODE -ne 0) {
+        exit $LASTEXITCODE
+    }
+
+    if (Test-Path -Path ./dist/linux-arm64/gei-linux-arm64) {
+        Remove-Item ./dist/linux-arm64/gei-linux-arm64
+    }
+
+    Rename-Item ./dist/linux-arm64/gei gei-linux-arm64
 }
 
 if ((Test-Path env:SKIP_MACOS) -And $env:SKIP_MACOS.ToUpper() -eq "TRUE") {
@@ -190,6 +218,19 @@ else {
     }
 
     Rename-Item ./dist/linux-x64/bbs2gh bbs2gh-linux-amd64
+
+    # linux-arm64 build
+    dotnet publish src/bbs2gh/bbs2gh.csproj -c Release -o dist/linux-arm64/ -r linux-arm64 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion
+
+    if ($LASTEXITCODE -ne 0) {
+        exit $LASTEXITCODE
+    }
+
+    if (Test-Path -Path ./dist/linux-arm64/bbs2gh-linux-arm64) {
+        Remove-Item ./dist/linux-arm64/bbs2gh-linux-arm64
+    }
+
+    Rename-Item ./dist/linux-arm64/bbs2gh bbs2gh-linux-arm64
 }
 
 if ((Test-Path env:SKIP_MACOS) -And $env:SKIP_MACOS.ToUpper() -eq "TRUE") {

src/Octoshift/Factories/GithubApiFactory.cs

@@ -32,7 +32,7 @@ GithubApi ISourceGithubApiFactory.Create(string apiUrl, string uploadsUrl, strin
         uploadsUrl ??= DEFAULT_UPLOADS_URL;
         sourcePersonalAccessToken ??= _environmentVariableProvider.SourceGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("Default"), _versionProvider, _retryPolicy, _dateTimeProvider, sourcePersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 
@@ -42,7 +42,7 @@ GithubApi ISourceGithubApiFactory.CreateClientNoSsl(string apiUrl, string upload
         uploadsUrl ??= DEFAULT_UPLOADS_URL;
         sourcePersonalAccessToken ??= _environmentVariableProvider.SourceGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("NoSSL"), _versionProvider, _retryPolicy, _dateTimeProvider, sourcePersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 
@@ -52,7 +52,7 @@ GithubApi ITargetGithubApiFactory.Create(string apiUrl, string uploadsUrl, strin
         uploadsUrl ??= DEFAULT_UPLOADS_URL;
         targetPersonalAccessToken ??= _environmentVariableProvider.TargetGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("Default"), _versionProvider, _retryPolicy, _dateTimeProvider, targetPersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy, _environmentVariableProvider);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 }

src/Octoshift/Services/ArchiveUploader.cs

@@ -11,18 +11,26 @@ namespace OctoshiftCLI.Services;
 
 public class ArchiveUploader
 {
+    private const int BYTES_PER_MEBIBYTE = 1024 * 1024;
+    private const int MIN_MULTIPART_MEBIBYTES = 5; // 5 MiB minimum size for multipart upload. Don't allow overrides smaller than this.
+    private const int DEFAULT_MULTIPART_MEBIBYTES = 100;
+
     private readonly GithubClient _client;
     private readonly string _uploadsUrl;
     private readonly OctoLogger _log;
-    internal int _streamSizeLimit = 100 * 1024 * 1024; // 100 MiB
+    private readonly EnvironmentVariableProvider _environmentVariableProvider;
+    internal int _streamSizeLimit = DEFAULT_MULTIPART_MEBIBYTES * BYTES_PER_MEBIBYTE; // 100 MiB stored in bytes
     private readonly RetryPolicy _retryPolicy;
 
-    public ArchiveUploader(GithubClient client, string uploadsUrl, OctoLogger log, RetryPolicy retryPolicy)
+    public ArchiveUploader(GithubClient client, string uploadsUrl, OctoLogger log, RetryPolicy retryPolicy, EnvironmentVariableProvider environmentVariableProvider)
     {
         _client = client;
         _uploadsUrl = uploadsUrl;
         _log = log;
         _retryPolicy = retryPolicy;
+        _environmentVariableProvider = environmentVariableProvider;
+
+        SetStreamSizeLimitFromEnvironment();
     }
     public virtual async Task<string> Upload(Stream archiveContent, string archiveName, string orgDatabaseId)
     {
@@ -160,4 +168,23 @@ private Uri GetNextUrl(IEnumerable<KeyValuePair<string, IEnumerable<string>>> he
         }
         throw new OctoshiftCliException("Location header is missing in the response, unable to retrieve next URL for multipart upload.");
     }
+
+    private void SetStreamSizeLimitFromEnvironment()
+    {
+        var envValue = _environmentVariableProvider.GithubOwnedStorageMultipartMebibytes();
+        if (!int.TryParse(envValue, out var limitInMebibytes) || limitInMebibytes <= 0)
+        {
+            return;
+        }
+
+        if (limitInMebibytes < MIN_MULTIPART_MEBIBYTES)
+        {
+            _log.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES is set to {limitInMebibytes} MiB, but the minimum value is {MIN_MULTIPART_MEBIBYTES} MiB. Using default value of {DEFAULT_MULTIPART_MEBIBYTES} MiB.");
+            return;
+        }
+
+        var limitBytes = (int)((long)limitInMebibytes * BYTES_PER_MEBIBYTE);
+        _streamSizeLimit = limitBytes;
+        _log.LogInformation($"Multipart upload part size set to {limitInMebibytes} MiB.");
+    }
 }

src/Octoshift/Services/EnvironmentVariableProvider.cs

@@ -18,6 +18,7 @@ public class EnvironmentVariableProvider
     private const string SMB_PASSWORD = "SMB_PASSWORD";
     private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK";
     private const string GEI_SKIP_VERSION_CHECK = "GEI_SKIP_VERSION_CHECK";
+    private const string GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES = "GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES";
 
     private readonly OctoLogger _logger;
 
@@ -65,6 +66,9 @@ public virtual string SkipStatusCheck(bool throwIfNotFound = false) =>
     public virtual string SkipVersionCheck(bool throwIfNotFound = false) =>
         GetValue(GEI_SKIP_VERSION_CHECK, throwIfNotFound);
 
+    public virtual string GithubOwnedStorageMultipartMebibytes(bool throwIfNotFound = false) =>
+        GetValue(GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES, throwIfNotFound);
+
     private string GetValue(string name, bool throwIfNotFound)
     {
         var value = Environment.GetEnvironmentVariable(name);

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

@@ -60,7 +60,8 @@ public BbsToGithub(ITestOutputHelper output)
         _targetGithubHttpClient = new HttpClient();
         _targetGithubClient = new GithubClient(_logger, _targetGithubHttpClient, new VersionChecker(_versionClient, _logger), new RetryPolicy(_logger), new DateTimeProvider(), targetGithubToken);
         var retryPolicy = new RetryPolicy(_logger);
-        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, _logger, retryPolicy);
+        var environmentVariableProvider = new EnvironmentVariableProvider(_logger);
+        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, _logger, retryPolicy, environmentVariableProvider);
         _targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(_logger), _archiveUploader);
 
         _blobServiceClient = new BlobServiceClient(_azureStorageConnectionString);

src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs

@@ -49,10 +49,11 @@ public GhesToGithub(ITestOutputHelper output)
 
         _versionClient = new HttpClient();
         var retryPolicy = new RetryPolicy(logger);
-        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, logger, retryPolicy);
+        var environmentVariableProvider = new EnvironmentVariableProvider(logger);
 
         _sourceGithubHttpClient = new HttpClient();
         _sourceGithubClient = new GithubClient(logger, _sourceGithubHttpClient, new VersionChecker(_versionClient, logger), new RetryPolicy(logger), new DateTimeProvider(), sourceGithubToken);
+        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, logger, retryPolicy, environmentVariableProvider);
         _sourceGithubApi = new GithubApi(_sourceGithubClient, GHES_API_URL, new RetryPolicy(logger), _archiveUploader);
 
         _targetGithubHttpClient = new HttpClient();