Skip to content

Commit a3498cf

Browse files
domenicsim1domenicsim1
authored
google cloud storage feed support (#1769)
1 parent dcdd23d commit a3498cf

File tree

4 files changed

+219
-0
lines changed

4 files changed

+219
-0
lines changed

‎source/Calamari.Common/Features/Packages/FeedType.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ public enum FeedType
1717
GoogleContainerRegistry,
1818
ArtifactoryGeneric,
1919
Npm,
20+
GcsStorage,
2021
}
2122
}

‎source/Calamari.Shared/Calamari.Shared.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
<ProjectReference Include="..\Calamari.CloudAccounts\Calamari.CloudAccounts.csproj" />
3030
<ProjectReference Include="..\Calamari.Common\Calamari.Common.csproj" />
3131
<PackageReference Include="AWSSDK.S3" Version="3.7.8.8" />
32+
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
33+
<PackageReference Include="Google.Cloud.Storage.V1" Version="4.10.0" />
3234
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
3335
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
3436
<PrivateAssets>all</PrivateAssets>

‎source/Calamari.Shared/Integration/Packages/Download/GcsStoragePackageDownloader.cs‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
using System;
2+
using System.IO;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using Calamari.Common.Commands;
8+
using Calamari.Common.Features.Packages;
9+
using Calamari.Common.Plumbing.FileSystem;
10+
using Calamari.Common.Plumbing.Logging;
11+
using Calamari.Common.Plumbing.Variables;
12+
using Google.Apis.Auth.OAuth2;
13+
using Google.Cloud.Storage.V1;
14+
using Octopus.CoreUtilities.Extensions;
15+
using Octopus.Versioning;
16+
17+
namespace Calamari.Integration.Packages.Download
18+
{
19+
public class GcsStoragePackageDownloader : IPackageDownloader
20+
{
21+
const string Extension = ".zip";
22+
const char BucketFileSeparator = '/';
23+
24+
// first item will be used as the default extension before checking for others
25+
static readonly string[] KnownFileExtensions =
26+
{
27+
".", // try to find a singular file without extension first
28+
".zip", ".tar.gz", ".tar.bz2", ".tgz", ".tar.bz"
29+
};
30+
31+
readonly IVariables variables;
32+
readonly ILog log;
33+
readonly ICalamariFileSystem fileSystem;
34+
35+
static readonly IPackageDownloaderUtils PackageDownloaderUtils = new PackageDownloaderUtils();
36+
37+
public GcsStoragePackageDownloader(IVariables variables, ILog log, ICalamariFileSystem fileSystem)
38+
{
39+
this.variables = variables;
40+
this.log = log;
41+
this.fileSystem = fileSystem;
42+
}
43+
44+
string BuildFileName(string prefix, string version, string extension)
45+
{
46+
return $"{prefix}.{version}{extension}";
47+
}
48+
49+
(string BucketName, string Filename) GetBucketAndKey(string searchTerm)
50+
{
51+
var splitString = searchTerm.Split(new[] { BucketFileSeparator }, 2);
52+
if (splitString.Length == 0)
53+
return ("", "");
54+
if (splitString.Length == 1)
55+
return (splitString[0], "");
56+
57+
return (splitString[0], splitString[1]);
58+
}
59+
60+
public PackagePhysicalFileMetadata DownloadPackage(
61+
string packageId,
62+
IVersion version,
63+
string feedId,
64+
Uri feedUri,
65+
string? feedUsername,
66+
string? feedPassword,
67+
bool forcePackageDownload,
68+
int maxDownloadAttempts,
69+
TimeSpan downloadAttemptBackoff)
70+
{
71+
var (bucketName, prefix) = GetBucketAndKey(packageId);
72+
if (string.IsNullOrWhiteSpace(bucketName) || string.IsNullOrWhiteSpace(prefix))
73+
{
74+
throw new InvalidOperationException($"Invalid PackageId for GCS feed. Expecting format `<bucketName>/<packageId>`, but received {bucketName}/{prefix}");
75+
}
76+
77+
var cacheDirectory = PackageDownloaderUtils.GetPackageRoot(feedId);
78+
fileSystem.EnsureDirectoryExists(cacheDirectory);
79+
80+
Log.VerboseFormat($"Checking package cache for package {packageId} v{version.ToString()}");
81+
var downloaded = GetFileFromCache(packageId, version, forcePackageDownload, cacheDirectory, KnownFileExtensions);
82+
if (downloaded != null)
83+
{
84+
return downloaded;
85+
}
86+
87+
var retry = 0;
88+
for (; retry < maxDownloadAttempts; ++retry)
89+
{
90+
try
91+
{
92+
log.Verbose($"Attempting download of package {packageId} version {version} from GCS bucket {bucketName}. Attempt #{retry + 1}");
93+
94+
var storageClient = CreateStorageClient(feedPassword);
95+
string? foundFilePath = null;
96+
97+
for (int i = 0; i < KnownFileExtensions.Length && foundFilePath == null; i++)
98+
{
99+
var fileName = BuildFileName(prefix, version.ToString(), KnownFileExtensions[i]);
100+
foundFilePath = FindSingleFileInBucket(storageClient, bucketName, fileName, CancellationToken.None)
101+
.GetAwaiter()
102+
.GetResult();
103+
}
104+
105+
var fullFileName = !foundFilePath.IsNullOrEmpty()
106+
? foundFilePath
107+
: throw new Exception($"Unable to download package {packageId} {version}: file not found");
108+
109+
var knownExtension = KnownFileExtensions.FirstOrDefault(extension => fullFileName!.EndsWith(extension))
110+
?? Path.GetExtension(fullFileName)
111+
?? Extension;
112+
113+
// Now we know the extension check for the package in the local cache
114+
downloaded = GetFileFromCache(packageId, version, forcePackageDownload, cacheDirectory, new[] { knownExtension });
115+
if (downloaded != null)
116+
{
117+
return downloaded;
118+
}
119+
120+
var localDownloadName = Path.Combine(cacheDirectory, PackageName.ToCachedFileName(packageId, version, knownExtension));
121+
122+
using (var outputStream = File.Create(localDownloadName))
123+
{
124+
storageClient.DownloadObject(bucketName, fullFileName, outputStream);
125+
}
126+
127+
var packagePhysicalFileMetadata = PackagePhysicalFileMetadata.Build(localDownloadName);
128+
return packagePhysicalFileMetadata
129+
?? throw new CommandException($"Unable to retrieve metadata for package {packageId}, version {version}");
130+
}
131+
catch (Exception ex)
132+
{
133+
log.Verbose($"Download attempt #{retry + 1} failed, with error: {ex.Message}. Retrying in {downloadAttemptBackoff}");
134+
135+
if ((retry + 1) == maxDownloadAttempts)
136+
throw new CommandException($"Unable to download package {packageId} {version}: " + ex.Message);
137+
Thread.Sleep(downloadAttemptBackoff);
138+
}
139+
}
140+
141+
throw new CommandException($"Failed to download package {packageId} {version}. Attempted {retry} times.");
142+
}
143+
144+
static StorageClient CreateStorageClient(string? serviceAccountJsonKey)
145+
{
146+
if (string.IsNullOrEmpty(serviceAccountJsonKey))
147+
{
148+
// Use Application Default Credentials
149+
return StorageClient.Create();
150+
}
151+
152+
// Use service account JSON key (feedPassword contains the JSON key)
153+
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(serviceAccountJsonKey));
154+
var credential = GoogleCredential.FromStream(stream);
155+
return StorageClient.Create(credential);
156+
}
157+
158+
PackagePhysicalFileMetadata? GetFileFromCache(
159+
string packageId,
160+
IVersion version,
161+
bool forcePackageDownload,
162+
string cacheDirectory,
163+
string[] fileExtensions)
164+
{
165+
if (forcePackageDownload)
166+
return null;
167+
var downloaded = SourceFromCache(packageId, version, cacheDirectory, fileExtensions);
168+
if (downloaded == null)
169+
return null;
170+
Log.VerboseFormat("Package was found in cache. No need to download. Using file: '{0}'", downloaded.FullFilePath);
171+
return downloaded;
172+
}
173+
174+
PackagePhysicalFileMetadata? SourceFromCache(string packageId, IVersion version, string cacheDirectory, string[] knownExtensions)
175+
{
176+
var files = fileSystem.EnumerateFilesRecursively(cacheDirectory, PackageName.ToSearchPatterns(packageId, version, knownExtensions));
177+
178+
foreach (var file in files)
179+
{
180+
var package = PackageName.FromFile(file);
181+
if (package == null)
182+
continue;
183+
184+
var idMatches = string.Equals(package.PackageId, packageId, StringComparison.OrdinalIgnoreCase);
185+
var versionExactMatch = string.Equals(package.Version.ToString(), version.ToString(), StringComparison.OrdinalIgnoreCase);
186+
var semverMatches = package.Version.Equals(version);
187+
188+
if (idMatches && (semverMatches || versionExactMatch))
189+
return PackagePhysicalFileMetadata.Build(file, package);
190+
}
191+
192+
return null;
193+
}
194+
195+
static async Task<string?> FindSingleFileInBucket(
196+
StorageClient client,
197+
string bucketName,
198+
string prefix,
199+
CancellationToken cancellationToken)
200+
{
201+
var objects = client.ListObjectsAsync(bucketName, prefix);
202+
var objectsList = new System.Collections.Generic.List<string>();
203+
204+
await foreach (var obj in objects.WithCancellation(cancellationToken))
205+
{
206+
objectsList.Add(obj.Name);
207+
if (objectsList.Count > 1) break;
208+
}
209+
210+
return objectsList.Count == 1 ? objectsList[0] : null;
211+
}
212+
}
213+
}

‎source/Calamari.Shared/Integration/Packages/Download/PackageDownloaderStrategy.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,9 @@ public PackagePhysicalFileMetadata DownloadPackage(string packageId,
7373
case FeedType.S3:
7474
downloader = new S3PackageDownloader(variables, log, fileSystem);
7575
break;
76+
case FeedType.GcsStorage:
77+
downloader = new GcsStoragePackageDownloader(variables, log, fileSystem);
78+
break;
7679
case FeedType.ArtifactoryGeneric:
7780
downloader = new ArtifactoryPackageDownloader(log, fileSystem, variables);
7881
break;

0 commit comments

Comments
 (0)