diff --git a/ASGE.csproj b/ASGE.csproj
index f2a400a..e67ee11 100644
--- a/ASGE.csproj
+++ b/ASGE.csproj
@@ -1,95 +1,20 @@
-
-
-
-
- Debug
- AnyCPU
- {84285BB3-2556-45F3-965F-4667855A6EB6}
- Exe
- Properties
- ASGE
- ASGE
- v4.5.2
- 512
- true
-
-
- AnyCPU
- true
- full
- false
- bin\Debug\
- DEBUG;TRACE
- prompt
- 4
-
-
- AnyCPU
- pdbonly
- true
- bin\Release\
- TRACE
- prompt
- 4
-
-
-
- packages\CommandLineParser.1.9.71\lib\net45\CommandLine.dll
- True
-
-
- packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll
- True
-
-
- packages\Microsoft.Data.Edm.5.6.4\lib\net40\Microsoft.Data.Edm.dll
- True
-
-
- packages\Microsoft.Data.OData.5.6.4\lib\net40\Microsoft.Data.OData.dll
- True
-
-
- packages\Microsoft.Data.Services.Client.5.6.4\lib\net40\Microsoft.Data.Services.Client.dll
- True
-
-
- packages\WindowsAzure.Storage.7.0.0\lib\net40\Microsoft.WindowsAzure.Storage.dll
- True
-
-
- packages\Newtonsoft.Json.6.0.8\lib\net45\Newtonsoft.Json.dll
- True
-
-
-
-
- packages\System.Spatial.5.6.4\lib\net40\System.Spatial.dll
- True
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
+
+
+
+ Exe
+ net5.0
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/ASGE.sln b/ASGE.sln
index 4e859bf..eba5633 100644
--- a/ASGE.sln
+++ b/ASGE.sln
@@ -1,9 +1,9 @@
Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 14
-VisualStudioVersion = 14.0.25123.0
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.29613.14
MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ASGE", "ASGE.csproj", "{84285BB3-2556-45F3-965F-4667855A6EB6}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ASGE", "ASGE.csproj", "{729E9638-9F5D-459B-8A3C-48218587C394}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -11,12 +11,15 @@ Global
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
- {84285BB3-2556-45F3-965F-4667855A6EB6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
- {84285BB3-2556-45F3-965F-4667855A6EB6}.Debug|Any CPU.Build.0 = Debug|Any CPU
- {84285BB3-2556-45F3-965F-4667855A6EB6}.Release|Any CPU.ActiveCfg = Release|Any CPU
- {84285BB3-2556-45F3-965F-4667855A6EB6}.Release|Any CPU.Build.0 = Release|Any CPU
+ {729E9638-9F5D-459B-8A3C-48218587C394}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {729E9638-9F5D-459B-8A3C-48218587C394}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {729E9638-9F5D-459B-8A3C-48218587C394}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {729E9638-9F5D-459B-8A3C-48218587C394}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
+ GlobalSection(ExtensibilityGlobals) = postSolution
+ SolutionGuid = {1E19E560-5C8D-423E-A1F1-853F39E51721}
+ EndGlobalSection
EndGlobal
diff --git a/EnumerableExtensions.cs b/EnumerableExtensions.cs
new file mode 100644
index 0000000..e897fd1
--- /dev/null
+++ b/EnumerableExtensions.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace ASGE
+{
+ public static class EnumerableExtensions
+ {
+ ///
+ /// From https://devblogs.microsoft.com/pfxteam/implementing-a-simple-foreachasync-part-2/.
+ ///
+ public static Task ForEachAsync(this IEnumerable source, Func body, int dop = 4)
+ {
+ return Task.WhenAll(
+ from partition in Partitioner.Create(source).GetPartitions(dop)
+ select Task.Run(async delegate
+ {
+ using (partition)
+ while (partition.MoveNext())
+ await body(partition.Current);
+ }));
+ }
+ }
+}
diff --git a/Options.cs b/Options.cs
index 3034b06..37884e7 100644
--- a/Options.cs
+++ b/Options.cs
@@ -10,27 +10,27 @@ namespace ASGE
{
class Options
{
- [Option('a', "account", Required = false, MutuallyExclusiveSet = "Account and Key",
+ [Option('a', "account", Required = false, SetName = "Account and Key",
HelpText = "Storage account host. [mystorage]")]
public string StorageAccount { get; set; }
- [Option('k', "key", Required = false, MutuallyExclusiveSet = "Account and Key",
+ [Option('k', "key", Required = false, SetName = "Account and Key",
HelpText = "Storage account key.")]
public string StorageKey { get; set; }
- [Option('c', "connectionstring", Required = false, MutuallyExclusiveSet = "ConnectionString",
+ [Option('c', "connectionstring", Required = false, SetName = "ConnectionString",
HelpText = "Storage account connection string.")]
public string ConnectionString { get; set; }
- [OptionArray('e', "extensions", Required = true,
- HelpText = "Extensions to operate on. [.js, .css, .dat]")]
- public string[] Extensions { get; set; }
+ [Option('i', "include", Required = true,
+ HelpText = "Regular expressions to match files to operate on. [*\\.js, \\.css, \\.dat]")]
+ public IEnumerable Include { get; set; }
- [Option('r', "replace", Required = false, DefaultValue = false,
+ [Option('r', "replace", Required = false, Default = false,
HelpText = "Replace existing files in-place.")]
public bool Replace { get; set; }
- [Option('s', "simulate", Required = false, DefaultValue = false,
+ [Option('s', "simulate", Required = false, Default = false,
HelpText = "Do everything except write to blob store.")]
public bool Simulate { get; set; }
@@ -42,32 +42,23 @@ class Options
HelpText = "Container to search in.")]
public string Container { get; set; }
- [Option('x', "cacheage", Required = false, DefaultValue = 2592000,
- HelpText = "Duration for cache control max age header, in seconds. Default 2592000 (30 days).")]
- public int MaxAgeSeconds { get; set; }
+ [Option('h', "cachecontrol", Required = false, Default = null,
+ HelpText = "Cache-Control header to be sent for the resource from the server.")]
+ public string CacheControlHeader { get; set; }
- [Option('w', "wildcardcors", Required = false, DefaultValue = false,
+ [Option('w', "wildcardcors", Required = false, Default = false,
HelpText = "Enable wildcard CORS for this storage account.")]
public bool wildcard { get; set; }
-
- [HelpOption]
- public string GetUsage()
+ [Usage(ApplicationAlias = "ASGE")]
+ public static IEnumerable Examples
{
- var help = new HelpText
+ get
{
- Heading = new HeadingInfo("Azure Storage GZip Encoder", "1.0"),
- Copyright = new CopyrightInfo("Stefan Gordon", 2016),
- AdditionalNewLineAfterOption = true,
- AddDashesToOption = true
- };
- help.AddPreOptionsLine("https://github.com/stefangordon/azure-storage-gzip-encoding\n");
- help.AddPreOptionsLine("\nExample 1: asge --extensions .css .js --container assets --replace --connectionstring ");
- help.AddPreOptionsLine("Example 2: asge --extensions .css .js --container assets --newextension .gz --account mystorage --key ");
- help.AddPreOptionsLine("\nUse either connection string (-c) or account and key (-a and -k).");
- help.AddOptions(this);
- return help;
+ return new List() {
+ new Example("Azure Storage GZip Encoding", new Options { })
+ };
+ }
}
-
}
}
diff --git a/Program.cs b/Program.cs
index be507d7..c5478d7 100644
--- a/Program.cs
+++ b/Program.cs
@@ -1,60 +1,73 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
-using System.Linq;
-using System.Text;
using System.Threading.Tasks;
using CommandLine;
-using Microsoft.Azure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Auth;
using Microsoft.WindowsAzure.Storage.Blob;
+using Serilog;
namespace ASGE
{
class Program
{
- static void Main(string[] args)
+ static async Task Main(string[] args)
{
- var options = new Options();
- if (CommandLine.Parser.Default.ParseArguments(args, options))
+ Log.Logger = new LoggerConfiguration()
+ .WriteTo.Console()
+ .CreateLogger();
+
+ var result = await CommandLine.Parser.Default.ParseArguments(args)
+ .MapResult(
+ async opts => await RunOptionsAndReturnExitCode(opts),
+ async errs => await HandleParseError(errs));
+ Log.Information("Return code= {0}", result);
+ }
+ static async Task RunOptionsAndReturnExitCode(Options options)
+ {
+ if (string.IsNullOrEmpty(options.NewExtension) && !options.Replace)
+ {
+ Log.Error("Must provide either -r (in-place replacement) or -n (new extension/postfix to append to compressed version).");
+ return -1;
+ }
+
+ CloudStorageAccount storageAccount;
+
+ if (!string.IsNullOrEmpty(options.ConnectionString))
+ {
+ storageAccount = CloudStorageAccount.Parse(options.ConnectionString);
+ }
+ else if (!string.IsNullOrEmpty(options.StorageAccount) && !string.IsNullOrEmpty(options.StorageKey))
+ {
+ storageAccount = new CloudStorageAccount(new StorageCredentials(options.StorageAccount, options.StorageKey), true);
+ }
+ else
+ {
+ Log.Error("Must provide either storagAccount+storageKey or connectionString.");
+ return -1;
+ }
+
+ CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
+ CloudBlobContainer blobContainer = blobClient.GetContainerReference(options.Container);
+
+ // Do the compression work
+ await Utility.EnsureGzipFiles(blobContainer, options.Include, options.Replace, options.NewExtension, options.CacheControlHeader, options.Simulate);
+
+ // Enable CORS if appropriate
+ if (options.wildcard)
{
- if (string.IsNullOrEmpty(options.NewExtension) && !options.Replace)
- {
- Console.WriteLine("Must provide either -r (in-place replacement) or -n (new extension/postfix to append to compressed version).");
- return;
- }
-
- CloudStorageAccount storageAccount;
-
- if (!string.IsNullOrEmpty(options.ConnectionString))
- {
- storageAccount = CloudStorageAccount.Parse(options.ConnectionString);
- }
- else if (!string.IsNullOrEmpty(options.StorageAccount) && !String.IsNullOrEmpty(options.StorageKey))
- {
- storageAccount = new CloudStorageAccount(new StorageCredentials(options.StorageAccount, options.StorageKey), true);
- }
- else
- {
- options.GetUsage();
- return;
- }
-
- CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
- CloudBlobContainer blobContainer = blobClient.GetContainerReference(options.Container);
-
- // Do the compression work
- Utility.EnsureGzipFiles(blobContainer, options.Extensions, options.Replace, options.NewExtension, options.MaxAgeSeconds, options.Simulate);
-
- // Enable CORS if appropriate
- if (options.wildcard)
- {
- Utility.SetWildcardCorsOnBlobService(storageAccount);
- }
-
- Trace.TraceInformation("Complete.");
+ await Utility.SetWildcardCorsOnBlobService(storageAccount);
}
+
+ Log.Information("Complete.");
+ return 0;
+ }
+
+ //in case of errors or --help or --version
+ static async Task HandleParseError(IEnumerable errs)
+ {
+ return await Task.FromResult(-1);
}
}
}
diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs
index b7aff2b..195fa29 100644
--- a/Properties/AssemblyInfo.cs
+++ b/Properties/AssemblyInfo.cs
@@ -5,11 +5,7 @@
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
-[assembly: AssemblyTitle("ASGE")]
[assembly: AssemblyDescription("")]
-[assembly: AssemblyConfiguration("")]
-[assembly: AssemblyCompany("")]
-[assembly: AssemblyProduct("ASGE")]
[assembly: AssemblyCopyright("Copyright © 2016")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@@ -21,16 +17,3 @@
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("84285bb3-2556-45f3-965f-4667855a6eb6")]
-
-// Version information for an assembly consists of the following four values:
-//
-// Major Version
-// Minor Version
-// Build Number
-// Revision
-//
-// You can specify all the values or you can default the Build and Revision Numbers
-// by using the '*' as shown below:
-// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.0.0.0")]
-[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/Utility.cs b/Utility.cs
index 808101e..34e270a 100644
--- a/Utility.cs
+++ b/Utility.cs
@@ -1,112 +1,129 @@
using System;
+using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
-using System.Text;
+using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.Shared.Protocol;
+using Serilog;
namespace ASGE
{
static class Utility
{
- public static void EnsureGzipFiles(CloudBlobContainer container, IEnumerable extensions, bool inPlace, string newExtension, int cacheControlMaxAgeSeconds, bool simulate)
+ public static async Task EnsureGzipFiles(CloudBlobContainer container, IEnumerable include, bool inPlace, string newExtension, string cacheControlHeader, bool simulate)
{
- Trace.TraceInformation("Enumerating files.");
+ Log.Information("Enumerating files.");
- string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
+ BlobContinuationToken blobContinuationToken = null;
+ do
+ {
+ var resultSegment = await container.ListBlobsSegmentedAsync(
+ prefix: null,
+ useFlatBlobListing: true,
+ blobListingDetails: BlobListingDetails.Metadata,
+ maxResults: null,
+ currentToken: blobContinuationToken,
+ options: null,
+ operationContext: null
+ );
+
+ blobContinuationToken = resultSegment.ContinuationToken;
+ await resultSegment.Results.ForEachAsync(
+ async (blobInfo) =>
+ await EnsureGzipOneFile(container, include, inPlace, newExtension, simulate, blobInfo, cacheControlHeader));
+
+ } while (blobContinuationToken != null); // Loop while the continuation token is not null.
+ }
- var blobInfos = container.ListBlobs(null, true, BlobListingDetails.Metadata);
+ private static async Task EnsureGzipOneFile(CloudBlobContainer container, IEnumerable include, bool inPlace, string newExtension, bool simulate, IListBlobItem blobInfo, string cacheControlHeader)
+ {
+ CloudBlob gzipBlob = null;
+ CloudBlob blob = (CloudBlob)blobInfo;
- Parallel.ForEach(blobInfos, (blobInfo) =>
+ // Only work with desired extensions
+ string extension = Path.GetExtension(blobInfo.Uri.LocalPath);
+ if (!include.Any(i => Regex.IsMatch(blobInfo.Uri.LocalPath, i)))
{
- CloudBlob gzipBlob = null;
- CloudBlob blob = (CloudBlob)blobInfo;
+ return;
+ }
- // Only work with desired extensions
- string extension = Path.GetExtension(blobInfo.Uri.LocalPath);
- if (!extensions.Contains(extension, StringComparer.OrdinalIgnoreCase))
+ // Check if it is already done
+ if (inPlace)
+ {
+ if (string.Equals(blob.Properties.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase))
{
+ Log.Information("Skipping already compressed blob: " + blob.Name);
return;
}
+ }
+ else
+ {
+ string gzipUrl = blob.Name + newExtension;
+ gzipBlob = container.GetBlockBlobReference(gzipUrl);
- // Check if it is already done
- if (inPlace)
+ if (await gzipBlob.ExistsAsync())
{
- if (string.Equals(blob.Properties.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase))
- {
- Trace.TraceInformation("Skipping already compressed blob: " + blob.Name);
- return;
- }
- }
- else
- {
- string gzipUrl = blob.Name + newExtension;
- gzipBlob = container.GetBlockBlobReference(gzipUrl);
-
- if (gzipBlob.Exists())
- {
- Trace.TraceInformation("Skipping already compressed blob: " + blob.Name);
- return;
- }
+ Log.Information("Skipping already compressed blob: " + blob.Name);
+ return;
}
+ }
- // Compress blob contents
- Trace.TraceInformation("Downloading blob: " + blob.Name);
+ // Compress blob contents
+ Log.Information("Downloading blob: " + blob.Name);
- byte[] compressedBytes;
+ byte[] compressedBytes;
- using (MemoryStream memoryStream = new MemoryStream())
+ using (MemoryStream memoryStream = new MemoryStream())
+ {
+ using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress))
+ using (var blobStream = await blob.OpenReadAsync())
{
- using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress))
- using (var blobStream = blob.OpenRead())
- {
- blobStream.CopyTo(gzipStream);
- }
-
- compressedBytes = memoryStream.ToArray();
+ blobStream.CopyTo(gzipStream);
}
- // Blob to write to
- CloudBlockBlob destinationBlob;
+ compressedBytes = memoryStream.ToArray();
+ }
- if (inPlace)
- {
- destinationBlob = (CloudBlockBlob)blob;
- }
- else
- {
- destinationBlob = (CloudBlockBlob)gzipBlob;
- }
+ // Blob to write to
+ CloudBlockBlob destinationBlob;
- if (simulate)
- {
- Trace.TraceInformation("NOT writing blob, due to simulation: " + blob.Name);
- }
- else
- {
- // Upload the compressed bytes to the new blob
- Trace.TraceInformation("Writing blob: " + blob.Name);
- destinationBlob.UploadFromByteArray(compressedBytes, 0, compressedBytes.Length);
-
- // Set the blob headers
- Trace.TraceInformation("Configuring headers");
- destinationBlob.Properties.CacheControl = cacheControlHeader;
- destinationBlob.Properties.ContentType = blob.Properties.ContentType;
- destinationBlob.Properties.ContentEncoding = "gzip";
- destinationBlob.SetProperties();
- }
+ if (inPlace)
+ {
+ destinationBlob = (CloudBlockBlob)blob;
+ }
+ else
+ {
+ destinationBlob = (CloudBlockBlob)gzipBlob;
+ }
- });
+ if (simulate)
+ {
+ Log.Information("NOT writing blob, due to simulation: " + blob.Name);
+ }
+ else
+ {
+ // Upload the compressed bytes to the new blob
+ Log.Information("Writing blob: " + blob.Name);
+ await destinationBlob.UploadFromByteArrayAsync(compressedBytes, 0, compressedBytes.Length);
+
+ // Set the blob headers
+ Log.Information("Configuring headers");
+ destinationBlob.Properties.CacheControl = cacheControlHeader;
+ destinationBlob.Properties.ContentType = blob.Properties.ContentType;
+ destinationBlob.Properties.ContentEncoding = "gzip";
+ await destinationBlob.SetPropertiesAsync();
+ }
}
- public static void SetWildcardCorsOnBlobService(this CloudStorageAccount storageAccount)
+ public static async Task SetWildcardCorsOnBlobService(this CloudStorageAccount storageAccount)
{
- storageAccount.SetCORSPropertiesOnBlobService(cors =>
+ await storageAccount.SetCORSPropertiesOnBlobService(cors =>
{
var wildcardRule = new CorsRule() { AllowedMethods = CorsHttpMethods.Get, AllowedOrigins = { "*" } };
cors.CorsRules.Clear();
@@ -115,20 +132,20 @@ public static void SetWildcardCorsOnBlobService(this CloudStorageAccount storage
});
}
- public static void SetCORSPropertiesOnBlobService(this CloudStorageAccount storageAccount,
+ public static async Task SetCORSPropertiesOnBlobService(this CloudStorageAccount storageAccount,
Func alterCorsRules)
{
- Trace.TraceInformation("Configuring CORS.");
+ Log.Information("Configuring CORS.");
if (storageAccount == null || alterCorsRules == null) throw new ArgumentNullException();
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
- ServiceProperties serviceProperties = blobClient.GetServiceProperties();
+ ServiceProperties serviceProperties = await blobClient.GetServicePropertiesAsync();
serviceProperties.Cors = alterCorsRules(serviceProperties.Cors) ?? new CorsProperties();
- blobClient.SetServiceProperties(serviceProperties);
+ await blobClient.SetServicePropertiesAsync(serviceProperties);
}
}
}
diff --git a/packages.config b/packages.config
deleted file mode 100644
index 95c3a82..0000000
--- a/packages.config
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file