Bug fixes, add simulation.

Author: stefangordon

BlobGZipUtility/BlobGZipUtility.csproj BlobGZipUtility/ASGE.csprojBlobGZipUtility/BlobGZipUtility.csproj renamed to BlobGZipUtility/ASGE.csproj

@@ -7,8 +7,8 @@
     <ProjectGuid>{84285BB3-2556-45F3-965F-4667855A6EB6}</ProjectGuid>
     <OutputType>Exe</OutputType>
     <AppDesignerFolder>Properties</AppDesignerFolder>
-    <RootNamespace>BlobGZipUtility</RootNamespace>
-    <AssemblyName>BlobGZipUtility</AssemblyName>
+    <RootNamespace>ASGE</RootNamespace>
+    <AssemblyName>ASGE</AssemblyName>
     <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>

BlobGZipUtility/BlobGZipUtility.sln BlobGZipUtility/ASGE.slnBlobGZipUtility/BlobGZipUtility.sln renamed to BlobGZipUtility/ASGE.sln

@@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 14
 VisualStudioVersion = 14.0.25123.0
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BlobGZipUtility", "BlobGZipUtility.csproj", "{84285BB3-2556-45F3-965F-4667855A6EB6}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ASGE", "ASGE.csproj", "{84285BB3-2556-45F3-965F-4667855A6EB6}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

BlobGZipUtility/Options.cs

@@ -6,12 +6,12 @@
 using CommandLine;
 using CommandLine.Text;
 
-namespace BlobGZipUtility
+namespace ASGE
 {
     class Options
     {
         [Option('a', "account", Required = false, MutuallyExclusiveSet = "Account and Key",
-          HelpText = "Storage account host. [sample.blob.core.windows.net]")]
+          HelpText = "Storage account host. [mystorage]")]
         public string StorageAccount { get; set; }
 
         [Option('k', "key", Required = false, MutuallyExclusiveSet = "Account and Key",
@@ -26,30 +26,41 @@ class Options
             HelpText = "Extensions to operate on. [.js, .css, .dat]")]
         public string[] Extensions { get; set; }
 
-        [Option('r', "replace", Required = false, 
+        [Option('r', "replace", Required = false, DefaultValue = false, 
             HelpText = "Replace existing files in-place.")]
         public bool Replace { get; set; }
 
+        [Option('s', "simulate", Required = false, DefaultValue = false,
+            HelpText = "Do everything except write to blob store.")]
+        public bool Simulate { get; set; }
+
         [Option('n', "newextension", Required = false, 
             HelpText = "Copy file with a new postfix. [.gz]")]
         public string NewExtension { get; set; }
 
-        [Option('f', "container", Required = false,
+        [Option('f', "container", Required = true,
             HelpText = "Container to search in.")]
         public string Container { get; set; }
 
+        [Option('x', "cacheage", Required = false,  DefaultValue = 2592000,
+            HelpText = "Duration for cache control max age header, in seconds.  Default 2592000 (30 days).")]
+        public int MaxAgeSeconds { get; set; }
+
+
         [HelpOption]
         public string GetUsage()
         {
             var help = new HelpText
             {
-                Heading = new HeadingInfo("<<app title>>", "<<app version>>"),
-                Copyright = new CopyrightInfo("<<app author>>", 2014),
+                Heading = new HeadingInfo("Azure Storage GZip Encoder", "1.0"),
+                Copyright = new CopyrightInfo("Stefan Gordon", 2016),                
                 AdditionalNewLineAfterOption = true,
                 AddDashesToOption = true
             };
-            help.AddPreOptionsLine("<<license details here.>>");
-            help.AddPreOptionsLine("Usage: app -p Someone");
+            help.AddPreOptionsLine("https://github.com/stefangordon/azure-storage-gzip-encoding\n");
+            help.AddPreOptionsLine("\nExample 1: asge --extensions .css .js --container assets --replace --connectionstring <conn string>");
+            help.AddPreOptionsLine("Example 2: asge --extensions .css .js --container assets --newextension .gz --account mystorage --key <storage key>");
+            help.AddPreOptionsLine("\nUse either connection string (-c) or account and key (-a and -k).");
             help.AddOptions(this);
             return help;
         }

BlobGZipUtility/Program.cs

@@ -9,7 +9,7 @@
 using Microsoft.WindowsAzure.Storage.Auth;
 using Microsoft.WindowsAzure.Storage.Blob;
 
-namespace BlobGZipUtility
+namespace ASGE
 {
     class Program
     {
@@ -18,6 +18,12 @@ static void Main(string[] args)
             var options = new Options();
             if (CommandLine.Parser.Default.ParseArguments(args, options))
             {
+                if (string.IsNullOrEmpty(options.NewExtension) && !options.Replace)
+                {
+                    Console.WriteLine("Must provide either -r (in-place replacement) or -n (new extension/postfix to append to compressed version).");
+                    return;
+                }
+
                 CloudStorageAccount storageAccount;
 
                 if (!string.IsNullOrEmpty(options.ConnectionString))
@@ -37,17 +43,14 @@ static void Main(string[] args)
                 CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
                 CloudBlobContainer blobContainer = blobClient.GetContainerReference(options.Container);
 
-                Utility.EnsureGzipFiles(blobContainer, options.Extensions, options.Replace, options.NewExtension, (int)TimeSpan.FromDays(30).TotalSeconds);
+                Utility.EnsureGzipFiles(blobContainer, options.Extensions, options.Replace, options.NewExtension, options.MaxAgeSeconds, options.Simulate);
 
             }
             else
             {
                 // Display the default usage information
                 //Console.WriteLine(options.GetUsage());
             }
-
-            Console.WriteLine("Press any key to exit.");
-            Console.ReadKey();
         }
     }
 }

BlobGZipUtility/Properties/AssemblyInfo.cs

@@ -5,11 +5,11 @@
 // General Information about an assembly is controlled through the following 
 // set of attributes. Change these attribute values to modify the information
 // associated with an assembly.
-[assembly: AssemblyTitle("BlobGZipUtility")]
+[assembly: AssemblyTitle("ASGE")]
 [assembly: AssemblyDescription("")]
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("")]
-[assembly: AssemblyProduct("BlobGZipUtility")]
+[assembly: AssemblyProduct("ASGE")]
 [assembly: AssemblyCopyright("Copyright ©  2016")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]

BlobGZipUtility/Utility.cs

@@ -8,31 +8,30 @@
 using System.Threading.Tasks;
 using Microsoft.WindowsAzure.Storage.Blob;
 
-namespace BlobGZipUtility
+namespace ASGE
 {
     class Utility
     {
 
 
-        public static void EnsureGzipFiles(CloudBlobContainer container, IEnumerable<string> extensions, bool inPlace, string newExtension, int cacheControlMaxAgeSeconds)
+        public static void EnsureGzipFiles(CloudBlobContainer container, IEnumerable<string> extensions, bool inPlace, string newExtension, int cacheControlMaxAgeSeconds, bool simulate)
         {
             Trace.TraceInformation("Enumerating files.");
 
             string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
 
-            var blobInfos = container.ListBlobs(null, true);
-            //Parallel.ForEach(blobInfos, (blobInfo) =>
-            foreach(var blobInfo in blobInfos)
+            var blobInfos = container.ListBlobs(null, true, BlobListingDetails.Metadata);
+
+            Parallel.ForEach(blobInfos, (blobInfo) =>            
             {
-                CloudBlob gzipBlob = null;
-                string blobUrl = blobInfo.Uri.ToString();
+                CloudBlob gzipBlob = null;                
                 CloudBlob blob = (CloudBlob)blobInfo;
 
                 // Only work with desired extensions
                 string extension = Path.GetExtension(blobInfo.Uri.LocalPath);
                 if (!extensions.Contains(extension, StringComparer.OrdinalIgnoreCase))
                 {
-                    continue;
+                    return;
                 }
 
                 // Check if it is already done
@@ -41,58 +40,68 @@ public static void EnsureGzipFiles(CloudBlobContainer container, IEnumerable<str
                     if (string.Equals(blob.Properties.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase))
                     {
                         Trace.TraceInformation("Skipping already compressed blob: " + blob.Name);
-                        continue;
+                        return;
                     }
                 }
                 else
                 {
-                    string gzipUrl = blobUrl + newExtension;
-                    gzipBlob = container.GetBlobReference(gzipUrl);
-
+                    string gzipUrl = blob.Name + newExtension;
+                    gzipBlob = container.GetBlockBlobReference(gzipUrl);
+                         
                     if (gzipBlob.Exists())
                     {
                         Trace.TraceInformation("Skipping already compressed blob: " + blob.Name);
-                        continue;
+                        return;
                     }
                 }
 
-                // create a gzip version of the file
+                // Compress blob contents
+                Trace.TraceInformation("Downloading blob: " + blob.Name);
+
+                byte[] compressedBytes;
+
                 using (MemoryStream memoryStream = new MemoryStream())
                 {
-                    // push the original blob into the gzip stream
                     using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress))
                     using (var blobStream = blob.OpenRead())
                     {
                         blobStream.CopyTo(gzipStream);
                     }
+                    
+                    compressedBytes = memoryStream.ToArray();
+                }
 
-                    // the gzipStream MUST be closed before its safe to read from the memory stream
-                    byte[] compressedBytes = memoryStream.ToArray();
-
-                    // Blob to write to 
-                    CloudBlockBlob destinationBlob;
+                // Blob to write to 
+                CloudBlockBlob destinationBlob;
 
-                    if (inPlace)
-                    {                        
-                        destinationBlob = (CloudBlockBlob)blob;
-                    }
-                    else
-                    {
-                        destinationBlob = (CloudBlockBlob)gzipBlob;                        
-                    }
+                if (inPlace)
+                {                        
+                    destinationBlob = (CloudBlockBlob)blob;
+                }
+                else
+                {
+                    destinationBlob = (CloudBlockBlob)gzipBlob;                        
+                }
 
+                if (simulate)
+                {
+                    Trace.TraceInformation("NOT writing blob, due to simulation: " + blob.Name);
+                }
+                else
+                { 
+                    // Upload the compressed bytes to the new blob
                     Trace.TraceInformation("Writing blob: " + blob.Name);
-
-                    // upload the compressed bytes to the new blob
                     destinationBlob.UploadFromByteArray(compressedBytes, 0, compressedBytes.Length);
-
-                    // set the blob headers
+                              
+                    // Set the blob headers
+                    Trace.TraceInformation("Configuring headers");
                     destinationBlob.Properties.CacheControl = cacheControlHeader;
                     destinationBlob.Properties.ContentType = blob.Properties.ContentType;
                     destinationBlob.Properties.ContentEncoding = "gzip";
                     destinationBlob.SetProperties();
                 }
-            }//);
+
+            });
         }
     }
 }