Download browser (#60)

* Downloader feature complete

Author: kblok

lib/PuppeteerSharp.Tests/Puppeteer/DownloaderTests.cs

@@ -0,0 +1,26 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace PuppeteerSharp.Tests.Puppeteer
+{
+    public class DownloaderTests
+    {
+        [Fact(Skip = "Long run")]
+        public async Task ShouldDownloadChromium()
+        {
+            var downloadsFolder = Path.Combine(Directory.GetCurrentDirectory(), ".test-chromium");
+            var dirInfo = new DirectoryInfo(downloadsFolder);
+            var downloader = new Downloader(downloadsFolder);
+
+            if (dirInfo.Exists)
+            {
+                dirInfo.Delete(true);
+            }
+
+            await downloader.DownloadRevisionAsync(PuppeteerLaunchTests.ChromiumRevision);
+            Assert.True(new FileInfo(downloader.GetExecutablePath(PuppeteerLaunchTests.ChromiumRevision)).Exists);
+        }
+    }
+}

lib/PuppeteerSharp.Tests/Puppeteer/PuppeteerLaunch.cs renamed to lib/PuppeteerSharp.Tests/Puppeteer/PuppeteerLaunchTests.cs

@@ -8,33 +8,32 @@
 
 namespace PuppeteerSharp.Tests.Puppeteer
 {
-    public class PuppeteerLaunch
+    public class PuppeteerLaunchTests
     {
         private const int HttpsPort = 8908;
         private const string HttpsPrefix = "https://localhost:8908";
+        public const int ChromiumRevision = 526987;
 
         private Dictionary<string, object> _defaultBrowserOptions = new Dictionary<string, object>()
         {
-            { "executablePath", Environment.GetEnvironmentVariable("CHROME") },
             { "slowMo", Convert.ToInt32(Environment.GetEnvironmentVariable("SLOW_MO") ?? "0") },
             { "headless", Convert.ToBoolean(Environment.GetEnvironmentVariable("HEADLESS") ?? "true") },
             { "args", new[] { "--no-sandbox" }},
             { "timeout", 0}
         };
 
+        public PuppeteerLaunchTests()
+        {
+            Downloader.CreateDefault().DownloadRevisionAsync(ChromiumRevision).GetAwaiter().GetResult();
+        }
+
         [Fact]
-        public async Task should_support_ignoreHTTPSErrors_option()
+        public async Task ShouldSupportIgnoreHTTPSErrorsOption()
         {
             var options = _defaultBrowserOptions.Clone();
             options.Add("ignoreHTTPSErrors", true);
 
-            var puppeteerOptions = new PuppeteerOptions()
-            {
-
-                ChromiumRevision = "526987"
-            };
-
-            var browser = await PuppeteerSharp.Puppeteer.LaunchAsync(options, puppeteerOptions);
+            var browser = await PuppeteerSharp.Puppeteer.LaunchAsync(options, ChromiumRevision);
             var page = await browser.NewPageAsync();
 
             var response = await page.GoToAsync(HttpsPrefix + "/empty.html");

lib/PuppeteerSharp/Downloader.cs

@@ -1,14 +1,24 @@
 using System;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
-using System.Reflection;
+using System.IO.Compression;
+using System.Net;
 using System.Runtime.InteropServices;
+using System.Threading.Tasks;
 
 namespace PuppeteerSharp
 {
     public class Downloader
     {
-        private string _downloadsFolder;
+        private readonly string _downloadsFolder;
         private const string DefaultDownloadHost = "https://storage.googleapis.com";
+        private static readonly Dictionary<Platform, string> _downloadUrls = new Dictionary<Platform, string> {
+            {Platform.Linux, "{0}/chromium-browser-snapshots/Linux_x64/{1}/chrome-linux.zip"},
+            {Platform.MacOS, "{0}/chromium-browser-snapshots/Mac/{1}/chrome-mac.zip"},
+            {Platform.Win32, "{0}/chromium-browser-snapshots/Win/{1}/chrome-win32.zip"},
+            {Platform.Win64, "{0}/chromium-browser-snapshots/Win_x64/{1}/chrome-win32.zip"}
+        };
         private string _downloadHost;
 
         public Downloader(string downloadsFolder)
@@ -17,24 +27,29 @@ public Downloader(string downloadsFolder)
             _downloadHost = DefaultDownloadHost;
         }
 
+        #region Public Methods
+
         public static Downloader CreateDefault()
         {
             var downloadsFolder = Path.Combine(Directory.GetCurrentDirectory(), ".local-chromium");
             return new Downloader(downloadsFolder);
         }
 
-        internal static Platform CurrentPlatform()
+        internal static Platform CurrentPlatform
         {
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
-                return Platform.MacOS;
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-                return Platform.Linux;
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-                return IntPtr.Size == 8 ? Platform.Win64 : Platform.Win32;
-            return Platform.Unknown;
+            get
+            {
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+                    return Platform.MacOS;
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                    return Platform.Linux;
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                    return IntPtr.Size == 8 ? Platform.Win64 : Platform.Win32;
+                return Platform.Unknown;
+            }
         }
 
-        internal RevisionInfo RevisionInfo(Platform platform, string revision)
+        internal RevisionInfo RevisionInfo(Platform platform, int revision)
         {
             var result = new RevisionInfo
             {
@@ -45,7 +60,47 @@ internal RevisionInfo RevisionInfo(Platform platform, string revision)
             return result;
         }
 
-        private static string GetExecutablePath(Platform platform, string folderPath)
+        public async Task DownloadRevisionAsync(int revision)
+        {
+            var url = string.Format(_downloadUrls[CurrentPlatform], _downloadHost, revision);
+            var zipPath = Path.Combine(_downloadsFolder, $"download-{CurrentPlatform.ToString()}-{revision}.zip");
+            var folderPath = GetFolderPath(CurrentPlatform, revision);
+
+            if (new DirectoryInfo(folderPath).Exists)
+            {
+                return;
+            }
+
+            var downloadFolder = new DirectoryInfo(_downloadsFolder);
+            if (!downloadFolder.Exists)
+            {
+                downloadFolder.Create();
+            }
+
+            await new WebClient().DownloadFileTaskAsync(new Uri(url), zipPath);
+
+            if (CurrentPlatform == Platform.MacOS)
+            {
+                //ZipFile and many others unzip libraries have issues extracting .app files
+                //Until we have a clear solution we'll call the native unzip tool
+                //https://github.com/dotnet/corefx/issues/15516
+                NativeExtractToDirectory(zipPath, folderPath);
+            }
+            else
+            {
+                ZipFile.ExtractToDirectory(zipPath, folderPath);
+            }
+
+            new FileInfo(zipPath).Delete();
+
+        }
+
+        public string GetExecutablePath(int revision)
+        {
+            return GetExecutablePath(CurrentPlatform, GetFolderPath(CurrentPlatform, revision));
+        }
+
+        public static string GetExecutablePath(Platform platform, string folderPath)
         {
             switch (platform)
             {
@@ -62,9 +117,24 @@ private static string GetExecutablePath(Platform platform, string folderPath)
             }
         }
 
-        private string GetFolderPath(Platform platform, string revision)
+        #endregion
+
+        #region Private Methods
+
+        private string GetFolderPath(Platform platform, int revision)
         {
             return Path.Combine(_downloadsFolder, $"{platform.ToString()}-{revision}");
         }
+
+        private void NativeExtractToDirectory(string zipPath, string folderPath)
+        {
+            var process = new Process();
+            process.StartInfo.FileName = "unzip";
+            process.StartInfo.Arguments = $"{zipPath} -d {folderPath}";
+            process.Start();
+            process.WaitForExit();
+        }
+
+        #endregion
     }
 }

lib/PuppeteerSharp/Launcher.cs

@@ -46,7 +46,7 @@ public Launcher()
         {
         }
 
-        internal static async Task<Browser> LaunchAsync(Dictionary<string, object> options, PuppeteerOptions puppeteerOptions)
+        internal static async Task<Browser> LaunchAsync(Dictionary<string, object> options, int chromiumRevision)
         {
             var chromeArguments = new List<string>(_defaultArgs);
 
@@ -94,8 +94,7 @@ internal static async Task<Browser> LaunchAsync(Dictionary<string, object> optio
             if (string.IsNullOrEmpty(chromeExecutable))
             {
                 var downloader = Downloader.CreateDefault();
-                var revisionInfo = downloader.RevisionInfo(Downloader.CurrentPlatform(),
-                                                           puppeteerOptions.ChromiumRevision);
+                var revisionInfo = downloader.RevisionInfo(Downloader.CurrentPlatform, chromiumRevision);
                 chromeExecutable = revisionInfo.ExecutablePath;
             }
 

lib/PuppeteerSharp/NetworkManager.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics.Contracts;
 using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
@@ -135,12 +134,15 @@ private void OnLoadingFailed(MessageEventArgs e)
             {
                 var request = _requestIdToRequest[e.MessageData.requestId.ToString()];
 
-                request.Failure = e.MessageData.errorText;
+                request.Failure = e.MessageData.errorText.ToString();
                 request.CompleteTaskWrapper.SetResult(true);
                 _requestIdToRequest.Remove(request.RequestId);
-                _interceptionIdToRequest.Remove(request.InterceptionId);
-                _attemptedAuthentications.Remove(request.InterceptionId);
 
+                if (request.InterceptionId != null)
+                {
+                    _interceptionIdToRequest.Remove(request.InterceptionId);
+                    _attemptedAuthentications.Remove(request.InterceptionId);
+                }
                 RequestFailed(this, new RequestEventArgs()
                 {
                     Request = request
@@ -224,9 +226,6 @@ private async Task OnRequestInterceptedAsync(MessageEventArgs e)
 
             if (!string.IsNullOrEmpty(e.MessageData.redirectUrl))
             {
-                Contract.Ensures(_interceptionIdToRequest.ContainsKey(e.MessageData.interceptionId),
-                                 "INTERNAL ERROR: failed to find request for interception redirect.");
-
                 var request = _interceptionIdToRequest[e.MessageData.interceptionId];
 
                 HandleRequestRedirect(request, e.MessageData.responseStatusCode, e.MessageData.responseHeaders);

lib/PuppeteerSharp/Puppeteer.cs

@@ -10,9 +10,9 @@ public Puppeteer()
         {
         }
 
-        public static async Task<Browser> LaunchAsync(Dictionary<string, object> options, PuppeteerOptions puppeteerOptions) 
+        public static async Task<Browser> LaunchAsync(Dictionary<string, object> options, int chromiumRevision)
         {
-            return await Launcher.LaunchAsync(options, puppeteerOptions);
+            return await Launcher.LaunchAsync(options, chromiumRevision);
         }
     }
 }

lib/PuppeteerSharp/PuppeteerOptions.cs

@@ -5,7 +5,7 @@ namespace PuppeteerSharp
 {
     public struct RevisionInfo
     {
-        public string Revision { get; set; }
+        public int Revision { get; set; }
         public string FolderPath { get; set; }
         public string ExecutablePath { get; set; }
         public bool Downloaded => Directory.Exists(FolderPath);