added importer to complete migration from LiGet

Author: tomzo

CHANGELOG.md

@@ -4,6 +4,7 @@
  * Added compatibility mode with LiGet to keep the same endpoints
  * V2 includes dependencies in package query results
  * Switch production base image to slim stretch
+ * added importer to complete migration from LiGet
 
 ### 0.1.0 (2018-Nov-09)
 

docker-compose-liget-compat.yml

@@ -15,6 +15,7 @@ services:
     image: ${AIT_DOCKER_IMAGE_NAME}:${AIT_DOCKER_IMAGE_TAG}
     volumes:
       - ${IDE_WORK}/e2e/liget-compat/data/:/var/baget
+      - ${IDE_WORK}/e2e/input:/data/simple #simulates LiGet's storage
     env_file:
       - ${ENV_FILE}
     mem_limit: 550m

docker-scripts/run.sh

@@ -48,6 +48,10 @@ chown $NEWUID:$NEWGID -R /home/baget
 # Start server
 ###########################################################################
 
+if [ ! -z ${BAGET_IMPORT_ON_BOOT+x} ]; then
+  sudo -u baget -E -H dotnet /app/BaGet.dll import --path ${BAGET_IMPORT_ON_BOOT}
+fi
+
 cd /app
 if [[ $NEWGID != 0 ]]; then
   exec sudo -u baget -E -H dotnet /app/BaGet.dll

e2e/liget-compat/run.sh

@@ -16,8 +16,8 @@ cat << EOF > /home/ide/.nuget/NuGet/NuGet.Config
 </configuration>
 EOF
 
-echo "Sleeping 4s to wait for server to be ready"
-sleep 4
+echo "Sleeping 10s to wait for server to be ready"
+sleep 10
 
 COMPAT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 E2E_DIR="$COMPAT_DIR/.."
@@ -37,9 +37,8 @@ EOF
 
 mono $E2E_DIR/.paket/paket.bootstrapper.exe
 
-# BaGet would have all private packages imported already
+# BaGet running with BAGET_IMPORT_ON_BOOT should have all private packages imported already
 cd $E2E_DIR/input
-dotnet nuget push baget-two/bin/Debug/baget-two.2.1.0.nupkg --source http://nuget:9090/v3/index.json --api-key NUGET-SERVER-API-KEY
 
 cd $COMPAT_DIR/paket
 rm -rf paket-files

readme.md

@@ -114,6 +114,21 @@ This will enable following behavior:
  - `/api/cache/v3/index.json` returns same content as original BaGet's `/cache/v3/index.json`.
  - `/api/v2/*` returns **V2** resources, same as `/v2/*`
 
+### Importing packages
+
+To make transition from LiGet or any other server which keeps `.nupkg` files in a directory,
+there is an `import` command:
+```
+dotnet BaGet.dll import --path dir
+```
+In the docker image you can setup environment variable - `BAGET_IMPORT_ON_BOOT=/data/simple`
+which will cause baget to first search for `nupkg` files in `$BAGET_IMPORT_ON_BOOT`, before starting server.
+Packages which were already added are skipped.
+Setting `BAGET_IMPORT_ON_BOOT=/data/simple` is sufficient for migration from LiGet.
+
+*Note: you only need to set this variable once to perform initial migration.
+You should unset it in later deployments to avoid uncessary scanning.*
+
 # Development
 
 We rely heavily on docker to create reproducible development environment.

src/BaGet.Core/Services/PackageImporter.cs

@@ -0,0 +1,28 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace BaGet.Core.Services
+{
+    public class PackageImporter
+    {
+        private readonly IIndexingService _indexingService;
+
+        public PackageImporter(IIndexingService indexingService) {
+            this._indexingService = indexingService;
+        }
+
+        public async Task ImportAsync(string pkgDirectory, TextWriter output)
+        {
+            string[] files = Directory.GetFiles(pkgDirectory, "*.nupkg", SearchOption.AllDirectories);
+            foreach (string file in files)
+            {
+                output.Write("Importing package {0} ", file);
+                using(var uploadStream = File.OpenRead(file)) {
+                    var result = await _indexingService.IndexAsync(uploadStream);
+                    output.WriteLine(result);
+                }
+            }            
+        }
+    }
+}

src/BaGet/Extensions/ServiceCollectionExtensions.cs

@@ -55,6 +55,8 @@ public static IServiceCollection ConfigureBaGet(
             services.ConfigureSearchProviders();
             services.ConfigureAuthenticationProviders();
 
+            services.AddTransient<PackageImporter, PackageImporter>();
+
             return services;
         }
 

src/BaGet/Program.cs

@@ -1,12 +1,18 @@
-using BaGet.Core.Mirror;
+using System;
+using BaGet.Core.Configuration;
+using BaGet.Core.Entities;
+using BaGet.Core.Mirror;
+using BaGet.Core.Services;
 using BaGet.Extensions;
 using Gelf.Extensions.Logging;
 using McMaster.Extensions.CommandLineUtils;
 using Microsoft.AspNetCore;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 
 namespace BaGet
 {
@@ -35,6 +41,36 @@ await provider
                             .ImportAsync();
                     });
                 });
+
+                var optionImportPath = import.Option("-p|--path <directory>", "Directory with .nupkg files at any depth", CommandOptionType.SingleValue);
+                import.OnExecute(async () => {
+                    var importDirectory = optionImportPath.HasValue()
+                        ? optionImportPath.Value()
+                        : ".";
+
+                    Console.WriteLine($"Importing packages from {importDirectory}");
+                    var provider = CreateHostBuilder(args).Build().Services;
+
+                    var scopeFactory = provider.GetRequiredService<IServiceScopeFactory>();
+                    // Run migrations if enabled
+                    var databaseOptions = provider.GetRequiredService<IOptions<BaGetOptions>>()
+                            .Value
+                            .Database;
+                    if(databaseOptions.RunMigrations) {
+                        using (var scope = scopeFactory.CreateScope())
+                        {
+                            scope.ServiceProvider
+                                .GetRequiredService<IContext>()
+                                .Database
+                                .Migrate();
+                        }
+                    }
+
+                    await provider
+                        .GetRequiredService<PackageImporter>()
+                        .ImportAsync(importDirectory, Console.Out);
+                    return 0;
+                });
             });
 
             app.OnExecute(() =>

tasks.sh

@@ -80,14 +80,15 @@ case "${command}" in
     rm -rf e2e/liget-compat/data/packages/*
     rm -rf e2e/liget-compat/data/cache/*
     export LiGetCompat__Enabled=true
+    export BAGET_IMPORT_ON_BOOT=/data/simple
     ide --idefile Idefile.liget-compat "e2e/liget-compat/run.sh"
     ;;
   all)
     ide "./build.sh --target All"
     ./tasks.sh build_docker
     ./tasks.sh test_docker
     ./tasks.sh liget_compat_docker
-    ./tasks.sh stress_docker    
+    ./tasks.sh stress_docker
     ;;
   prepare_code_release)
     version=$2

tests/BaGet.Tests/Services/PackageImporterIntegrationTest.cs

@@ -0,0 +1,43 @@
+using System;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+using BaGet.Core.Services;
+using BaGet.Tests.Support;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace BaGet.Tests.Services
+{
+    [Trait("Category", "integration")] // because it uses external nupkg files
+    public class PackageImporterIntegrationTest
+    {
+        public ITestOutputHelper Helper { get; private set; }
+
+        private TestServer server;
+
+        public PackageImporterIntegrationTest(ITestOutputHelper helper)
+        {
+            Helper = helper ?? throw new ArgumentNullException(nameof(helper));
+            server = TestServerBuilder.Create().TraceToTestOutputHelper(Helper, LogLevel.Error).Build();
+        }
+
+        [Fact]
+        public async Task ImportExamplePackages()
+        {
+            var importer = server.Host.Services.GetRequiredService<PackageImporter>();
+            using(var ms = new MemoryStream()) {
+                var writer = new StreamWriter(ms, Encoding.UTF8);
+                await importer.ImportAsync(TestResources.GetE2eInputDirectory(), writer);
+                await writer.FlushAsync();
+                string text = Encoding.UTF8.GetString(ms.ToArray());
+                Assert.Contains("baget-test1.1.0.0.nupkg Success", text);
+                Assert.Contains("baget-two.1.0.0.nupkg Success", text);
+                Assert.Contains("baget-two.2.1.0.nupkg Success", text);
+            }
+        }
+    }
+}