Merge pull request #8 from danielpacak/issue_7_async_processing

feat: Process scan requests asynchronously

Author: steven-zou

.travis.yml

@@ -17,6 +17,7 @@ install:
 script:
 - make lint
 - make test
+- make test-integration
 - curl -sL https://git.io/goreleaser | bash -s -- --snapshot --skip-publish --rm-dist
 after_success:
 - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -19,5 +19,8 @@ lint:
 test: $(SOURCES)
 	GO111MODULE=on go test -v -short -race -timeout 30s -coverprofile=coverage.txt -covermode=atomic ./...
 
+test-integration: build
+	GO111MODULE=on go test -v -tags=integration ./test/integration/...
+
 container: build
 	docker build --no-cache -t $(IMAGE) .

README.md

@@ -31,7 +31,13 @@ Configuration of the adapter is done via environment variables at startup.
 | `SCANNER_TLS_CLIENTCAS` | | An array of absolute paths to x509 CA files that will be added to host's root CA set. |
 | `SCANNER_API_SERVER_READ_TIMEOUT`  | `15s` | The maximum duration for reading the entire request, including the body. |
 | `SCANNER_API_SERVER_WRITE_TIMEOUT` | `15s` | The maximum duration before timing out writes of the response. |
+| `SCANNER_API_SERVER_IDLE_TIMEOUT`  | `60s` | The maximum amount of time to wait for the next request when keep-alives are enabled. |
 | `SCANNER_CLAIR_URL`                | `http://harbor-harbor-clair:6060` | Clair URL |
+| `SCANNER_STORE_REDIS_URL`       | `redis://harbor-harbor-redis:6379` | Redis server URI for a redis store. |
+| `SCANNER_STORE_REDIS_NAMESPACE` | `harbor.scanner.clair:store` | A namespace for keys in a redis store. |
+| `SCANNER_STORE_REDIS_POOL_MAX_ACTIVE` | `5`  | The max number of connections allocated by the pool for a redis store. |
+| `SCANNER_STORE_REDIS_POOL_MAX_IDLE`   | `5`  | The max number of idle connections in the pool for a redis store. |
+| `SCANNER_STORE_REDIS_SCAN_JOB_TTL`    | `1h` | The time to live for persisting scan jobs and associated scan reports. |
 
 ## Deploy to minikube
 

cmd/harbor-scanner-clair/main.go

@@ -2,14 +2,15 @@ package main
 
 import (
 	"context"
+	"github.com/goharbor/harbor-scanner-clair/pkg/clair"
 	"github.com/goharbor/harbor-scanner-clair/pkg/etc"
 	"github.com/goharbor/harbor-scanner-clair/pkg/http/api"
 	"github.com/goharbor/harbor-scanner-clair/pkg/http/api/v1"
-	"github.com/goharbor/harbor-scanner-clair/pkg/model"
+	"github.com/goharbor/harbor-scanner-clair/pkg/persistence/redis"
 	"github.com/goharbor/harbor-scanner-clair/pkg/registry"
-	"github.com/goharbor/harbor-scanner-clair/pkg/scanner/clair"
+	"github.com/goharbor/harbor-scanner-clair/pkg/scanner"
+	"github.com/goharbor/harbor-scanner-clair/pkg/work"
 	log "github.com/sirupsen/logrus"
-	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
@@ -28,55 +29,46 @@ func main() {
 	log.SetReportCaller(false)
 	log.SetFormatter(&log.JSONFormatter{})
 
+	config, err := etc.GetConfig()
+	if err != nil {
+		log.Fatalf("Error: %v", err)
+	}
+
+	store := redis.NewStore(config.Store)
+
+	workPool := work.New()
+
 	log.WithFields(log.Fields{
 		"version":  version,
 		"commit":   commit,
 		"built_at": date,
 	}).Info("Starting harbor-scanner-clair")
 
-	clairConfig, err := etc.GetClairConfig()
-	if err != nil {
-		log.Fatalf("Error: %v", err)
-	}
+	registryClientFactory := registry.NewClientFactory(config.TLS)
+	clairClient := clair.NewClient(config.TLS, config.Clair)
+	adapter := scanner.NewAdapter(registryClientFactory, clairClient, scanner.NewTransformer())
 
-	tlsConfig, err := etc.GetTLSConfig()
-	if err != nil {
-		log.Fatalf("Error: %v", err)
-	}
-
-	registryClientFactory := registry.NewClientFactory(tlsConfig)
-	clairClient := clair.NewClient(tlsConfig, clairConfig)
-	scanner := clair.NewScanner(registryClientFactory, clairClient, model.NewTransformer())
-
-	apiConfig, err := etc.GetAPIConfig()
-	if err != nil {
-		log.Fatalf("Error: %v", err)
-	}
+	enqueuer := scanner.NewEnqueuer(workPool, adapter, store)
 
-	apiHandler := v1.NewAPIHandler(scanner)
+	apiHandler := v1.NewAPIHandler(enqueuer, store)
 
-	server := api.NewServer(apiConfig, apiHandler)
+	apiServer := api.NewServer(config.API, apiHandler)
 
 	shutdownComplete := make(chan struct{})
 	go func() {
 		sigint := make(chan os.Signal, 1)
-		signal.Notify(sigint, os.Interrupt, syscall.SIGTERM)
+		signal.Notify(sigint, syscall.SIGINT, syscall.SIGTERM)
 		captured := <-sigint
 		log.WithField("signal", captured.String()).Debug("Trapped os signal")
 
-		log.Debug("API server shutdown started")
-		if err := server.Shutdown(context.Background()); err != nil {
-			log.WithError(err).Error("Error while shutting down server")
-		}
-		log.Debug("API server shutdown completed")
+		apiServer.Shutdown(context.Background())
+		workPool.Shutdown()
+
 		close(shutdownComplete)
 	}()
 
-	go func() {
-		if err = server.ListenAndServe(); err != http.ErrServerClosed {
-			log.Fatalf("Error: %v", err)
-		}
-		log.Debug("ListenAndServe returned")
-	}()
+	workPool.Start()
+	apiServer.ListenAndServe()
+
 	<-shutdownComplete
 }

go.mod

@@ -3,11 +3,15 @@ module github.com/goharbor/harbor-scanner-clair
 go 1.13
 
 require (
+	github.com/Microsoft/hcsshim v0.8.6 // indirect
 	github.com/caarlos0/env/v6 v6.0.0
 	github.com/docker/distribution v2.7.1+incompatible
+	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.2
-	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/prometheus/client_golang v1.2.1
 	github.com/sirupsen/logrus v1.4.2
 	github.com/stretchr/testify v1.3.0
+	github.com/testcontainers/testcontainers-go v0.0.8
+	golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898
 )