feat: support to retrieve reports in async-mode (#16)

Author: marcuzzuu

.github/workflows/ci.yaml

@@ -1,10 +1,10 @@
 name: CI
 
 on:
-  push:
-    branches:
-    - master
   pull_request:
+    branches:
+      - master
+  push:
     branches:
     - master
 
@@ -22,18 +22,20 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v2
 
-    - name: Cache modules
-      uses: actions/cache@v1
+    - name: Cache Go modules
+      uses: actions/cache@v3
       with:
-        path: ~/go/pkg/mod
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
 
     - name: Get dependencies
       run: |
         go get -v -t -d ./...
-        go get github.com/onsi/ginkgo/ginkgo
+        go install github.com/onsi/ginkgo/ginkgo@v1.16.5
 
     - name: Build
       run: go build ./...

.gitignore

@@ -14,3 +14,5 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+.idea

cmd/harbor-scanner-sysdig-secure/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -27,12 +28,19 @@ func main() {
 		os.Exit(1)
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	log.SetOutput(os.Stdout)
 	log.SetLevel(log.TraceLevel)
 	log.Info("Starting harbor-scanner-sysdig-secure")
 
-	apiHandler := v1.NewAPIHandler(getAdapter(), log.StandardLogger())
+	adapter := getAdapter()
+	if viper.GetBool("async_mode") {
+		adapter = scanner.NewAsyncAdapter(ctx, adapter, log.StandardLogger(), scanner.DefaultAsyncAdapterRefreshRate)
+	}
 
+	apiHandler := v1.NewAPIHandler(adapter, log.StandardLogger())
 	apiServer := api.NewServer(apiHandler)
 
 	log.Fatal(apiServer.ListenAndServe())
@@ -45,6 +53,7 @@ func configure() error {
 	pflag.String("secure_url", "https://secure.sysdig.com", "Sysdig Secure URL Endpoint")
 	pflag.Bool("verify_ssl", true, "Verify SSL when connecting to Sysdig Secure URL Endpoint")
 	pflag.Bool("inline_scanning", false, "Use Inline Scanning Adapter")
+	pflag.Bool("async_mode", false, "Use Async-Mode to perform reports retrieval")
 	pflag.String("namespace_name", "", "Namespace where inline scanning jobs are spawned")
 	pflag.String("secret_name", "", "Secret which keeps the inline scanning secrets ")
 	pflag.String("inline_scanning_extra_params", "", "Extra parameters to provide to inline-scanner")

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/golang/mock v1.4.3
 	github.com/gorilla/handlers v1.4.2
 	github.com/gorilla/mux v1.7.4
-	github.com/onsi/ginkgo v1.16.4
+	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.10.1
 	github.com/sirupsen/logrus v1.5.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -224,8 +224,8 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=

pkg/harbor/model.go

@@ -11,6 +11,8 @@ const (
 	ScanAdapterErrorMimeType           = "application/vnd.scanner.adapter.error+json; version=1.0"
 )
 
+type ScanRequestID string
+
 type Scanner struct {
 	Name    string `json:"name,omitempty"`
 	Vendor  string `json:"vendor,omitempty"`
@@ -53,7 +55,7 @@ type ErrorResponse struct {
 }
 
 type ScanResponse struct {
-	ID string `json:"id"`
+	ID ScanRequestID `json:"id"`
 }
 
 type VulnerabilityReport struct {

pkg/http/api/v1/handler.go

@@ -14,6 +14,10 @@ import (
 	"github.com/sysdiglabs/harbor-scanner-sysdig-secure/pkg/scanner"
 )
 
+const (
+	DefaultRefreshTimeInSeconds = 60
+)
+
 type requestHandler struct {
 	adapter scanner.Adapter
 	logger  Logger
@@ -91,27 +95,27 @@ func (h *requestHandler) scan(res http.ResponseWriter, req *http.Request) {
 func (h *requestHandler) getReport(res http.ResponseWriter, req *http.Request) {
 	vars := mux.Vars(req)
 
-	vulnerabilityReport, err := h.adapter.GetVulnerabilityReport(vars["scan_request_id"])
+	vulnerabilityReport, err := h.adapter.GetVulnerabilityReport(harbor.ScanRequestID(vars["scan_request_id"]))
 	if err != nil {
 		h.logRequestError(req, err)
 		switch err {
 		case scanner.ErrScanRequestIDNotFound:
 			res.WriteHeader(http.StatusNotFound)
-			json.NewEncoder(res).Encode(errorResponseFromError(err))
-		case scanner.ErrVulnerabiltyReportNotReady:
-			res.Header().Set("Refresh-After", "120")
+			_ = json.NewEncoder(res).Encode(errorResponseFromError(err))
+		case scanner.ErrVulnerabilityReportNotReady:
+			res.Header().Set("Refresh-After", fmt.Sprintf("%d", DefaultRefreshTimeInSeconds))
 			res.Header().Set("Location", req.URL.String())
 			res.WriteHeader(http.StatusFound)
 		default:
 			res.Header().Set("Content-Type", harbor.ScanAdapterErrorMimeType)
 			res.WriteHeader(http.StatusInternalServerError)
-			json.NewEncoder(res).Encode(errorResponseFromError(err))
+			_ = json.NewEncoder(res).Encode(errorResponseFromError(err))
 		}
 		return
 	}
 
 	res.Header().Set("Content-Type", harbor.ScanReportMimeType)
-	json.NewEncoder(res).Encode(vulnerabilityReport)
+	_ = json.NewEncoder(res).Encode(vulnerabilityReport)
 }
 
 func (h *requestHandler) logRequestError(req *http.Request, err error) {

pkg/http/api/v1/handler_test.go

@@ -3,11 +3,13 @@ package v1_test
 import (
 	"encoding/json"
 	"errors"
-	log "github.com/sirupsen/logrus"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -192,25 +194,28 @@ var _ = Describe("Harbor Scanner Sysdig Secure API Adapter", func() {
 	})
 
 	Context("GET /api/v1/scan/{scan_request_id}/report", func() {
+		reqID := harbor.ScanRequestID("scan-request-id")
+		reqPath := fmt.Sprintf("/api/v1/scan/%s/report", reqID)
+
 		It("returns OK", func() {
-			adapter.EXPECT().GetVulnerabilityReport("scan-request-id").Return(vulnerabilityReport(), nil)
+			adapter.EXPECT().GetVulnerabilityReport(reqID).Return(vulnerabilityReport(), nil)
 
-			response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+			response := doGetRequest(handler, reqPath)
 
 			Expect(response.StatusCode).To(Equal(http.StatusOK))
 		})
 
 		It("returns scanner.adapter.vuln.report.harbor Mime Type", func() {
-			adapter.EXPECT().GetVulnerabilityReport("scan-request-id").Return(vulnerabilityReport(), nil)
+			adapter.EXPECT().GetVulnerabilityReport(reqID).Return(vulnerabilityReport(), nil)
 
-			response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+			response := doGetRequest(handler, reqPath)
 
 			Expect(response.Header.Get("Content-Type")).To(Equal("application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"))
 		})
 
 		It("returns a valid scanner.vuln.report.harbor as JSON", func() {
-			adapter.EXPECT().GetVulnerabilityReport("scan-request-id").Return(vulnerabilityReport(), nil)
-			response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+			adapter.EXPECT().GetVulnerabilityReport(reqID).Return(vulnerabilityReport(), nil)
+			response := doGetRequest(handler, reqPath)
 
 			var result harbor.VulnerabilityReport
 			json.NewDecoder(response.Body).Decode(&result)
@@ -220,59 +225,59 @@ var _ = Describe("Harbor Scanner Sysdig Secure API Adapter", func() {
 
 		Context("when scan_request_id doesn't exist", func() {
 			BeforeEach(func() {
-				adapter.EXPECT().GetVulnerabilityReport("scan-request-id").Return(vulnerabilityReport(), scanner.ErrScanRequestIDNotFound)
+				adapter.EXPECT().GetVulnerabilityReport(reqID).Return(vulnerabilityReport(), scanner.ErrScanRequestIDNotFound)
 			})
 
 			It("returns NOT_FOUND", func() {
-				response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+				response := doGetRequest(handler, reqPath)
 
 				Expect(response.StatusCode).To(Equal(http.StatusNotFound))
 			})
 		})
 
 		Context("when image is still being scanned", func() {
 			BeforeEach(func() {
-				adapter.EXPECT().GetVulnerabilityReport("scan-request-id").Return(vulnerabilityReport(), scanner.ErrVulnerabiltyReportNotReady)
+				adapter.EXPECT().GetVulnerabilityReport(reqID).Return(vulnerabilityReport(), scanner.ErrVulnerabilityReportNotReady)
 			})
 
 			It("returns FOUND", func() {
-				response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+				response := doGetRequest(handler, reqPath)
 
 				Expect(response.StatusCode).To(Equal(http.StatusFound))
 			})
 
 			It("returns the interval after request should be retried", func() {
-				response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+				response := doGetRequest(handler, reqPath)
 
-				Expect(response.Header.Get("Refresh-After")).To(Equal("120"))
+				Expect(response.Header.Get("Refresh-After")).To(Equal(fmt.Sprintf("%d", v1.DefaultRefreshTimeInSeconds)))
 			})
 
 			It("returns the Location header with the URL to check", func() {
-				response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+				response := doGetRequest(handler, reqPath)
 
-				Expect(response.Header.Get("Location")).To(Equal("/api/v1/scan/scan-request-id/report"))
+				Expect(response.Header.Get("Location")).To(Equal(reqPath))
 			})
 		})
 
 		Context("when other unexpected errors happen", func() {
 			BeforeEach(func() {
-				adapter.EXPECT().GetVulnerabilityReport("scan-request-id").Return(vulnerabilityReport(), ErrUnexpected)
+				adapter.EXPECT().GetVulnerabilityReport(reqID).Return(vulnerabilityReport(), ErrUnexpected)
 			})
 
 			It("returns INTERNAL_SERVER_ERROR", func() {
-				response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+				response := doGetRequest(handler, reqPath)
 
 				Expect(response.StatusCode).To(Equal(http.StatusInternalServerError))
 			})
 
 			It("returns scanner.adapter.error mime type", func() {
-				response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+				response := doGetRequest(handler, reqPath)
 
 				Expect(response.Header.Get("Content-Type")).To(Equal("application/vnd.scanner.adapter.error+json; version=1.0"))
 			})
 
 			It("returns a the error encoded as JSON", func() {
-				response := doGetRequest(handler, "/api/v1/scan/scan-request-id/report")
+				response := doGetRequest(handler, reqPath)
 
 				var result harbor.ErrorResponse
 				json.NewDecoder(response.Body).Decode(&result)

pkg/scanner/adapter.go

@@ -7,13 +7,13 @@ import (
 )
 
 var (
-	ErrScanRequestIDNotFound      = errors.New("scanRequestID cannot be found")
-	ErrVulnerabiltyReportNotReady = errors.New("image is being scanned and report is still not ready")
+	ErrScanRequestIDNotFound       = errors.New("scanRequestID cannot be found")
+	ErrVulnerabilityReportNotReady = errors.New("image is being scanned and report is still not ready")
 )
 
 //go:generate mockgen -source=$GOFILE -destination=./mocks/${GOFILE} -package=mocks
 type Adapter interface {
 	GetMetadata() (harbor.ScannerAdapterMetadata, error)
 	Scan(req harbor.ScanRequest) (harbor.ScanResponse, error)
-	GetVulnerabilityReport(scanResponseID string) (harbor.VulnerabilityReport, error)
+	GetVulnerabilityReport(scanResponseID harbor.ScanRequestID) (harbor.VulnerabilityReport, error)
 }