feat: Add proxy support and update to inline-scan V2 final version (#7)

airadier · web-flow · commit e7a301b05e9f · 2021-01-27T17:57:03.000+01:00
* feat: Add proxy support and update to inline-scan V2 final version

* Add --registry-skip-tls and --sysdig-skip-tls when verify_ssl is false

* fix: restore env after tests, or randomized execution might fail
diff --git a/cmd/harbor-scanner-sysdig-secure/main.go b/cmd/harbor-scanner-sysdig-secure/main.go
@@ -82,7 +82,8 @@ func getAdapter() scanner.Adapter {
 			clientset,
 			viper.GetString("secure_url"),
 			viper.GetString("namespace_name"),
-			viper.GetString("secret_name"))
+			viper.GetString("secret_name"),
+			viper.GetBool("verify_ssl"))
 	}
 
 	log.Info("Using backend-scanning adapter")
diff --git a/pkg/http/api/v1/handler.go b/pkg/http/api/v1/handler.go
@@ -16,7 +16,7 @@ import (
 
 type requestHandler struct {
 	adapter scanner.Adapter
-	logger Logger
+	logger  Logger
 }
 
 type Logger interface {
@@ -28,7 +28,7 @@ type Logger interface {
 func NewAPIHandler(adapter scanner.Adapter, logger Logger) http.Handler {
 	handler := requestHandler{
 		adapter: adapter,
-		logger: logger,
+		logger:  logger,
 	}
 
 	router := mux.NewRouter()
diff --git a/pkg/scanner/inline_adapter.go b/pkg/scanner/inline_adapter.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
+	"os"
 	"time"
 
 	batchv1 "k8s.io/api/batch/v1"
@@ -22,16 +23,18 @@ type inlineAdapter struct {
 	secureURL string
 	namespace string
 	secret    string
+	verifySSL bool
 	jobTTL    int32
 }
 
-func NewInlineAdapter(secureClient secure.Client, k8sClient kubernetes.Interface, secureURL string, namespace string, secret string) Adapter {
+func NewInlineAdapter(secureClient secure.Client, k8sClient kubernetes.Interface, secureURL string, namespace string, secret string, verifySSL bool) Adapter {
 	return &inlineAdapter{
 		BaseAdapter: BaseAdapter{secureClient: secureClient},
 		k8sClient:   k8sClient,
 		secureURL:   secureURL,
 		namespace:   namespace,
 		secret:      secret,
+		verifySSL:   verifySSL,
 		jobTTL:      int32(24 * time.Hour.Seconds()),
 	}
 }
@@ -64,6 +67,34 @@ func (i *inlineAdapter) buildJob(req harbor.ScanRequest) *batchv1.Job {
 	user, password := getUserAndPasswordFrom(req.Registry.Authorization)
 	userPassword := fmt.Sprintf("%s:%s", user, password)
 
+	var envVars = []corev1.EnvVar{
+		{
+			Name: "SYSDIG_API_TOKEN",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: i.secret,
+					},
+					Key: "sysdig_secure_api_token",
+				},
+			},
+		},
+	}
+
+	// Propagate local proxy variables to the Job
+	envVars = appendLocalEnvVar(envVars, "http_proxy")
+	envVars = appendLocalEnvVar(envVars, "https_proxy")
+	envVars = appendLocalEnvVar(envVars, "HTTPS_PROXY")
+	envVars = appendLocalEnvVar(envVars, "no_proxy")
+	envVars = appendLocalEnvVar(envVars, "NO_PROXY")
+
+	cmdString := fmt.Sprintf("/sysdig-inline-scan.sh --sysdig-url %s -d %s --registry-skip-tls --registry-auth-basic '%s' ", i.secureURL, req.Artifact.Digest, userPassword)
+	// Add --sysdig-skip-tls only if insecure
+	if !i.verifySSL {
+		cmdString += "--sysdig-skip-tls "
+	}
+
+	cmdString += getImageFrom(req)
 	return &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
@@ -76,25 +107,13 @@ func (i *inlineAdapter) buildJob(req harbor.ScanRequest) *batchv1.Job {
 					Containers: []corev1.Container{
 						{
 							Name:    "scanner",
-							Image:   "sysdiglabs/sysdig-inline-scan:harbor-1.0",
+							Image:   "quay.io/sysdig/secure-inline-scan:2",
 							Command: []string{"/bin/sh"},
 							Args: []string{
 								"-c",
-								fmt.Sprintf("/sysdig-inline-scan.sh -s %s -k '$(SYSDIG_SECURE_API_TOKEN)' -d %s -P -n -u %s %s || true", i.secureURL, req.Artifact.Digest, userPassword, getImageFrom(req)),
-							},
-							Env: []corev1.EnvVar{
-								{
-									Name: "SYSDIG_SECURE_API_TOKEN",
-									ValueFrom: &corev1.EnvVarSource{
-										SecretKeyRef: &corev1.SecretKeySelector{
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: i.secret,
-											},
-											Key: "sysdig_secure_api_token",
-										},
-									},
-								},
+								fmt.Sprintf("%s || true", cmdString),
 							},
+							Env: envVars,
 						},
 					},
 				},
@@ -103,6 +122,17 @@ func (i *inlineAdapter) buildJob(req harbor.ScanRequest) *batchv1.Job {
 	}
 }
 
+func appendLocalEnvVar(envVars []corev1.EnvVar, key string) []corev1.EnvVar {
+	if value, exists := os.LookupEnv(key); exists {
+		envVars = append(envVars, corev1.EnvVar{
+			Name:  key,
+			Value: value,
+		})
+	}
+
+	return envVars
+}
+
 func jobName(repository string, shaDigest string) string {
 	return fmt.Sprintf(
 		"inline-scan-%x",
diff --git a/pkg/scanner/inline_adapter_test.go b/pkg/scanner/inline_adapter_test.go
@@ -2,13 +2,10 @@ package scanner_test
 
 import (
 	"context"
-	"encoding/base64"
-	"encoding/json"
-	"strings"
-
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"os"
 
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -31,6 +28,34 @@ const (
 	resourceName = "inline-scan-1e668f7cc4c27e915cfed9793808357e"
 )
 
+type envItem struct {
+	value   string
+	defined bool
+}
+
+func saveEnv(keys []string) map[string]envItem {
+	envItems := make(map[string]envItem)
+	for _, key := range keys {
+		value, defined := os.LookupEnv(key)
+		envItems[key] = envItem{
+			value:   value,
+			defined: defined,
+		}
+	}
+
+	return envItems
+}
+
+func restoreEnv(savedItems map[string]envItem) {
+	for key, item := range savedItems {
+		if item.defined {
+			os.Setenv(key, item.value)
+		} else {
+			os.Unsetenv(key)
+		}
+	}
+}
+
 var _ = Describe("InlineAdapter", func() {
 	var (
 		controller    *gomock.Controller
@@ -43,7 +68,7 @@ var _ = Describe("InlineAdapter", func() {
 		controller = gomock.NewController(GinkgoT())
 		client = mocks.NewMockClient(controller)
 		k8sClient = fake.NewSimpleClientset()
-		inlineAdapter = scanner.NewInlineAdapter(client, k8sClient, secureURL, namespace, secret)
+		inlineAdapter = scanner.NewInlineAdapter(client, k8sClient, secureURL, namespace, secret, true)
 	})
 
 	AfterEach(func() {
@@ -65,6 +90,40 @@ var _ = Describe("InlineAdapter", func() {
 			Expect(result).To(Equal(job()))
 		})
 
+		It("proxy env vars are included in the Job environment", func() {
+
+			savedEnv := saveEnv([]string{"http_proxy", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"})
+
+			os.Setenv("http_proxy", "http_proxy-value")
+			os.Setenv("https_proxy", "https_proxy-value")
+			os.Setenv("HTTPS_PROXY", "HTTPS_PROXY-value")
+			os.Setenv("no_proxy", "no_proxy-value")
+			os.Setenv("NO_PROXY", "NO_PROXY-value")
+
+			inlineAdapter.Scan(scanRequest())
+
+			restoreEnv(savedEnv)
+
+			result, _ := k8sClient.BatchV1().Jobs(namespace).Get(context.Background(), resourceName, metav1.GetOptions{})
+
+			Expect(result.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "http_proxy", Value: "http_proxy-value"}))
+			Expect(result.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "https_proxy", Value: "https_proxy-value"}))
+			Expect(result.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "HTTPS_PROXY", Value: "HTTPS_PROXY-value"}))
+			Expect(result.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "no_proxy", Value: "no_proxy-value"}))
+			Expect(result.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "NO_PROXY", Value: "NO_PROXY-value"}))
+		})
+
+		It("adds --sysdig-skip-tls in insecure", func() {
+
+			inlineAdapter = scanner.NewInlineAdapter(client, k8sClient, secureURL, namespace, secret, false)
+
+			inlineAdapter.Scan(scanRequest())
+
+			result, _ := k8sClient.BatchV1().Jobs(namespace).Get(context.Background(), resourceName, metav1.GetOptions{})
+
+			Expect(result.Spec.Template.Spec.Containers[0].Args).To(ContainElement(ContainSubstring("--sysdig-skip-tls")))
+		})
+
 		Context("when a job already exists", func() {
 			It("returns the scanID for checking if scan has finished", func() {
 				k8sClient.BatchV1().Jobs(namespace).Create(context.Background(), activeJob(), metav1.CreateOptions{})
@@ -121,19 +180,6 @@ var _ = Describe("InlineAdapter", func() {
 	})
 })
 
-func getUserAndPasswordFromSecret(k8sClient kubernetes.Interface, namespace string, name string) (string, string) {
-	secret, _ := k8sClient.CoreV1().Secrets(namespace).Get(context.Background(), name, metav1.GetOptions{})
-
-	var parsed map[string]interface{}
-	json.Unmarshal(secret.Data["config.json"], &parsed)
-
-	encodedCredentials := parsed["auths"].(map[string]interface{})["harbor.sysdig-demo.zone"].(map[string]interface{})["auth"].(string)
-	basicAuthCredentials, _ := base64.StdEncoding.DecodeString(encodedCredentials)
-	credentials := strings.Split(string(basicAuthCredentials), ":")
-
-	return credentials[0], credentials[1]
-}
-
 func job() *batchv1.Job {
 	jobTTL := int32(86400)
 	return &batchv1.Job{
@@ -149,15 +195,15 @@ func job() *batchv1.Job {
 					Containers: []corev1.Container{
 						{
 							Name:    "scanner",
-							Image:   "sysdiglabs/sysdig-inline-scan:harbor-1.0",
+							Image:   "quay.io/sysdig/secure-inline-scan:2",
 							Command: []string{"/bin/sh"},
 							Args: []string{
 								"-c",
-								"/sysdig-inline-scan.sh -s https://secure.sysdig.com -k '$(SYSDIG_SECURE_API_TOKEN)' -d an image digest -P -n -u robot$9f6711d1-834d-11ea-867f-76103d08dca8:eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1OTAwMDk5OTksImlhdCI6MTU4NzQxNzk5OSwiaXNzIjoiaGFyYm9yLXRva2VuLWRlZmF1bHRJc3N1ZXIiLCJpZCI6OSwicGlkIjoyLCJhY2Nlc3MiOlt7IlJlc291cmNlIjoiL3Byb2plY3QvMi9yZXBvc2l0b3J5IiwiQWN0aW9uIjoic2Nhbm5lci1wdWxsIiwiRWZmZWN0IjoiIn1dfQ.A3_aTzvxqSTvl26pQKa97ay15zRPC9K55NE0WbEyOsY3m0KFz-HuSDatncWLSYvOlcGVdysKlF3JXYWIjQ7tEI4V76WA9UMoi-fr9vEEdWLF5C1uWZJOz_S72sQ3G1BzsLp3HyWe9ZN5EBK9mhXzYNv2rONYrr0UJeBmNnMf2mU3sH71OO_G6JvRl5fwFSLSYx8nQs82PhfVhx50wRuWl_zyeCCDy_ytLzjRBvZwKuI9iVIxgM1pRfKG15NWMHfl0lcYnjm7f1_WFGKtVddkLOTICK0_FPtef1L8A16ozo_2NA32WD9PstdcTuD37XbZ6AFXUAZFoZLfCEW97mtIZBY2uYMwDQtc6Nme4o3Ya-MnBEIAs9Vi9d5a4pkf7Two-xjI-9ESgVz79YqL-_OnecQPNJ9yAFtJuxQ7StfsCIZx84hh5VdcZmW9jlezRHh4hTAjsNmrOBFTAjPyaXk98Se3Fj0Ev3bChod63og4frE7_fE7HnoBKVPHRAdBhJ2yrAiPymfij_kD4ke1Vb0AxmGGOwRP2K3TZNqEdKcq89lU6lHYV2UfrWchuF3u4ieNEC1BGu1_m_c55f0YZH1FAq6evCyA0JnFuXzO4cCxC7WHzXXRGSC9Lm3LF7cbaZAgFj5d34gbgUQmJst8nPlpW-KtwRL-pHC6mipunCBv9bU harbor.sysdig-demo.zone/sysdig/agent:9.7.0 || true",
+								"/sysdig-inline-scan.sh --sysdig-url https://secure.sysdig.com -d an image digest --registry-skip-tls --registry-auth-basic 'robot$9f6711d1-834d-11ea-867f-76103d08dca8:eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1OTAwMDk5OTksImlhdCI6MTU4NzQxNzk5OSwiaXNzIjoiaGFyYm9yLXRva2VuLWRlZmF1bHRJc3N1ZXIiLCJpZCI6OSwicGlkIjoyLCJhY2Nlc3MiOlt7IlJlc291cmNlIjoiL3Byb2plY3QvMi9yZXBvc2l0b3J5IiwiQWN0aW9uIjoic2Nhbm5lci1wdWxsIiwiRWZmZWN0IjoiIn1dfQ.A3_aTzvxqSTvl26pQKa97ay15zRPC9K55NE0WbEyOsY3m0KFz-HuSDatncWLSYvOlcGVdysKlF3JXYWIjQ7tEI4V76WA9UMoi-fr9vEEdWLF5C1uWZJOz_S72sQ3G1BzsLp3HyWe9ZN5EBK9mhXzYNv2rONYrr0UJeBmNnMf2mU3sH71OO_G6JvRl5fwFSLSYx8nQs82PhfVhx50wRuWl_zyeCCDy_ytLzjRBvZwKuI9iVIxgM1pRfKG15NWMHfl0lcYnjm7f1_WFGKtVddkLOTICK0_FPtef1L8A16ozo_2NA32WD9PstdcTuD37XbZ6AFXUAZFoZLfCEW97mtIZBY2uYMwDQtc6Nme4o3Ya-MnBEIAs9Vi9d5a4pkf7Two-xjI-9ESgVz79YqL-_OnecQPNJ9yAFtJuxQ7StfsCIZx84hh5VdcZmW9jlezRHh4hTAjsNmrOBFTAjPyaXk98Se3Fj0Ev3bChod63og4frE7_fE7HnoBKVPHRAdBhJ2yrAiPymfij_kD4ke1Vb0AxmGGOwRP2K3TZNqEdKcq89lU6lHYV2UfrWchuF3u4ieNEC1BGu1_m_c55f0YZH1FAq6evCyA0JnFuXzO4cCxC7WHzXXRGSC9Lm3LF7cbaZAgFj5d34gbgUQmJst8nPlpW-KtwRL-pHC6mipunCBv9bU' harbor.sysdig-demo.zone/sysdig/agent:9.7.0 || true",
 							},
 							Env: []corev1.EnvVar{
 								{
-									Name: "SYSDIG_SECURE_API_TOKEN",
+									Name: "SYSDIG_API_TOKEN",
 									ValueFrom: &corev1.EnvVarSource{
 										SecretKeyRef: &corev1.SecretKeySelector{
 											LocalObjectReference: corev1.LocalObjectReference{
diff --git a/pkg/secure/client.go b/pkg/secure/client.go
@@ -38,16 +38,17 @@ type Client interface {
 }
 
 func NewClient(apiToken string, secureURL string, verifySSL bool) Client {
+	// Clone DefaultTransport to use proxy settings and default timeouts
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+
+	if !verifySSL {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
 	return &client{
 		apiToken:  apiToken,
 		secureURL: secureURL,
-		client: http.Client{
-			Transport: &http.Transport{
-				TLSClientConfig: &tls.Config{
-					InsecureSkipVerify: verifySSL,
-				},
-			},
-		},
+		client:    http.Client{Transport: transport},
 	}
 }
 

Original file line number	Diff line number	Diff line change
`@@ -82,7 +82,8 @@ func getAdapter() scanner.Adapter {`
`82`	`82`	`clientset,`
`83`	`83`	`viper.GetString("secure_url"),`
`84`	`84`	`viper.GetString("namespace_name"),`
`85`		`- viper.GetString("secret_name"))`
	`85`	`+ viper.GetString("secret_name"),`
	`86`	`+ viper.GetBool("verify_ssl"))`
`86`	`87`	`}`
`87`	`88`
`88`	`89`	`log.Info("Using backend-scanning adapter")`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ import (`
`16`	`16`
`17`	`17`	`type requestHandler struct {`
`18`	`18`	`adapter scanner.Adapter`
`19`		`- logger Logger`
	`19`	`+ logger Logger`
`20`	`20`	`}`
`21`	`21`
`22`	`22`	`type Logger interface {`
`@@ -28,7 +28,7 @@ type Logger interface {`
`28`	`28`	`func NewAPIHandler(adapter scanner.Adapter, logger Logger) http.Handler {`
`29`	`29`	`handler := requestHandler{`
`30`	`30`	`adapter: adapter,`
`31`		`- logger: logger,`
	`31`	`+ logger: logger,`
`32`	`32`	`}`
`33`	`33`
`34`	`34`	`router := mux.NewRouter()`