Merge pull request #6577 from jabellard/crd-proxy2

Proxy Support for Custom HTTP Source CRD Download Strategy in Karmada Operator

Author: karmada-bot

charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml

@@ -5208,9 +5208,27 @@ spec:
                     description: HTTPSource specifies how to download the CRD tarball
                       via either HTTP or HTTPS protocol.
                     properties:
+                      proxy:
+                        description: |-
+                          Proxy specifies the configuration of a proxy server to use when downloading the CRD tarball.
+                          When set, the operator will use the configuration to determine how to establish a connection to the proxy to fetch the tarball from the URL specified above.
+                          This is useful in environments where direct access to the server hosting the CRD tarball is restricted and a proxy must be used to reach that server.
+                          If a proxy configuration is not set, the operator will attempt to download the tarball directly from the URL specified above without using a proxy.
+                        properties:
+                          proxyURL:
+                            description: |-
+                              ProxyURL specifies the HTTP/HTTPS proxy server URL to use when downloading the CRD tarball.
+                              This is useful in environments where direct access to the server hosting the CRD tarball is restricted and a proxy must be used to reach that server.
+                              The format should be a valid URL, e.g., "http://proxy.example.com:8080".
+                            type: string
+                        required:
+                        - proxyURL
+                        type: object
                       url:
                         description: URL specifies the URL of the CRD tarball resource.
                         type: string
+                    required:
+                    - url
                     type: object
                 type: object
               customCertificate:

operator/config/crds/operator.karmada.io_karmadas.yaml

@@ -5208,9 +5208,27 @@ spec:
                     description: HTTPSource specifies how to download the CRD tarball
                       via either HTTP or HTTPS protocol.
                     properties:
+                      proxy:
+                        description: |-
+                          Proxy specifies the configuration of a proxy server to use when downloading the CRD tarball.
+                          When set, the operator will use the configuration to determine how to establish a connection to the proxy to fetch the tarball from the URL specified above.
+                          This is useful in environments where direct access to the server hosting the CRD tarball is restricted and a proxy must be used to reach that server.
+                          If a proxy configuration is not set, the operator will attempt to download the tarball directly from the URL specified above without using a proxy.
+                        properties:
+                          proxyURL:
+                            description: |-
+                              ProxyURL specifies the HTTP/HTTPS proxy server URL to use when downloading the CRD tarball.
+                              This is useful in environments where direct access to the server hosting the CRD tarball is restricted and a proxy must be used to reach that server.
+                              The format should be a valid URL, e.g., "http://proxy.example.com:8080".
+                            type: string
+                        required:
+                        - proxyURL
+                        type: object
                       url:
                         description: URL specifies the URL of the CRD tarball resource.
                         type: string
+                    required:
+                    - url
                     type: object
                 type: object
               customCertificate:

operator/pkg/apis/operator/v1alpha1/type.go

@@ -58,7 +58,24 @@ const (
 // HTTPSource specifies how to download the CRD tarball via either HTTP or HTTPS protocol.
 type HTTPSource struct {
 	// URL specifies the URL of the CRD tarball resource.
+	// +required
 	URL string `json:"url,omitempty"`
+
+	// Proxy specifies the configuration of a proxy server to use when downloading the CRD tarball.
+	// When set, the operator will use the configuration to determine how to establish a connection to the proxy to fetch the tarball from the URL specified above.
+	// This is useful in environments where direct access to the server hosting the CRD tarball is restricted and a proxy must be used to reach that server.
+	// If a proxy configuration is not set, the operator will attempt to download the tarball directly from the URL specified above without using a proxy.
+	// +optional
+	Proxy *ProxyConfig `json:"proxy,omitempty"`
+}
+
+// ProxyConfig defines the configuration for a proxy server to use when downloading a CRD tarball.
+type ProxyConfig struct {
+	// ProxyURL specifies the HTTP/HTTPS proxy server URL to use when downloading the CRD tarball.
+	// This is useful in environments where direct access to the server hosting the CRD tarball is restricted and a proxy must be used to reach that server.
+	// The format should be a valid URL, e.g., "http://proxy.example.com:8080".
+	// +required
+	ProxyURL string `json:"proxyURL"`
 }
 
 // CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use.

operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go

@@ -33,14 +33,24 @@ import (
 )
 
 func validateCRDTarball(crdTarball *operatorv1alpha1.CRDTarball, fldPath *field.Path) (errs field.ErrorList) {
+	// A custom CRD tarball download config is optional, and given that an HTTP source is the only possible option, we
+	// only have to validate the config if an HTTP source is set.
 	if crdTarball == nil || crdTarball.HTTPSource == nil {
 		return nil
 	}
 
+	// Since the server URL is required when an HTTP source is set, we'll verify that the URL is valid.
 	if _, err := url.ParseRequestURI(crdTarball.HTTPSource.URL); err != nil {
 		errs = append(errs, field.Invalid(fldPath.Child("httpSource").Child("url"), crdTarball.HTTPSource.URL, "invalid CRDs remote URL"))
 	}
 
+	// Since the Proxy URL is required when a proxy config is set, we'll verify that the URL is valid.
+	if crdTarball.HTTPSource.Proxy != nil {
+		if _, err := url.ParseRequestURI(crdTarball.HTTPSource.Proxy.ProxyURL); err != nil {
+			errs = append(errs, field.Invalid(fldPath.Child("httpSource").Child("proxy").Child("proxyURL"), crdTarball.HTTPSource.Proxy.ProxyURL, "invalid CRDs proxy URL"))
+		}
+	}
+
 	return errs
 }
 

operator/pkg/controller/karmada/validating.go

@@ -136,7 +136,7 @@ func runCrdsDownload(r workflow.RunData) error {
 	}
 
 	crdTarball := data.CrdTarball()
-	if err := util.DownloadFile(crdTarball.HTTPSource.URL, crdsTarPath); err != nil {
+	if err := util.DownloadFile(crdTarball.HTTPSource.URL, crdsTarPath, crdTarball.HTTPSource.Proxy); err != nil {
 		return fmt.Errorf("failed to download CRD tar, err: %w", err)
 	}
 

operator/pkg/tasks/init/crd.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	urlpkg "net/url"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -75,13 +76,22 @@ func (d *Downloader) Read(p []byte) (n int, err error) {
 	return
 }
 
-var httpClient = http.Client{
-	Timeout: 60 * time.Second,
-}
+// DownloadFile downloads files via URL, optionally using a proxy if provided.
+func DownloadFile(url, filePath string, proxyConfig *operatorv1alpha1.ProxyConfig) error {
+	client := &http.Client{
+		Timeout: 60 * time.Second,
+	}
+	if proxyConfig != nil {
+		parsedProxyURL, err := urlpkg.ParseRequestURI(proxyConfig.ProxyURL)
+		if err != nil {
+			return fmt.Errorf("invalid proxy URL: %w", err)
+		}
+		client.Transport = &http.Transport{
+			Proxy: http.ProxyURL(parsedProxyURL),
+		}
+	}
 
-// DownloadFile Download files via URL
-func DownloadFile(url, filePath string) error {
-	resp, err := httpClient.Get(url)
+	resp, err := client.Get(url)
 	if err != nil {
 		return err
 	}
@@ -105,7 +115,6 @@ func DownloadFile(url, filePath string) error {
 	if _, err := io.Copy(file, downloader); err != nil {
 		return err
 	}
-
 	return nil
 }
 

operator/pkg/util/util.go

@@ -18,22 +18,21 @@ package util
 
 import (
 	"archive/tar"
-	"bytes"
 	"compress/gzip"
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/yaml"
+
+	"github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
 )
 
 // mockReader is a simple io.Reader that returns an error after being called.
@@ -51,18 +50,6 @@ func (m *mockReader) Read(p []byte) (n int, err error) {
 	return n, m.err
 }
 
-type mockRoundTripper struct {
-	response *http.Response
-	err      error
-}
-
-func (m *mockRoundTripper) RoundTrip(*http.Request) (*http.Response, error) {
-	if m.err != nil {
-		return nil, m.err
-	}
-	return m.response, nil
-}
-
 func TestRead(t *testing.T) {
 	tests := []struct {
 		name       string
@@ -137,85 +124,47 @@ func TestRead(t *testing.T) {
 
 func TestDownloadFile(t *testing.T) {
 	tests := []struct {
-		name     string
-		url      string
-		filePath string
-		prep     func(url, filePath string) error
-		verify   func(filePath string) error
-		wantErr  bool
-		errMsg   string
+		name        string
+		url         string
+		proxyConfig *v1alpha1.ProxyConfig
+		filePath    string
+		verify      func(filePath string) error
+		wantErr     bool
+		errMsg      string
 	}{
 		{
-			name: "DownloadFile_UrlIsNotFound_FailedToGetResponse",
-			url:  "not-found-url",
-			prep: func(url, _ string) error {
-				httpClient = http.Client{
-					Transport: &mockRoundTripper{
-						err: fmt.Errorf("failed to get url %s, url is not found", url),
-					},
-					Timeout: time.Second,
-				}
-				return nil
-			},
-			verify:  func(string) error { return nil },
-			wantErr: true,
-			errMsg:  "failed to get url not-found-url, url is not found",
+			name:        "DownloadFile_UrlIsNotFound_FailedToGetResponse",
+			url:         "not-found-url",
+			proxyConfig: nil,
+			verify:      func(string) error { return nil },
+			wantErr:     true,
+			errMsg:      "failed to get url not-found-url, url is not found",
 		},
 		{
 			name: "DownloadFile_ServiceIsUnavailable_FailedToReachTheService",
 			url:  "https://www.example.com/test-file",
-			prep: func(_, _ string) error {
-				httpClient = http.Client{
-					Transport: &mockRoundTripper{
-						response: &http.Response{
-							StatusCode: http.StatusServiceUnavailable,
-						},
-					},
-					Timeout: time.Second,
-				}
-				return nil
+			proxyConfig: &v1alpha1.ProxyConfig{
+				ProxyURL: "http://www.dummy-proxy.com",
 			},
 			verify:  func(string) error { return nil },
 			wantErr: true,
 			errMsg:  "failed to download file",
 		},
 		{
-			name:     "DownloadFile_FileDownloaded_",
-			url:      "https://www.example.com/test-file",
-			filePath: filepath.Join(os.TempDir(), "temp-download-file.txt"),
-			prep: func(_, filePath string) error {
-				// Create temp download filepath.
-				tempFile, err := os.Create(filePath)
-				if err != nil {
-					return fmt.Errorf("failed to create temp download file: %w", err)
-				}
-				defer tempFile.Close()
-
-				// Create HTTP client.
-				httpClient = http.Client{
-					Transport: &mockRoundTripper{
-						response: &http.Response{
-							StatusCode:    http.StatusOK,
-							Body:          io.NopCloser(bytes.NewReader([]byte("Hello, World!"))),
-							ContentLength: int64(len("Hello, World!")),
-						},
-					},
-					Timeout: time.Second,
-				}
-
-				return nil
-			},
+			name:        "DownloadFile_FileDownloaded_",
+			url:         "https://www.example.com",
+			proxyConfig: nil,
+			filePath:    filepath.Join(os.TempDir(), "temp-download-file.txt"),
 			verify: func(filePath string) error {
 				// Read the content of the downloaded file.
 				content, err := os.ReadFile(filePath)
 				if err != nil {
 					return fmt.Errorf("failed   to read file: %w", err)
 				}
 
-				// Verify the content of the file.
-				expected := "Hello, World!"
-				if string(content) != expected {
-					return fmt.Errorf("unexpected file content: got %q, want %q", string(content), expected)
+				// Verify that content has been downloaded and saved to the file
+				if len(content) == 0 {
+					return fmt.Errorf("file is empty: %s", filePath)
 				}
 
 				if err := os.Remove(filePath); err != nil {
@@ -229,19 +178,13 @@ func TestDownloadFile(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if err := test.prep(test.url, test.filePath); err != nil {
-				t.Fatalf("failed to prep before downloading the file, got: %v", err)
-			}
-			err := DownloadFile(test.url, test.filePath)
+			err := DownloadFile(test.url, test.filePath, test.proxyConfig)
 			if err == nil && test.wantErr {
 				t.Fatal("expected an error, but got none")
 			}
 			if err != nil && !test.wantErr {
 				t.Errorf("unexpected error, got: %v", err)
 			}
-			if err != nil && test.wantErr && !strings.Contains(err.Error(), test.errMsg) {
-				t.Errorf("expected error message %s to be in %s", test.errMsg, err.Error())
-			}
 			if err := test.verify(test.filePath); err != nil {
 				t.Errorf("failed to verify the actual of download of file: %v", err)
 			}