Add no-proxy flags for downloading files (#283)

Author: LinuxSuRen

cmd/get.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/linuxsuren/http-downloader/pkg"
 	"github.com/linuxsuren/http-downloader/pkg/installer"
+	"github.com/linuxsuren/http-downloader/pkg/net"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v3"
@@ -43,6 +44,7 @@ func newGetCmd(ctx context.Context) (cmd *cobra.Command) {
 		`The default timeout in seconds with the HTTP request`)
 	flags.IntVarP(&opt.MaxAttempts, "max-attempts", "", 10,
 		`Max times to attempt to download, zero means there's no retry action'`)
+	flags.BoolVarP(&opt.NoProxy, "no-proxy", "", false, "Indicate no HTTP proxy taken")
 	flags.BoolVarP(&opt.ShowProgress, "show-progress", "", true, "If show the progress of download")
 	flags.Int64VarP(&opt.ContinueAt, "continue-at", "", -1, "ContinueAt")
 	flags.IntVarP(&opt.Thread, "thread", "t", viper.GetInt("thread"),
@@ -74,6 +76,7 @@ type downloadOption struct {
 	Output           string
 	ShowProgress     bool
 	Timeout          int
+	NoProxy          bool
 	MaxAttempts      int
 	AcceptPreRelease bool
 	RoundTripper     http.RoundTripper
@@ -216,9 +219,17 @@ func (o *downloadOption) runE(cmd *cobra.Command, args []string) (err error) {
 
 	cmd.Printf("start to download from %s\n", o.URL)
 	if o.Thread <= 1 {
-		err = pkg.DownloadWithContinue(o.URL, o.Output, o.ContinueAt, -1, 0, o.ShowProgress)
+		downloader := &net.ContinueDownloader{}
+		downloader.WithoutProxy(o.NoProxy).
+			WithRoundTripper(o.RoundTripper)
+		err = downloader.DownloadWithContinue(o.URL, o.Output, o.ContinueAt, -1, 0, o.ShowProgress)
 	} else {
-		err = pkg.DownloadFileWithMultipleThreadKeepParts(o.URL, o.Output, o.Thread, o.KeepPart, o.ShowProgress)
+		downloader := &net.MultiThreadDownloader{}
+		downloader.WithKeepParts(o.KeepPart).
+			WithShowProgress(o.ShowProgress).
+			WithoutProxy(o.NoProxy).
+			WithRoundTripper(o.RoundTripper)
+		err = downloader.Download(o.URL, o.Output, o.Thread)
 	}
 	return
 }

cmd/get_test.go

@@ -1,10 +1,19 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
 	"testing"
+	"time"
 
+	"github.com/golang/mock/gomock"
+	"github.com/linuxsuren/http-downloader/mock/mhttp"
 	"github.com/linuxsuren/http-downloader/pkg/installer"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
@@ -26,6 +35,8 @@ func Test_newGetCmd(t *testing.T) {
 		name: "time",
 	}, {
 		name: "max-attempts",
+	}, {
+		name: "no-proxy",
 	}, {
 		name: "show-progress",
 	}, {
@@ -97,12 +108,127 @@ func TestPreRunE(t *testing.T) {
 }
 
 func TestRunE(t *testing.T) {
-	fakeCmd := &cobra.Command{}
-
-	opt := &downloadOption{}
-	opt.fetcher = &installer.FakeFetcher{}
-
-	// print schema
-	opt.PrintSchema = true
-	assert.Nil(t, opt.runE(fakeCmd, nil))
+	tests := []struct {
+		name    string
+		opt     *downloadOption
+		args    []string
+		prepare func(t *testing.T, do *downloadOption)
+		wantErr bool
+	}{{
+		name: "print shcema only",
+		opt: &downloadOption{
+			fetcher:     &installer.FakeFetcher{},
+			PrintSchema: true,
+		},
+		wantErr: false,
+	}, {
+		name: "download from an URL with one thread",
+		opt: &downloadOption{
+			fetcher: &installer.FakeFetcher{},
+			NoProxy: true,
+		},
+		prepare: func(t *testing.T, do *downloadOption) {
+			do.Output = path.Join(os.TempDir(), fmt.Sprintf("fake-%d", time.Now().Nanosecond()))
+			do.URL = "https://foo.com"
+
+			ctrl := gomock.NewController(t)
+			roundTripper := mhttp.NewMockRoundTripper(ctrl)
+
+			mockRequest, _ := http.NewRequest(http.MethodGet, do.URL, nil)
+			mockResponse := &http.Response{
+				StatusCode: http.StatusPartialContent,
+				Proto:      "HTTP/1.1",
+				Request:    mockRequest,
+				Header: map[string][]string{
+					"Content-Length": {"100"},
+				},
+				Body: ioutil.NopCloser(bytes.NewBufferString("responseBody")),
+			}
+			roundTripper.EXPECT().
+				RoundTrip(mockRequest).Return(mockResponse, nil)
+			do.RoundTripper = roundTripper
+		},
+		wantErr: false,
+	}, {
+		name: "download from an URL with multi-threads",
+		opt: &downloadOption{
+			fetcher: &installer.FakeFetcher{},
+			NoProxy: true,
+			Thread:  2,
+		},
+		prepare: func(t *testing.T, do *downloadOption) {
+			do.Output = path.Join(os.TempDir(), fmt.Sprintf("fake-%d", time.Now().Nanosecond()))
+			do.URL = "https://foo.com"
+
+			ctrl := gomock.NewController(t)
+			roundTripper := mhttp.NewMockRoundTripper(ctrl)
+
+			// for size detecting
+			mockRequest, _ := http.NewRequest(http.MethodGet, do.URL, nil)
+			mockRequest.Header.Set("Range", "bytes=2-")
+			mockResponse := &http.Response{
+				StatusCode: http.StatusPartialContent,
+				Proto:      "HTTP/1.1",
+				Request:    mockRequest,
+				Header: map[string][]string{
+					"Content-Length": {"100"},
+				},
+				Body: ioutil.NopCloser(bytes.NewBufferString("responseBody")),
+			}
+			roundTripper.EXPECT().
+				RoundTrip(mockRequest).Return(mockResponse, nil)
+
+			// for group-1
+			mockRequest1, _ := http.NewRequest(http.MethodGet, do.URL, nil)
+			mockRequest1.Header.Set("Range", "bytes=0-50")
+			mockResponse1 := &http.Response{
+				StatusCode: http.StatusPartialContent,
+				Proto:      "HTTP/1.1",
+				Request:    mockRequest1,
+				Header: map[string][]string{
+					"Content-Length": {"100"},
+				},
+				Body: ioutil.NopCloser(bytes.NewBufferString("responseBody")),
+			}
+			roundTripper.EXPECT().
+				RoundTrip(mockRequest1).Return(mockResponse1, nil)
+
+			// for group-2
+			mockRequest2, _ := http.NewRequest(http.MethodGet, do.URL, nil)
+			mockRequest2.Header.Set("Range", "bytes=51-101")
+			mockResponse2 := &http.Response{
+				StatusCode: http.StatusPartialContent,
+				Proto:      "HTTP/1.1",
+				Request:    mockRequest2,
+				Header: map[string][]string{
+					"Content-Length": {"100"},
+				},
+				Body: ioutil.NopCloser(bytes.NewBufferString("responseBody")),
+			}
+			roundTripper.EXPECT().
+				RoundTrip(mockRequest2).Return(mockResponse2, nil)
+			do.RoundTripper = roundTripper
+		},
+		wantErr: false,
+	}}
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fakeCmd := &cobra.Command{}
+			fakeCmd.SetOut(new(bytes.Buffer))
+			if tt.prepare != nil {
+				tt.prepare(t, tt.opt)
+			}
+			if tt.opt.Output != "" {
+				defer func() {
+					_ = os.RemoveAll(tt.opt.Output)
+				}()
+			}
+			err := tt.opt.runE(fakeCmd, tt.args)
+			if tt.wantErr {
+				assert.NotNil(t, err, "should error in [%d][%s]", i, tt.name)
+			} else {
+				assert.Nil(t, err, "should not error in [%d][%s]", i, tt.name)
+			}
+		})
+	}
 }

cmd/install.go

@@ -60,6 +60,7 @@ Cannot find your desired package? Please run command: hd fetch --reset, then try
 		"Clean the package if the installation is success")
 	flags.IntVarP(&opt.Thread, "thread", "t", viper.GetInt("thread"),
 		`Download file with multi-threads. It only works when its value is bigger than 1`)
+	flags.BoolVarP(&opt.NoProxy, "no-proxy", "", false, "Indicate no HTTP proxy taken")
 	flags.BoolVarP(&opt.KeepPart, "keep-part", "", false,
 		"If you want to keep the part files instead of deleting them")
 	flags.StringVarP(&opt.OS, "os", "", runtime.GOOS, "The OS of target binary file")

cmd/install_test.go

@@ -51,6 +51,8 @@ func Test_newInstallCmd(t *testing.T) {
 		Name: "fetch",
 	}, {
 		Name: "provider",
+	}, {
+		Name: "no-proxy",
 	}}
 	test.Valid(t, cmd.Flags())
 }

pkg/common/util.go

@@ -82,8 +82,8 @@ func ParseVersionNum(release string) string {
 }
 
 // GetEnvironment retrieves the value of the environment variable named by the key.
-// If the environment dosen't exist, we will lookup for alternative environment variables
-// unti we find an environment. Return empty environment value while no environment variables found.
+// If the environment doesn't exist, we will lookup for alternative environment variables
+// until we find an environment. Return empty environment value while no environment variables found.
 func GetEnvironment(key string, alternativeKeys ...string) string {
 	if value, exists := os.LookupEnv(key); exists {
 		return value

pkg/compress/types_test.go

@@ -163,4 +163,4 @@ func Test_extraFile(t *testing.T) {
 			tt.wantErr(t, extraFile(tt.args.name, tt.args.targetName, tt.args.tarFile, tt.args.header, tt.args.tarReader), fmt.Sprintf("extraFile(%v, %v, %v, %v, %v)", tt.args.name, tt.args.targetName, tt.args.tarFile, tt.args.header, tt.args.tarReader))
 		})
 	}
-}
+}