feat: support to download magnet link from an url (#335)

* feat: support to download magnet link from an url

* add unit tests

---------

Co-authored-by: rick <[email protected]>

Author: LinuxSuRen

cmd/get.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/linuxsuren/http-downloader/pkg/exec"
+	"io"
 	"net/http"
 	"net/url"
 	sysos "os"
@@ -13,6 +13,11 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/linuxsuren/http-downloader/pkg/exec"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/charset"
+
 	"github.com/linuxsuren/http-downloader/pkg"
 	"github.com/linuxsuren/http-downloader/pkg/installer"
 	"github.com/linuxsuren/http-downloader/pkg/net"
@@ -62,6 +67,7 @@ func newGetCmd(ctx context.Context) (cmd *cobra.Command) {
 		"Print the category list")
 	flags.IntVarP(&opt.PrintVersionCount, "print-version-count", "", 20,
 		"The number of the version list")
+	flags.BoolVarP(&opt.Magnet, "magnet", "", false, "Fetch magnet list from a website")
 
 	_ = cmd.RegisterFlagCompletionFunc("proxy-github", ArrayCompletion("gh.api.99988866.xyz",
 		"ghproxy.com", "mirror.ghproxy.com"))
@@ -74,6 +80,7 @@ func newDownloadOption(ctx context.Context) *downloadOption {
 		RoundTripper: getRoundTripper(ctx),
 		fetcher:      &installer.DefaultFetcher{},
 		wait:         &sync.WaitGroup{},
+		execer:       exec.DefaultExecer{},
 	}
 }
 
@@ -91,6 +98,7 @@ type downloadOption struct {
 	MaxAttempts      int
 	AcceptPreRelease bool
 	RoundTripper     http.RoundTripper
+	Magnet           bool
 
 	ContinueAt int64
 
@@ -111,6 +119,7 @@ type downloadOption struct {
 	org           string
 	repo          string
 	fetcher       installer.Fetcher
+	execer        exec.Execer
 	ExpectVersion string // should be like >v1.1.0
 }
 
@@ -170,7 +179,7 @@ func (o *downloadOption) preRunE(cmd *cobra.Command, args []string) (err error)
 
 	targetURL := args[0]
 	o.Package = &installer.HDConfig{}
-	if strings.HasPrefix(targetURL, "magnet:?") {
+	if o.Magnet || strings.HasPrefix(targetURL, "magnet:?") {
 		// download via external tool
 		o.URL = targetURL
 		return
@@ -208,6 +217,21 @@ func (o *downloadOption) preRunE(cmd *cobra.Command, args []string) (err error)
 	return
 }
 
+func findAnchor(n *html.Node) (items []string) {
+	if n.Type == html.ElementNode && n.Data == "a" {
+		for _, a := range n.Attr {
+			if a.Key == "href" && strings.Contains(a.Val, "magnet") {
+				items = append(items, strings.TrimSpace(n.FirstChild.Data))
+				break
+			}
+		}
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		items = append(items, findAnchor(c)...)
+	}
+	return
+}
+
 func (o *downloadOption) runE(cmd *cobra.Command, args []string) (err error) {
 	defer func() {
 		if o.cancel != nil {
@@ -247,8 +271,8 @@ func (o *downloadOption) runE(cmd *cobra.Command, args []string) (err error) {
 		return
 	}
 
-	if strings.HasPrefix(o.URL, "magnet:?") {
-		err = downloadMagnetFile(o.ProxyGitHub, o.URL)
+	if o.Magnet || strings.HasPrefix(o.URL, "magnet:?") {
+		err = downloadMagnetFile(o.ProxyGitHub, o.URL, o.execer)
 		return
 	}
 
@@ -273,9 +297,8 @@ func (o *downloadOption) runE(cmd *cobra.Command, args []string) (err error) {
 	return
 }
 
-func downloadMagnetFile(proxyGitHub, target string) (err error) {
+func downloadMagnetFile(proxyGitHub, target string, execer exec.Execer) (err error) {
 	targetCmd := "gotorrent"
-	execer := exec.DefaultExecer{}
 	is := installer.Installer{
 		Provider:    "github",
 		Execer:      execer,
@@ -287,6 +310,43 @@ func downloadMagnetFile(proxyGitHub, target string) (err error) {
 		return
 	}
 
+	if strings.HasPrefix(target, "http") {
+		var resp *http.Response
+		if resp, err = http.Get(target); err == nil && resp.StatusCode == http.StatusOK {
+			var data []byte
+			data, err = io.ReadAll(resp.Body)
+			if err != nil {
+				return
+			}
+
+			var reader io.Reader
+			if reader, err = charset.NewReader(strings.NewReader(string(data)), "UTF-8"); err != nil {
+				return
+			}
+
+			var docutf8 *html.Node
+			if docutf8, err = html.Parse(reader); err != nil {
+				return
+			}
+			items := findAnchor(docutf8)
+
+			if len(items) > 1 {
+				selector := &survey.Select{
+					Message: "Select item",
+					Options: items,
+				}
+				err = survey.AskOne(selector, &target)
+			} else if len(items) > 0 {
+				target = items[0]
+			}
+		}
+	}
+
+	fmt.Println(target)
+	if target == "" || err != nil {
+		return
+	}
+
 	var targetBinary string
 	if targetBinary, err = execer.LookPath(targetCmd); err == nil {
 		sysCallArgs := []string{targetCmd}

cmd/get_test.go

@@ -14,7 +14,9 @@ import (
 	"time"
 
 	"github.com/golang/mock/gomock"
+	"github.com/h2non/gock"
 	"github.com/linuxsuren/http-downloader/mock/mhttp"
+	"github.com/linuxsuren/http-downloader/pkg/exec"
 	"github.com/linuxsuren/http-downloader/pkg/installer"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
@@ -238,3 +240,59 @@ func TestRunE(t *testing.T) {
 		})
 	}
 }
+
+func TestDownloadMagnetFile(t *testing.T) {
+	tests := []struct {
+		name        string
+		proxyGitHub string
+		target      string
+		prepare     func()
+		execer      exec.Execer
+		expectErr   bool
+	}{{
+		name:        "proxyGitHub and target is empty",
+		proxyGitHub: "",
+		target:      "",
+		execer:      exec.FakeExecer{},
+	}, {
+		name:        "failed to download",
+		proxyGitHub: "",
+		target:      "fake",
+		execer:      exec.FakeExecer{ExpectError: errors.New("error")},
+		expectErr:   true,
+	}, {
+		name:        "one target item",
+		proxyGitHub: "",
+		target:      "http://fake.com",
+		prepare: func() {
+			gock.New("http://fake.com").
+				Get("/").
+				Reply(http.StatusOK).
+				File("testdata/magnet.html")
+		},
+		execer:    exec.FakeExecer{},
+		expectErr: false,
+	}, {
+		name:        "HTTP server error response",
+		proxyGitHub: "",
+		target:      "http://fake.com",
+		prepare: func() {
+			gock.New("http://fake.com").
+				Get("/").
+				ReplyError(errors.New("error"))
+		},
+		execer:    exec.FakeExecer{},
+		expectErr: true,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			defer gock.Off()
+			if tt.prepare == nil {
+				tt.prepare = func() {}
+			}
+			tt.prepare()
+			err := downloadMagnetFile(tt.proxyGitHub, tt.target, tt.execer)
+			assert.Equal(t, tt.expectErr, err != nil, err)
+		})
+	}
+}

cmd/install_test.go

@@ -151,7 +151,10 @@ func TestShouldInstall(t *testing.T) {
 	assert.True(t, exist)
 
 	// not exist
-	opt.execer = &exec.FakeExecer{ExpectError: errors.New("fake")}
+	opt.execer = &exec.FakeExecer{
+		ExpectError:         errors.New("fake"),
+		ExpectLookPathError: errors.New("error"),
+	}
 	should, exist = opt.shouldInstall()
 	assert.True(t, should)
 	assert.False(t, exist)

cmd/testdata/magnet.html

@@ -0,0 +1,13 @@
+<html>
+    <head></head>
+    <table>
+        <tr>
+            <td>
+                <a href="magnet:?xxx">magnet:?xxx</a>
+            </td>
+            <td>
+                <a href="https://github.com">github</a>
+            </td>
+        </tr>
+    </table>
+</html>

pkg/exec/fake-command.go

@@ -7,15 +7,16 @@ import (
 
 // FakeExecer is for the unit test purposes
 type FakeExecer struct {
-	ExpectError  error
-	ExpectOutput string
-	ExpectOS     string
-	ExpectArch   string
+	ExpectError         error
+	ExpectLookPathError error
+	ExpectOutput        string
+	ExpectOS            string
+	ExpectArch          string
 }
 
 // LookPath is a fake method
 func (f FakeExecer) LookPath(path string) (string, error) {
-	return "", f.ExpectError
+	return "", f.ExpectLookPathError
 }
 
 // Command is a fake method

pkg/exec/fake-command_test.go

@@ -9,15 +9,15 @@ import (
 
 func TestLookPath(t *testing.T) {
 	fake := FakeExecer{
-		ExpectError:  errors.New("fake"),
-		ExpectOutput: "output",
-		ExpectOS:     "os",
-		ExpectArch:   "arch",
+		ExpectLookPathError: errors.New("fake"),
+		ExpectOutput:        "output",
+		ExpectOS:            "os",
+		ExpectArch:          "arch",
 	}
 	_, err := fake.LookPath("")
 	assert.NotNil(t, err)
 
-	fake.ExpectError = nil
+	fake.ExpectLookPathError = nil
 	_, err = fake.LookPath("")
 	assert.Nil(t, err)
 

pkg/net/fake.go

@@ -0,0 +1,16 @@
+// Package net provides net related functions
+package net
+
+import "fmt"
+
+// FakeReader is a fake reader for the test purpose
+type FakeReader struct {
+	ExpectErr error
+}
+
+// Read is a fake method
+func (e *FakeReader) Read(p []byte) (n int, err error) {
+	err = e.ExpectErr
+	fmt.Println(err, "fake")
+	return
+}

pkg/net/fake_test.go

@@ -0,0 +1,17 @@
+package net_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/linuxsuren/http-downloader/pkg/net"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFakeReader(t *testing.T) {
+	reader := &net.FakeReader{
+		ExpectErr: errors.New("error"),
+	}
+	_, err := reader.Read(nil)
+	assert.NotNil(t, err)
+}

pkg/os/apt/init_test.go

@@ -32,8 +32,9 @@ func TestCommonCase(t *testing.T) {
 
 	errRegistry := &core.FakeRegistry{}
 	SetInstallerRegistry(errRegistry, exec.FakeExecer{
-		ExpectError: errors.New("error"),
-		ExpectOS:    "linux",
+		ExpectLookPathError: errors.New("error"),
+		ExpectError:         errors.New("error"),
+		ExpectOS:            "linux",
 	})
 	errRegistry.Walk(func(s string, i core.Installer) {
 		t.Run(s, func(t *testing.T) {