Merge pull request #12 from NullpointerW/builtin-downloader

builtin-downloader

Author: NullpointerW

.gitignore

@@ -4,4 +4,8 @@
 output.log
 build/
 test/env.yaml
-__debug_bin.exe
+__debug_bin.exe
+*bangumi.log
+*.torrent.bolt.db
+*panic.log
+*output.log

Makefile

@@ -2,7 +2,7 @@ NAME=anicat
 BUILDIR=build
 
 SRV_FILE=service-exec/service.go
-VERSION := $(patsubst v%,%,$(shell git describe --tags || echo "x.x.x"))
+VERSION := $(patsubst v%,%,$(shell git describe --tags || echo "unknown"))
 GOBUILD=CGO_ENABLED=0 go build  -ldflags '-X "github.com/NullpointerW/anicat/conf.Ver=$(VERSION)"'
 
 

conf/env.go

@@ -13,7 +13,7 @@ import (
 )
 
 var (
-	Ver    = "x.x.x"
+	Ver    = "unknown"
 	projlk = "https://github.com/NullpointerW/AniCat"
 	SrvCTL = len(os.Args) > 1 && (os.Args[1] == "install" || os.Args[1] == "uninstall" || os.Args[1] == "start")
 )
@@ -59,7 +59,8 @@ type Environment struct {
 			SkipSSL      bool   `yaml:"skipssl"`
 		} `yaml:"email"`
 	} `yaml:"push"`
-	BgmiLog bool `yaml:"bangumi-log"`
+	BgmiLog           bool `yaml:"bangumi-log"`
+	BuiltinDownloader bool `yaml:"builtin-downloader"`
 }
 
 func (env *Environment) Print() {
@@ -81,8 +82,10 @@ func (env *Environment) Print() {
 		log.Info(logStruct, "crawling setting")
 		logStruct.Clear()
 	}
-	logStruct.Append("qbt-webUrl", env.Qbt.Url, "qbt-apiRequestTimeout(ms)", env.Qbt.Timeout)
-	log.Info(logStruct, "qbt setting")
+	if !env.BuiltinDownloader {
+		logStruct.Append("qbt-webUrl", env.Qbt.Url, "qbt-apiRequestTimeout(ms)", env.Qbt.Timeout)
+		log.Info(logStruct, "qbt setting")
+	}
 }
 
 func (env *Environment) EmailPrint() {

conf/flag.go

@@ -23,6 +23,9 @@ func flagInit() {
 	flag.BoolVar(&Testing, "t", false, "testing mode")
 	testing.Init()
 	flag.Parse()
+
+	// Testing=true
+
 	if Testing {
 		IdeDebugging = true
 	}

crawl/information/bgmi.go

@@ -84,6 +84,10 @@ func DoScrape(url string) (tips map[string]string, err error) {
 		if ls != nil {
 			for _, l := range ls {
 				t := htmlquery.FindOne(l, "./span")
+				if t == nil {
+					continue
+				}
+				// fmt.Println(t)
 				tt := htmlquery.InnerText(t)
 				lt := htmlquery.InnerText(l)
 				lt = strings.Replace(lt, tt, "", 1)

crawl/information/tmdb.go

@@ -49,7 +49,7 @@ func FloderSearch(typ, searchstr string) (name, date string, err error) {
 		agent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
 		r.Headers.Set("User-Agent", agent)
 		r.Headers.Set("Accept-Language", "zh-CN,zh;q=0.9")
-		log.Info(log.NewUrlStruct(r.URL, "source", "TMDB"), "fetching folderInfo")
+		log.Info(log.NewUrlStruct(r.URL, "source", "TMDB", "serachstring", searchstr), "fetching folderInfo")
 	})
 
 	c.OnError(func(_ *colly.Response, e error) {

debug/pprof.go

@@ -0,0 +1,23 @@
+package debug
+
+import (
+	// "fmt"
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/NullpointerW/anicat/conf"
+)
+
+func init() {
+	if conf.IdeDebugging {
+		go func() {
+			err := http.ListenAndServe("127.0.0.1:6879", nil)
+			if err != nil {
+				panic(err)
+			}
+		}()
+		fmt.Println("pprof enable,listen on localhost:6879/debug/pprof/")
+	}
+
+}

downloader/builtin/client.go

@@ -1,53 +1,140 @@
-package main
+package builtin
 
 import (
-	"context"
+	"bufio"
+	"crypto/rand"
 	"fmt"
-	"log"
+	"net/http"
+	"runtime/debug"
 	"time"
 
+	CFG "github.com/NullpointerW/anicat/conf"
+	"github.com/NullpointerW/anicat/log"
 	"github.com/anacrolix/torrent"
+	"github.com/anacrolix/torrent/storage"
 )
 
-func main() {
-	// Create a new torrent client with default configuration
-	client, err := torrent.NewClient(nil)
-	if err != nil {
-		log.Fatalf("error creating torrent client: %v", err)
+var DefaultDownLoader *Downloader
+
+func init() {
+	if CFG.Env.BuiltinDownloader {
+		log.Info(log.Struct{"github", "https://github.com/anacrolix/torrent"}, "builtin-enabled, using anacrolix/torrent")
+		InitDownloader()
+	}
+}
+func InitDownloader() {
+	cfg := &DownloaderConfig{
+		BaseDir:    "./.db",
+		FakePeerID: true,
+		NopUpload:  true,
+		Seeker:     NewHttpSeeker(),
 	}
-	defer client.Close()
+	DefaultDownLoader = NewDownloader(cfg)
+}
+
+type Downloader struct {
+	client *torrent.Client
+	TorrentSeeker
+	extraTrackers [][]string
+}
+type FileName interface {
+	Name() storage.FilePathMaker
+}
 
-	// Define the magnet link
-	magnetLink := "magnet:?xt=urn:btih:fbc74348498175b4caec790054e922d28312a106&tr=http%3a%2f%2ft.nyaatracker.com%2fannounce&tr=http%3a%2f%2ftracker.kamigami.org%3a2710%2fannounce&tr=http%3a%2f%2fshare.camoe.cn%3a8080%2fannounce&tr=http%3a%2f%2fopentracker.acgnx.se%2fannounce&tr=http%3a%2f%2fanidex.moe%3a6969%2fannounce&tr=http%3a%2f%2ft.acg.rip%3a6699%2fannounce&tr=https%3a%2f%2ftr.bangumi.moe%3a9696%2fannounce&tr=udp%3a%2f%2ftr.bangumi.moe%3a6969%2fannounce&tr=http%3a%2f%2fopen.acgtracker.com%3a1096%2fannounce&tr=udp%3a%2f%2ftracker.opentrackr.org%3a1337%2fannounce"
+type FileOption interface {
+	FileName
+	Dir() storage.TorrentDirFilePathMaker
+}
+type DownloaderConfig struct {
+	BaseDir    string
+	Seeker     TorrentSeeker
+	NopUpload  bool
+	FakePeerID bool
+}
 
-	// Add the magnet link to the client
-	t, err := client.AddMagnet(magnetLink)
+func NewDownloader(c *DownloaderConfig) *Downloader {
+	cfg := torrent.NewDefaultClientConfig()
+	cfg.Seed = !c.NopUpload
+	cfg.NoUpload = c.NopUpload
+	fop := storage.NewFileClientOpts{}
+	fop.ClientBaseDir = c.BaseDir
+	cfg.DefaultStorage = storage.NewFileOpts(fop)
+	if c.Seeker == nil {
+		c.Seeker = NewHttpSeeker()
+	}
+	if c.FakePeerID {
+		f := "-qB419E-" // qBittorrent
+		var b [20]byte
+		n := copy(b[:], ([]byte)(f))
+		_, err := rand.Read(b[n:])
+		if err != nil {
+			panic("builtin-downloader: error generating peer id")
+		}
+		cfg.PeerID = (string)(b[:])
+		cfg.HTTPUserAgent = "qBittorrent/v4.1.9.14"
+		mainPath := "github.com/NullpointerW/anicat"
+		mainVersion := "unknown"
+		if buildInfo, ok := debug.ReadBuildInfo(); ok {
+			mainPath = buildInfo.Main.Path
+			mainVersion = buildInfo.Main.Version
+		}
+		exhskVer := fmt.Sprintf(
+			"%v %v (%v %v)",
+			mainPath,
+			mainVersion,
+			"qBittorrent",
+			"v4.1.9.14",
+		)
+		cfg.ExtendedHandshakeClientVersion = exhskVer
+	}
+	client, err := torrent.NewClient(cfg)
 	if err != nil {
-		log.Fatalf("error adding magnet link: %v", err)
+		panic(err)
+	}
+	peerIDStr := func(p torrent.PeerID) string {
+		b := ([20]byte)(p)
+		// fmt.Println((string)(b[:]))
+		return string(b[:])
 	}
 
-	// Wait for the torrent information to be fetched or timeout
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-	defer cancel()
-
-	<-t.GotInfo()
-	t.DownloadAll()
+	log.Info(log.Struct{"version", cfg.ExtendedHandshakeClientVersion, "userAgent", cfg.HTTPUserAgent, "peerID", peerIDStr(client.PeerID()), "upnpID", cfg.UpnpID}, "torrent-client initialized")
+	return &Downloader{client, c.Seeker, extraTrackers()}
 
-	go func() {
-		for {
-			fmt.Printf(" w %d", t.BytesCompleted())
-			time.Sleep(1 * time.Second)
-		}
-	}()
+}
 
-	select {
-	case <-t.Complete.On():
-		log.Println("torrent download complete")
-	case <-ctx.Done():
-		log.Println("torrent download timeout")
+func (d *Downloader) Download(s string, fOp FileOption, seeker TorrentSeeker) (t *torrent.Torrent, err error) {
+	var ts *torrent.TorrentSpec
+	if seeker == nil {
+		ts, err = d.Seek(s)
+	} else {
+		ts, err = seeker.Seek(s)
+	}
+	if err != nil {
+		return nil, err
 	}
 
-	// Stream the torrent content to stdout or handle it as needed
-	//reader := t.NewReader()
-	//io.Copy(os.Stdout, reader)
+	fop := storage.NewFileClientOpts{}
+	fop.TorrentDirMaker = fOp.Dir()
+	fop.FilePathMaker = fOp.Name()
+	fop.PieceCompletion = storage.NewMapPieceCompletion()
+	ts.Storage = storage.NewFileOpts(fop)
+	t, _, err = d.client.AddTorrentSpec(ts)
+	t.AddTrackers(d.extraTrackers)
+	return
+}
+func extraTrackers() (trackers [][]string) {
+	provider := "https://cdn.jsdelivr.net/gh/DeSireFire/animeTrackerList/AT_all.txt"
+	h := http.Client{Timeout: time.Second * 5}
+	get, err := h.Get(provider)
+	if err != nil {
+		log.Error(log.Struct{"err", err}, "builtin-downloader: set tracker failed")
+		return nil
+	}
+	defer get.Body.Close()
+	r := bufio.NewScanner(get.Body)
+	var trackerURLs []string
+	for r.Scan() {
+		trackerURLs = append(trackerURLs, r.Text())
+	}
+	return append(trackers, trackerURLs)
 }