pikpak: fix login issue where token retrieval fails

This addresses the login issue caused by pikpak's recent cancellation 
of existing login methods and requirement for additional verifications. 

To resolve this, we've made the following changes:

1. Similar to lib/oauthutil, we've integrated a mechanism to handle 
captcha tokens.

2. A new pikpakClient has been introduced to wrap the existing 
rest.Client and incorporate the necessary headers including 
x-captcha-token for each request.

3. Several options have been added/removed to support persistent 
user/client identification.

* client_id: No longer configurable.
* client_secret: Deprecated as it's no longer used.
* user_agent: A new option that defaults to PC/Firefox's user agent 
but can be overridden using the --pikpak-user-agent flag.
* device_id: A new option that is randomly generated if invalid. 
It is recommended not to delete or change it frequently.
* captcha_token: A new option that is automatically managed 
by rclone, similar to the OAuth token.

Fixes rclone#7950 rclone#8005

Author: wiserain

backend/pikpak/api/types.go

@@ -513,6 +513,72 @@ type RequestDecompress struct {
 	DefaultParent bool             `json:"default_parent,omitempty"`
 }
 
+// ------------------------------------------------------------ authorization
+
+// CaptchaToken is a response to requestCaptchaToken api call
+type CaptchaToken struct {
+	CaptchaToken string `json:"captcha_token"`
+	ExpiresIn    int64  `json:"expires_in"` // currently 300s
+	// API doesn't provide Expiry field and thus it should be populated from ExpiresIn on retrieval
+	Expiry time.Time `json:"expiry,omitempty"`
+	URL    string    `json:"url,omitempty"` // a link for users to solve captcha
+}
+
+// expired reports whether the token is expired.
+// t must be non-nil.
+func (t *CaptchaToken) expired() bool {
+	if t.Expiry.IsZero() {
+		return false
+	}
+
+	expiryDelta := time.Duration(10) * time.Second // same as oauth2's defaultExpiryDelta
+	return t.Expiry.Round(0).Add(-expiryDelta).Before(time.Now())
+}
+
+// Valid reports whether t is non-nil, has an AccessToken, and is not expired.
+func (t *CaptchaToken) Valid() bool {
+	return t != nil && t.CaptchaToken != "" && !t.expired()
+}
+
+// CaptchaTokenRequest is to request for captcha token
+type CaptchaTokenRequest struct {
+	Action       string            `json:"action,omitempty"`
+	CaptchaToken string            `json:"captcha_token,omitempty"`
+	ClientID     string            `json:"client_id,omitempty"`
+	DeviceID     string            `json:"device_id,omitempty"`
+	Meta         *CaptchaTokenMeta `json:"meta,omitempty"`
+}
+
+// CaptchaTokenMeta contains meta info for CaptchaTokenRequest
+type CaptchaTokenMeta struct {
+	CaptchaSign   string `json:"captcha_sign,omitempty"`
+	ClientVersion string `json:"client_version,omitempty"`
+	PackageName   string `json:"package_name,omitempty"`
+	Timestamp     string `json:"timestamp,omitempty"`
+	UserID        string `json:"user_id,omitempty"` // webdrive uses this instead of UserName
+	UserName      string `json:"username,omitempty"`
+	Email         string `json:"email,omitempty"`
+	PhoneNumber   string `json:"phone_number,omitempty"`
+}
+
+// Token represents oauth2 token used for pikpak which needs to be converted to be compatible with oauth2.Token
+type Token struct {
+	TokenType    string `json:"token_type"`
+	AccessToken  string `json:"access_token"`
+	RefreshToken string `json:"refresh_token"`
+	ExpiresIn    int    `json:"expires_in"`
+	Sub          string `json:"sub"`
+}
+
+// Expiry returns expiry from expires in, so it should be called on retrieval
+// e must be non-nil.
+func (e *Token) Expiry() (t time.Time) {
+	if v := e.ExpiresIn; v != 0 {
+		return time.Now().Add(time.Duration(v) * time.Second)
+	}
+	return
+}
+
 // ------------------------------------------------------------
 
 // NOT implemented YET

backend/pikpak/helper.go

@@ -3,8 +3,10 @@ package pikpak
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
 	"crypto/sha1"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -14,10 +16,13 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/pikpak/api"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -262,15 +267,20 @@ func (f *Fs) getGcid(ctx context.Context, src fs.ObjectInfo) (gcid string, err e
 	if err != nil {
 		return
 	}
+	if src.Size() == 0 {
+		// If src is zero-length, the API will return
+		// Error "cid and file_size is required" (400)
+		// In this case, we can simply return cid == gcid
+		return cid, nil
+	}
 
 	params := url.Values{}
 	params.Set("cid", cid)
 	params.Set("file_size", strconv.FormatInt(src.Size(), 10))
 	opts := rest.Opts{
-		Method:       "GET",
-		Path:         "/drive/v1/resource/cid",
-		Parameters:   params,
-		ExtraHeaders: map[string]string{"x-device-id": f.deviceID},
+		Method:     "GET",
+		Path:       "/drive/v1/resource/cid",
+		Parameters: params,
 	}
 
 	info := struct {
@@ -408,6 +418,8 @@ func calcCid(ctx context.Context, src fs.ObjectInfo) (cid string, err error) {
 	return
 }
 
+// ------------------------------------------------------------ authorization
+
 // randomly generates device id used for request header 'x-device-id'
 //
 // original javascript implementation
@@ -428,3 +440,206 @@ func genDeviceID() string {
 	}
 	return string(base)
 }
+
+var md5Salt = []string{
+	"C9qPpZLN8ucRTaTiUMWYS9cQvWOE",
+	"+r6CQVxjzJV6LCV",
+	"F",
+	"pFJRC",
+	"9WXYIDGrwTCz2OiVlgZa90qpECPD6olt",
+	"/750aCr4lm/Sly/c",
+	"RB+DT/gZCrbV",
+	"",
+	"CyLsf7hdkIRxRm215hl",
+	"7xHvLi2tOYP0Y92b",
+	"ZGTXXxu8E/MIWaEDB+Sm/",
+	"1UI3",
+	"E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO",
+	"ihtqpG6FMt65+Xk+tWUH2",
+	"NhXXU9rg4XXdzo7u5o",
+}
+
+func md5Sum(text string) string {
+	hash := md5.Sum([]byte(text))
+	return hex.EncodeToString(hash[:])
+}
+
+func calcCaptchaSign(deviceID string) (timestamp, sign string) {
+	timestamp = fmt.Sprint(time.Now().UnixMilli())
+	str := fmt.Sprint(clientID, clientVersion, packageName, deviceID, timestamp)
+	for _, salt := range md5Salt {
+		str = md5Sum(str + salt)
+	}
+	sign = "1." + str
+	return
+}
+
+func newCaptchaTokenRequest(action, oldToken string, opt *Options) (req *api.CaptchaTokenRequest) {
+	req = &api.CaptchaTokenRequest{
+		Action:       action,
+		CaptchaToken: oldToken, // can be empty initially
+		ClientID:     clientID,
+		DeviceID:     opt.DeviceID,
+		Meta:         new(api.CaptchaTokenMeta),
+	}
+	switch action {
+	case "POST:/v1/auth/signin":
+		req.Meta.UserName = opt.Username
+	default:
+		timestamp, captchaSign := calcCaptchaSign(opt.DeviceID)
+		req.Meta.CaptchaSign = captchaSign
+		req.Meta.Timestamp = timestamp
+		req.Meta.ClientVersion = clientVersion
+		req.Meta.PackageName = packageName
+		req.Meta.UserID = opt.UserID
+	}
+	return
+}
+
+// CaptchaTokenSource stores updated captcha tokens in the config file
+type CaptchaTokenSource struct {
+	mu    sync.Mutex
+	m     configmap.Mapper
+	opt   *Options
+	token *api.CaptchaToken
+	ctx   context.Context
+	rst   *pikpakClient
+}
+
+// initialize CaptchaTokenSource from rclone.conf if possible
+func newCaptchaTokenSource(ctx context.Context, opt *Options, m configmap.Mapper) *CaptchaTokenSource {
+	token := new(api.CaptchaToken)
+	tokenString, ok := m.Get("captcha_token")
+	if !ok || tokenString == "" {
+		fs.Debugf(nil, "failed to read captcha token out of config file")
+	} else {
+		if err := json.Unmarshal([]byte(tokenString), token); err != nil {
+			fs.Debugf(nil, "failed to parse captcha token out of config file: %v", err)
+		}
+	}
+	return &CaptchaTokenSource{
+		m:     m,
+		opt:   opt,
+		token: token,
+		ctx:   ctx,
+		rst:   newPikpakClient(getClient(ctx, opt), opt),
+	}
+}
+
+// requestToken retrieves captcha token from API
+func (cts *CaptchaTokenSource) requestToken(ctx context.Context, req *api.CaptchaTokenRequest) (err error) {
+	opts := rest.Opts{
+		Method:  "POST",
+		RootURL: "https://user.mypikpak.com/v1/shield/captcha/init",
+	}
+	var info *api.CaptchaToken
+	_, err = cts.rst.CallJSON(ctx, &opts, &req, &info)
+	if err == nil && info.ExpiresIn != 0 {
+		// populate to Expiry
+		info.Expiry = time.Now().Add(time.Duration(info.ExpiresIn) * time.Second)
+		cts.token = info // update with a new one
+	}
+	return
+}
+
+func (cts *CaptchaTokenSource) refreshToken(opts *rest.Opts) (string, error) {
+	oldToken := ""
+	if cts.token != nil {
+		oldToken = cts.token.CaptchaToken
+	}
+	action := "GET:/drive/v1/about"
+	if opts.RootURL == "" && opts.Path != "" {
+		action = fmt.Sprintf("%s:%s", opts.Method, opts.Path)
+	} else if u, err := url.Parse(opts.RootURL); err == nil {
+		action = fmt.Sprintf("%s:%s", opts.Method, u.Path)
+	}
+	req := newCaptchaTokenRequest(action, oldToken, cts.opt)
+	if err := cts.requestToken(cts.ctx, req); err != nil {
+		return "", fmt.Errorf("failed to retrieve captcha token from api: %w", err)
+	}
+
+	// put it into rclone.conf
+	tokenBytes, err := json.Marshal(cts.token)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal captcha token: %w", err)
+	}
+	cts.m.Set("captcha_token", string(tokenBytes))
+	return cts.token.CaptchaToken, nil
+}
+
+// Invalidate resets existing captcha token for a forced refresh
+func (cts *CaptchaTokenSource) Invalidate() {
+	cts.mu.Lock()
+	cts.token.CaptchaToken = ""
+	cts.mu.Unlock()
+}
+
+// Token returns a valid captcha token
+func (cts *CaptchaTokenSource) Token(opts *rest.Opts) (string, error) {
+	cts.mu.Lock()
+	defer cts.mu.Unlock()
+	if cts.token.Valid() {
+		return cts.token.CaptchaToken, nil
+	}
+	return cts.refreshToken(opts)
+}
+
+// pikpakClient wraps rest.Client with a handle of captcha token
+type pikpakClient struct {
+	opt     *Options
+	client  *rest.Client
+	captcha *CaptchaTokenSource
+}
+
+// newPikpakClient takes an (oauth) http.Client and makes a new api instance for pikpak with
+// * error handler
+// * root url
+// * default headers
+func newPikpakClient(c *http.Client, opt *Options) *pikpakClient {
+	client := rest.NewClient(c).SetErrorHandler(errorHandler).SetRoot(rootURL)
+	for key, val := range map[string]string{
+		"Referer":          "https://mypikpak.com/",
+		"x-client-id":      clientID,
+		"x-client-version": clientVersion,
+		"x-device-id":      opt.DeviceID,
+		// "x-device-model":   "firefox%2F129.0",
+		// "x-device-name":    "PC-Firefox",
+		// "x-device-sign": fmt.Sprintf("wdi10.%sxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", opt.DeviceID),
+		// "x-net-work-type":    "NONE",
+		// "x-os-version":       "Win32",
+		// "x-platform-version": "1",
+		// "x-protocol-version": "301",
+		// "x-provider-name":    "NONE",
+		// "x-sdk-version":      "8.0.3",
+	} {
+		client.SetHeader(key, val)
+	}
+	return &pikpakClient{
+		client: client,
+		opt:    opt,
+	}
+}
+
+// This should be called right after pikpakClient initialized
+func (c *pikpakClient) SetCaptchaTokener(ctx context.Context, m configmap.Mapper) *pikpakClient {
+	c.captcha = newCaptchaTokenSource(ctx, c.opt, m)
+	return c
+}
+
+func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
+	if c.captcha != nil {
+		token, err := c.captcha.Token(opts)
+		if err != nil || token == "" {
+			return nil, fserrors.FatalError(fmt.Errorf("couldn't get captcha token: %v", err))
+		}
+		if opts.ExtraHeaders == nil {
+			opts.ExtraHeaders = make(map[string]string)
+		}
+		opts.ExtraHeaders["x-captcha-token"] = token
+	}
+	return c.client.CallJSON(ctx, opts, request, response)
+}
+
+func (c *pikpakClient) Call(ctx context.Context, opts *rest.Opts) (resp *http.Response, err error) {
+	return c.client.Call(ctx, opts)
+}