Fix missing Close when error occurs and abused connection pool (#35658)

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/bohde/codel v0.2.0
 	github.com/buildkite/terminal-to-html/v3 v3.16.8
 	github.com/caddyserver/certmagic v0.24.0
-	github.com/charmbracelet/git-lfs-transfer v0.2.0
+	github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20251013092601-6327009efd21
 	github.com/chi-middleware/proxy v1.1.1
 	github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21
 	github.com/djherbis/buffer v1.2.0
@@ -56,7 +56,7 @@ require (
 	github.com/go-co-op/gocron v1.37.0
 	github.com/go-enry/go-enry/v2 v2.9.2
 	github.com/go-git/go-billy/v5 v5.6.2
-	github.com/go-git/go-git/v5 v5.16.2
+	github.com/go-git/go-git/v5 v5.16.3
 	github.com/go-ldap/ldap/v3 v3.4.11
 	github.com/go-redsync/redsync/v4 v4.13.0
 	github.com/go-sql-driver/mysql v1.9.3
@@ -121,7 +121,7 @@ require (
 	golang.org/x/net v0.44.0
 	golang.org/x/oauth2 v0.30.0
 	golang.org/x/sync v0.17.0
-	golang.org/x/sys v0.36.0
+	golang.org/x/sys v0.37.0
 	golang.org/x/text v0.30.0
 	google.golang.org/grpc v1.75.0
 	google.golang.org/protobuf v1.36.8
@@ -298,9 +298,6 @@ replace github.com/hashicorp/go-version => github.com/6543/go-version v1.3.1
 
 replace github.com/nektos/act => gitea.com/gitea/act v0.261.7-0.20251003180512-ac6e4b751763
 
-// TODO: the only difference is in `PutObject`: the fork doesn't use `NewVerifyingReader(r, sha256.New(), oid, expectedSize)`, need to figure out why
-replace github.com/charmbracelet/git-lfs-transfer => gitea.com/gitea/git-lfs-transfer v0.2.0
-
 replace git.sr.ht/~mariusor/go-xsd-duration => gitea.com/gitea/go-xsd-duration v0.0.0-20220703122237-02e73435a078
 
 exclude github.com/gofrs/uuid v3.2.0+incompatible

go.sum

@@ -33,8 +33,6 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 gitea.com/gitea/act v0.261.7-0.20251003180512-ac6e4b751763 h1:ohdxegvslDEllZmRNDqpKun6L4Oq81jNdEDtGgHEV2c=
 gitea.com/gitea/act v0.261.7-0.20251003180512-ac6e4b751763/go.mod h1:Pg5C9kQY1CEA3QjthjhlrqOC/QOT5NyWNjOjRHw23Ok=
-gitea.com/gitea/git-lfs-transfer v0.2.0 h1:baHaNoBSRaeq/xKayEXwiDQtlIjps4Ac/Ll4KqLMB40=
-gitea.com/gitea/git-lfs-transfer v0.2.0/go.mod h1:UrXUCm3xLQkq15fu7qlXHUMlrhdlXHoi13KH2Dfiits=
 gitea.com/gitea/go-xsd-duration v0.0.0-20220703122237-02e73435a078 h1:BAFmdZpRW7zMQZQDClaCWobRj9uL1MR3MzpCVJvc5s4=
 gitea.com/gitea/go-xsd-duration v0.0.0-20220703122237-02e73435a078/go.mod h1:g/V2Hjas6Z1UHUp4yIx6bATpNzJ7DYtD0FG3+xARWxs=
 gitea.com/go-chi/binding v0.0.0-20240430071103-39a851e106ed h1:EZZBtilMLSZNWtHHcgq2mt6NSGhJSZBuduAlinMEmso=
@@ -219,6 +217,8 @@ github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20251013092601-6327009efd21 h1:2d64+4Jek9vjYwhY93AjbleiVH+AeWvPwPmDi1mfKFQ=
+github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20251013092601-6327009efd21/go.mod h1:fNlYtCHWTRC8MofQERZkVUNUWaOvZeTBqHn/amSbKZI=
 github.com/chi-middleware/proxy v1.1.1 h1:4HaXUp8o2+bhHr1OhVy+VjN0+L7/07JDcn6v7YrTjrQ=
 github.com/chi-middleware/proxy v1.1.1/go.mod h1:jQwMEJct2tz9VmtCELxvnXoMfa+SOdikvbVJVHv/M+0=
 github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
@@ -339,8 +339,8 @@ github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UN
 github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
-github.com/go-git/go-git/v5 v5.16.2 h1:fT6ZIOjE5iEnkzKyxTHK1W4HGAsPhqEqiSAssSO77hM=
-github.com/go-git/go-git/v5 v5.16.2/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
+github.com/go-git/go-git/v5 v5.16.3 h1:Z8BtvxZ09bYm/yYNgPKCzgWtaRqDTgIKRgIRHBfU6Z8=
+github.com/go-git/go-git/v5 v5.16.3/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
@@ -978,8 +978,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
-golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

modules/httplib/request.go

@@ -7,54 +7,58 @@ package httplib
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
-	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 )
 
-var defaultSetting = Settings{"GiteaServer", 60 * time.Second, 60 * time.Second, nil, nil}
-
-// newRequest returns *Request with specific method
-func newRequest(url, method string) *Request {
-	var resp http.Response
-	req := http.Request{
-		Method:     method,
-		Header:     make(http.Header),
-		Proto:      "HTTP/1.1",
-		ProtoMajor: 1,
-		ProtoMinor: 1,
+var defaultTransport = sync.OnceValue(func() http.RoundTripper {
+	return &http.Transport{
+		Proxy:       http.ProxyFromEnvironment,
+		DialContext: DialContextWithTimeout(10 * time.Second), // it is good enough in modern days
 	}
-	return &Request{url, &req, map[string]string{}, defaultSetting, &resp, nil}
-}
+})
 
-// NewRequest returns *Request with specific method
-func NewRequest(url, method string) *Request {
-	return newRequest(url, method)
+func DialContextWithTimeout(timeout time.Duration) func(ctx context.Context, netw, addr string) (net.Conn, error) {
+	return func(ctx context.Context, netw, addr string) (net.Conn, error) {
+		d := net.Dialer{Timeout: timeout}
+		conn, err := d.DialContext(ctx, netw, addr)
+		if err != nil {
+			return nil, err
+		}
+		return conn, nil
+	}
 }
 
-// Settings is the default settings for http client
-type Settings struct {
-	UserAgent        string
-	ConnectTimeout   time.Duration
-	ReadWriteTimeout time.Duration
-	TLSClientConfig  *tls.Config
-	Transport        http.RoundTripper
+func NewRequest(url, method string) *Request {
+	return &Request{
+		url: url,
+		req: &http.Request{
+			Method:     method,
+			Header:     make(http.Header),
+			Proto:      "HTTP/1.1", // FIXME: from legacy httplib, it shouldn't be hardcoded
+			ProtoMajor: 1,
+			ProtoMinor: 1,
+		},
+		params: map[string]string{},
+
+		// from legacy httplib, caller's must pay more attention to it, it will cause annoying bugs when the response takes a long time
+		readWriteTimeout: 60 * time.Second,
+	}
 }
 
-// Request provides more useful methods for requesting one url than http.Request.
 type Request struct {
-	url     string
-	req     *http.Request
-	params  map[string]string
-	setting Settings
-	resp    *http.Response
-	body    []byte
+	url    string
+	req    *http.Request
+	params map[string]string
+
+	readWriteTimeout time.Duration
+	transport        http.RoundTripper
 }
 
 // SetContext sets the request's Context
@@ -63,36 +67,24 @@ func (r *Request) SetContext(ctx context.Context) *Request {
 	return r
 }
 
-// SetTimeout sets connect time out and read-write time out for BeegoRequest.
-func (r *Request) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *Request {
-	r.setting.ConnectTimeout = connectTimeout
-	r.setting.ReadWriteTimeout = readWriteTimeout
+// SetTransport sets the request transport, if not set, will use httplib's default transport with environment proxy support
+// ATTENTION: the http.Transport has connection pool, so you should reuse it as much as possible, do not create a lot of transports
+func (r *Request) SetTransport(transport http.RoundTripper) *Request {
+	r.transport = transport
 	return r
 }
 
 func (r *Request) SetReadWriteTimeout(readWriteTimeout time.Duration) *Request {
-	r.setting.ReadWriteTimeout = readWriteTimeout
-	return r
-}
-
-// SetTLSClientConfig sets tls connection configurations if visiting https url.
-func (r *Request) SetTLSClientConfig(config *tls.Config) *Request {
-	r.setting.TLSClientConfig = config
+	r.readWriteTimeout = readWriteTimeout
 	return r
 }
 
-// Header add header item string in request.
+// Header set header item string in request.
 func (r *Request) Header(key, value string) *Request {
 	r.req.Header.Set(key, value)
 	return r
 }
 
-// SetTransport sets transport to
-func (r *Request) SetTransport(transport http.RoundTripper) *Request {
-	r.setting.Transport = transport
-	return r
-}
-
 // Param adds query param in to request.
 // params build query string as ?key1=value1&key2=value2...
 func (r *Request) Param(key, value string) *Request {
@@ -125,11 +117,9 @@ func (r *Request) Body(data any) *Request {
 	return r
 }
 
-func (r *Request) getResponse() (*http.Response, error) {
-	if r.resp.StatusCode != 0 {
-		return r.resp, nil
-	}
-
+// Response executes request client and returns the response.
+// Caller MUST close the response body if no error occurs.
+func (r *Request) Response() (*http.Response, error) {
 	var paramBody string
 	if len(r.params) > 0 {
 		var buf bytes.Buffer
@@ -160,59 +150,19 @@ func (r *Request) getResponse() (*http.Response, error) {
 		return nil, err
 	}
 
-	trans := r.setting.Transport
-	if trans == nil {
-		// create default transport
-		trans = &http.Transport{
-			TLSClientConfig: r.setting.TLSClientConfig,
-			Proxy:           http.ProxyFromEnvironment,
-			DialContext:     TimeoutDialer(r.setting.ConnectTimeout),
-		}
-	} else if t, ok := trans.(*http.Transport); ok {
-		if t.TLSClientConfig == nil {
-			t.TLSClientConfig = r.setting.TLSClientConfig
-		}
-		if t.DialContext == nil {
-			t.DialContext = TimeoutDialer(r.setting.ConnectTimeout)
-		}
-	}
-
 	client := &http.Client{
-		Transport: trans,
-		Timeout:   r.setting.ReadWriteTimeout,
+		Transport: r.transport,
+		Timeout:   r.readWriteTimeout,
 	}
-
-	if len(r.setting.UserAgent) > 0 && len(r.req.Header.Get("User-Agent")) == 0 {
-		r.req.Header.Set("User-Agent", r.setting.UserAgent)
-	}
-
-	resp, err := client.Do(r.req)
-	if err != nil {
-		return nil, err
+	if client.Transport == nil {
+		client.Transport = defaultTransport()
 	}
-	r.resp = resp
-	return resp, nil
-}
 
-// Response executes request client gets response manually.
-// Caller MUST close the response body if no error occurs
-func (r *Request) Response() (*http.Response, error) {
-	if r == nil {
-		return nil, errors.New("invalid request")
+	if r.req.Header.Get("User-Agent") == "" {
+		r.req.Header.Set("User-Agent", "GiteaHttpLib")
 	}
-	return r.getResponse()
-}
 
-// TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field.
-func TimeoutDialer(cTimeout time.Duration) func(ctx context.Context, net, addr string) (c net.Conn, err error) {
-	return func(ctx context.Context, netw, addr string) (net.Conn, error) {
-		d := net.Dialer{Timeout: cTimeout}
-		conn, err := d.DialContext(ctx, netw, addr)
-		if err != nil {
-			return nil, err
-		}
-		return conn, nil
-	}
+	return client.Do(r.req)
 }
 
 func (r *Request) GoString() string {

modules/lfstransfer/backend/backend.go

@@ -157,7 +157,7 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans
 }
 
 // Download implements transfer.Backend. The returned reader must be closed by the caller.
-func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, int64, error) {
+func (g *GiteaBackend) Download(oid string, args transfer.Args) (_ io.ReadCloser, _ int64, retErr error) {
 	idMapStr, exists := args[argID]
 	if !exists {
 		return nil, 0, ErrMissingID
@@ -188,7 +188,15 @@ func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser,
 	if err != nil {
 		return nil, 0, fmt.Errorf("failed to get response: %w", err)
 	}
-	// no need to close the body here by "defer resp.Body.Close()", see below
+	// We must return the ReaderCloser but not "ReadAll", to avoid OOM.
+	// "transfer.Backend" will check io.Closer interface and close the Body reader.
+	// So only close the Body when error occurs
+	defer func() {
+		if retErr != nil {
+			_ = resp.Body.Close()
+		}
+	}()
+
 	if resp.StatusCode != http.StatusOK {
 		return nil, 0, statusCodeToErr(resp.StatusCode)
 	}
@@ -197,7 +205,6 @@ func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser,
 	if err != nil {
 		return nil, 0, fmt.Errorf("failed to parse content length: %w", err)
 	}
-	// transfer.Backend will check io.Closer interface and close this Body reader
 	return resp.Body, respSize, nil
 }