Skip to content

Commit 84f12a2

Browse files
Merge branch 'main' into admin-ip-info
2 parents 57771b2 + c57304a commit 84f12a2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+893
-344
lines changed

assets/go-licenses.json

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

custom/conf/app.example.ini

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -940,7 +940,29 @@ LEVEL = Info
940940
;;
941941
;; Disable the code explore page.
942942
;DISABLE_CODE_PAGE = false
943+
944+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
945+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
946+
;[qos]
947+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
948+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
943949
;;
950+
;; Enable request quality of service and overload protection.
951+
; ENABLED = false
952+
;;
953+
;; The maximum number of concurrent requests that the server will
954+
;; process before enqueueing new requests. Default is "CpuNum * 4".
955+
; MAX_INFLIGHT =
956+
;;
957+
;; The maximum number of requests that can be enqueued before new
958+
;; requests will be dropped.
959+
; MAX_WAITING = 100
960+
;;
961+
;; Target maximum wait time a request may be enqueued for. Requests
962+
;; that are enqueued for less than this amount of time will not be
963+
;; dropped. When wait times exceed this amount, a portion of requests
964+
;; will be dropped until wait times have decreased below this amount.
965+
; TARGET_WAIT_TIME = 250ms
944966

945967
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
946968
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1423,7 +1445,6 @@ LEVEL = Info
14231445
;; or use comma separated list: inline-dollar, inline-parentheses, block-dollar, block-square-brackets
14241446
;; Defaults to "inline-dollar,block-dollar" to follow GitHub's behavior.
14251447
;MATH_CODE_BLOCK_DETECTION =
1426-
;;
14271448

14281449
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
14291450
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ require (
3232
github.com/aws/aws-sdk-go-v2/service/codecommit v1.28.1
3333
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
3434
github.com/blevesearch/bleve/v2 v2.4.2
35+
github.com/bohde/codel v0.2.0
3536
github.com/buildkite/terminal-to-html/v3 v3.16.8
3637
github.com/caddyserver/certmagic v0.22.0
3738
github.com/charmbracelet/git-lfs-transfer v0.2.0

go.sum

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi
179179
github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8=
180180
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
181181
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
182+
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
183+
github.com/bohde/codel v0.2.0 h1:fzF7ibgKmCfQbOzQCblmQcwzDRmV7WO7VMLm/hDvD3E=
184+
github.com/bohde/codel v0.2.0/go.mod h1:Idb1IRvTdwkRjIjguLIo+FXhIBhcpGl94o7xra6ggWk=
182185
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
183186
github.com/boombuler/barcode v1.0.2 h1:79yrbttoZrLGkL/oOI8hBrUKucwOL0oOjUgEguGMcJ4=
184187
github.com/boombuler/barcode v1.0.2/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
@@ -881,6 +884,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
881884
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
882885
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
883886
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
887+
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
884888
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
885889
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
886890
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
@@ -1025,6 +1029,8 @@ modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
10251029
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
10261030
mvdan.cc/xurls/v2 v2.6.0 h1:3NTZpeTxYVWNSokW3MKeyVkz/j7uYXYiMtXRUfmjbgI=
10271031
mvdan.cc/xurls/v2 v2.6.0/go.mod h1:bCvEZ1XvdA6wDnxY7jPPjEmigDtvtvPXAD/Exa9IMSk=
1032+
pgregory.net/rapid v0.4.2 h1:lsi9jhvZTYvzVpeG93WWgimPRmiJQfGFRNTEZh1dtY0=
1033+
pgregory.net/rapid v0.4.2/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
10281034
strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 h1:mUcz5b3FJbP5Cvdq7Khzn6J9OCUQJaBwgBkCR+MOwSs=
10291035
strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1:FJGmPh3vz9jSos1L/F91iAgnC/aejc0wIIrF2ZwJxdY=
10301036
xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo=

models/packages/descriptor.go

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
repo_model "code.gitea.io/gitea/models/repo"
1313
user_model "code.gitea.io/gitea/models/user"
14+
"code.gitea.io/gitea/modules/cache"
1415
"code.gitea.io/gitea/modules/json"
1516
"code.gitea.io/gitea/modules/packages/alpine"
1617
"code.gitea.io/gitea/modules/packages/arch"
@@ -102,22 +103,26 @@ func (pd *PackageDescriptor) CalculateBlobSize() int64 {
102103

103104
// GetPackageDescriptor gets the package description for a version
104105
func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDescriptor, error) {
105-
p, err := GetPackageByID(ctx, pv.PackageID)
106+
return getPackageDescriptor(ctx, pv, cache.NewEphemeralCache())
107+
}
108+
109+
func getPackageDescriptor(ctx context.Context, pv *PackageVersion, c *cache.EphemeralCache) (*PackageDescriptor, error) {
110+
p, err := cache.GetWithEphemeralCache(ctx, c, "package", pv.PackageID, GetPackageByID)
106111
if err != nil {
107112
return nil, err
108113
}
109-
o, err := user_model.GetUserByID(ctx, p.OwnerID)
114+
o, err := cache.GetWithEphemeralCache(ctx, c, "user", p.OwnerID, user_model.GetUserByID)
110115
if err != nil {
111116
return nil, err
112117
}
113118
var repository *repo_model.Repository
114119
if p.RepoID > 0 {
115-
repository, err = repo_model.GetRepositoryByID(ctx, p.RepoID)
120+
repository, err = cache.GetWithEphemeralCache(ctx, c, "repo", p.RepoID, repo_model.GetRepositoryByID)
116121
if err != nil && !repo_model.IsErrRepoNotExist(err) {
117122
return nil, err
118123
}
119124
}
120-
creator, err := user_model.GetUserByID(ctx, pv.CreatorID)
125+
creator, err := cache.GetWithEphemeralCache(ctx, c, "user", pv.CreatorID, user_model.GetUserByID)
121126
if err != nil {
122127
if errors.Is(err, util.ErrNotExist) {
123128
creator = user_model.NewGhostUser()
@@ -145,9 +150,13 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
145150
return nil, err
146151
}
147152

148-
pfds, err := GetPackageFileDescriptors(ctx, pfs)
149-
if err != nil {
150-
return nil, err
153+
pfds := make([]*PackageFileDescriptor, 0, len(pfs))
154+
for _, pf := range pfs {
155+
pfd, err := getPackageFileDescriptor(ctx, pf, c)
156+
if err != nil {
157+
return nil, err
158+
}
159+
pfds = append(pfds, pfd)
151160
}
152161

153162
var metadata any
@@ -221,7 +230,11 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
221230

222231
// GetPackageFileDescriptor gets a package file descriptor for a package file
223232
func GetPackageFileDescriptor(ctx context.Context, pf *PackageFile) (*PackageFileDescriptor, error) {
224-
pb, err := GetBlobByID(ctx, pf.BlobID)
233+
return getPackageFileDescriptor(ctx, pf, cache.NewEphemeralCache())
234+
}
235+
236+
func getPackageFileDescriptor(ctx context.Context, pf *PackageFile, c *cache.EphemeralCache) (*PackageFileDescriptor, error) {
237+
pb, err := cache.GetWithEphemeralCache(ctx, c, "package_file_blob", pf.BlobID, GetBlobByID)
225238
if err != nil {
226239
return nil, err
227240
}
@@ -251,9 +264,13 @@ func GetPackageFileDescriptors(ctx context.Context, pfs []*PackageFile) ([]*Pack
251264

252265
// GetPackageDescriptors gets the package descriptions for the versions
253266
func GetPackageDescriptors(ctx context.Context, pvs []*PackageVersion) ([]*PackageDescriptor, error) {
267+
return getPackageDescriptors(ctx, pvs, cache.NewEphemeralCache())
268+
}
269+
270+
func getPackageDescriptors(ctx context.Context, pvs []*PackageVersion, c *cache.EphemeralCache) ([]*PackageDescriptor, error) {
254271
pds := make([]*PackageDescriptor, 0, len(pvs))
255272
for _, pv := range pvs {
256-
pd, err := GetPackageDescriptor(ctx, pv)
273+
pd, err := getPackageDescriptor(ctx, pv, c)
257274
if err != nil {
258275
return nil, err
259276
}

modules/cache/context.go

Lines changed: 19 additions & 156 deletions
Original file line numberDiff line numberDiff line change
@@ -5,176 +5,39 @@ package cache
55

66
import (
77
"context"
8-
"sync"
98
"time"
10-
11-
"code.gitea.io/gitea/modules/log"
129
)
1310

14-
// cacheContext is a context that can be used to cache data in a request level context
15-
// This is useful for caching data that is expensive to calculate and is likely to be
16-
// used multiple times in a request.
17-
type cacheContext struct {
18-
data map[any]map[any]any
19-
lock sync.RWMutex
20-
created time.Time
21-
discard bool
22-
}
23-
24-
func (cc *cacheContext) Get(tp, key any) any {
25-
cc.lock.RLock()
26-
defer cc.lock.RUnlock()
27-
return cc.data[tp][key]
28-
}
29-
30-
func (cc *cacheContext) Put(tp, key, value any) {
31-
cc.lock.Lock()
32-
defer cc.lock.Unlock()
33-
34-
if cc.discard {
35-
return
36-
}
37-
38-
d := cc.data[tp]
39-
if d == nil {
40-
d = make(map[any]any)
41-
cc.data[tp] = d
42-
}
43-
d[key] = value
44-
}
45-
46-
func (cc *cacheContext) Delete(tp, key any) {
47-
cc.lock.Lock()
48-
defer cc.lock.Unlock()
49-
delete(cc.data[tp], key)
50-
}
11+
type cacheContextKeyType struct{}
5112

52-
func (cc *cacheContext) Discard() {
53-
cc.lock.Lock()
54-
defer cc.lock.Unlock()
55-
cc.data = nil
56-
cc.discard = true
57-
}
13+
var cacheContextKey = cacheContextKeyType{}
5814

59-
func (cc *cacheContext) isDiscard() bool {
60-
cc.lock.RLock()
61-
defer cc.lock.RUnlock()
62-
return cc.discard
63-
}
64-
65-
// cacheContextLifetime is the max lifetime of cacheContext.
66-
// Since cacheContext is used to cache data in a request level context, 5 minutes is enough.
67-
// If a cacheContext is used more than 5 minutes, it's probably misuse.
68-
const cacheContextLifetime = 5 * time.Minute
69-
70-
var timeNow = time.Now
71-
72-
func (cc *cacheContext) Expired() bool {
73-
return timeNow().Sub(cc.created) > cacheContextLifetime
74-
}
75-
76-
var cacheContextKey = struct{}{}
77-
78-
/*
79-
Since there are both WithCacheContext and WithNoCacheContext,
80-
it may be confusing when there is nesting.
81-
82-
Some cases to explain the design:
83-
84-
When:
85-
- A, B or C means a cache context.
86-
- A', B' or C' means a discard cache context.
87-
- ctx means context.Backgrand().
88-
- A(ctx) means a cache context with ctx as the parent context.
89-
- B(A(ctx)) means a cache context with A(ctx) as the parent context.
90-
- With is alias of WithCacheContext.
91-
- WithNo is alias of WithNoCacheContext.
92-
93-
So:
94-
- With(ctx) -> A(ctx)
95-
- With(With(ctx)) -> A(ctx), not B(A(ctx)), always reuse parent cache context if possible.
96-
- With(With(With(ctx))) -> A(ctx), not C(B(A(ctx))), ditto.
97-
- WithNo(ctx) -> ctx, not A'(ctx), don't create new cache context if we don't have to.
98-
- WithNo(With(ctx)) -> A'(ctx)
99-
- WithNo(WithNo(With(ctx))) -> A'(ctx), not B'(A'(ctx)), don't create new cache context if we don't have to.
100-
- With(WithNo(With(ctx))) -> B(A'(ctx)), not A(ctx), never reuse a discard cache context.
101-
- WithNo(With(WithNo(With(ctx)))) -> B'(A'(ctx))
102-
- With(WithNo(With(WithNo(With(ctx))))) -> C(B'(A'(ctx))), so there's always only one not-discard cache context.
103-
*/
15+
// contextCacheLifetime is the max lifetime of context cache.
16+
// Since context cache is used to cache data in a request level context, 5 minutes is enough.
17+
// If a context cache is used more than 5 minutes, it's probably abused.
18+
const contextCacheLifetime = 5 * time.Minute
10419

10520
func WithCacheContext(ctx context.Context) context.Context {
106-
if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
107-
if !c.isDiscard() {
108-
// reuse parent context
109-
return ctx
110-
}
111-
}
112-
// FIXME: review the use of this nolint directive
113-
return context.WithValue(ctx, cacheContextKey, &cacheContext{ //nolint:staticcheck
114-
data: make(map[any]map[any]any),
115-
created: timeNow(),
116-
})
117-
}
118-
119-
func WithNoCacheContext(ctx context.Context) context.Context {
120-
if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
121-
// The caller want to run long-life tasks, but the parent context is a cache context.
122-
// So we should disable and clean the cache data, or it will be kept in memory for a long time.
123-
c.Discard()
21+
if c := GetContextCache(ctx); c != nil {
12422
return ctx
12523
}
126-
127-
return ctx
24+
return context.WithValue(ctx, cacheContextKey, NewEphemeralCache(contextCacheLifetime))
12825
}
12926

130-
func GetContextData(ctx context.Context, tp, key any) any {
131-
if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
132-
if c.Expired() {
133-
// The warning means that the cache context is misused for long-life task,
134-
// it can be resolved with WithNoCacheContext(ctx).
135-
log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c)
136-
return nil
137-
}
138-
return c.Get(tp, key)
139-
}
140-
return nil
141-
}
142-
143-
func SetContextData(ctx context.Context, tp, key, value any) {
144-
if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
145-
if c.Expired() {
146-
// The warning means that the cache context is misused for long-life task,
147-
// it can be resolved with WithNoCacheContext(ctx).
148-
log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c)
149-
return
150-
}
151-
c.Put(tp, key, value)
152-
return
153-
}
154-
}
155-
156-
func RemoveContextData(ctx context.Context, tp, key any) {
157-
if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
158-
if c.Expired() {
159-
// The warning means that the cache context is misused for long-life task,
160-
// it can be resolved with WithNoCacheContext(ctx).
161-
log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c)
162-
return
163-
}
164-
c.Delete(tp, key)
165-
}
27+
func GetContextCache(ctx context.Context) *EphemeralCache {
28+
c, _ := ctx.Value(cacheContextKey).(*EphemeralCache)
29+
return c
16630
}
16731

16832
// GetWithContextCache returns the cache value of the given key in the given context.
33+
// FIXME: in some cases, the "context cache" should not be used, because it has uncontrollable behaviors
34+
// For example, these calls:
35+
// * GetWithContextCache(TargetID) -> OtherCodeCreateModel(TargetID) -> GetWithContextCache(TargetID)
36+
// Will cause the second call is not able to get the correct created target.
37+
// UNLESS it is certain that the target won't be changed during the request, DO NOT use it.
16938
func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) {
170-
v := GetContextData(ctx, groupKey, targetKey)
171-
if vv, ok := v.(T); ok {
172-
return vv, nil
173-
}
174-
t, err := f(ctx, targetKey)
175-
if err != nil {
176-
return t, err
39+
if c := GetContextCache(ctx); c != nil {
40+
return GetWithEphemeralCache(ctx, c, groupKey, targetKey, f)
17741
}
178-
SetContextData(ctx, groupKey, targetKey, t)
179-
return t, nil
42+
return f(ctx, targetKey)
18043
}

0 commit comments

Comments
 (0)