Skip to content

Commit 4cb2e05

Browse files
Merging to release-5.11.1: Merging to release-5.11: [TT-16532] Bundle Verification Significantly Increases Resource Consumption (#7731) (#7732) (#7737)
Merging to release-5.11: [TT-16532] Bundle Verification Significantly Increases Resource Consumption (#7731) (#7732) [TT-16532] Bundle Verification Significantly Increases Resource Consumption (#7731) PR for https://tyktech.atlassian.net/browse/TT-16532 This PR improves the memory consumption of `Verify` method by calculating MD5 and SHA256 hashes incrementally. `TestBundle_Verify` also extended with new cases. We also added a new configuration parameter to GW, called `skip_verify_existing_plugin_bundle` to skip verify on existing bundles on the disk. The default value is `false`. The environment variable is this: `TYK_GW_SKIPVERIFYEXISTINGPLUGINBUNDLE` A benchmark is also added to the PR, you can run the benchmark on your host: ``` go test -bench=BenchmarkBundle_Verify -benchmem -count=5 ./gateway ``` Before the fix: ``` goos: darwin goarch: arm64 pkg: github.com/TykTechnologies/tyk/gateway cpu: Apple M4 Pro BenchmarkBundle_Verify BenchmarkBundle_Verify/empty_file_list BenchmarkBundle_Verify/empty_file_list-12 2589482 446.6 ns/op 112 B/op 4 allocs/op BenchmarkBundle_Verify/single_small_file_1KB BenchmarkBundle_Verify/single_small_file_1KB-12 538592 2144 ns/op 3776 B/op 9 allocs/op BenchmarkBundle_Verify/single_large_file_1MB BenchmarkBundle_Verify/single_large_file_1MB-12 841 1416067 ns/op 4194389 B/op 20 allocs/op BenchmarkBundle_Verify/multiple_files_10x10KB BenchmarkBundle_Verify/multiple_files_10x10KB-12 8576 135169 ns/op 262579 B/op 33 allocs/op PASS ``` After fix: ``` goos: darwin goarch: arm64 pkg: github.com/TykTechnologies/tyk/gateway cpu: Apple M4 Pro BenchmarkBundle_Verify BenchmarkBundle_Verify/empty_file_list BenchmarkBundle_Verify/empty_file_list-12 2702305 410.7 ns/op 296 B/op 5 allocs/op BenchmarkBundle_Verify/single_small_file_1KB BenchmarkBundle_Verify/single_small_file_1KB-12 686098 1674 ns/op 376 B/op 7 allocs/op BenchmarkBundle_Verify/single_large_file_1MB BenchmarkBundle_Verify/single_large_file_1MB-12 1047 1146433 ns/op 377 B/op 7 allocs/op BenchmarkBundle_Verify/multiple_files_10x10KB BenchmarkBundle_Verify/multiple_files_10x10KB-12 10000 112470 ns/op 1099 B/op 25 allocs/op PASS ``` <!---TykTechnologies/jira-linter starts here--> ### Ticket Details <details> <summary> <a href="https://tyktech.atlassian.net/browse/TT-16532" title="TT-16532" target="_blank">TT-16532</a> </summary> | | | |---------|----| | Status | In Test | | Summary | [CRITICAL][regression] Bundle Verification Significantly Increases Resource Consumption | Generated at: 2026-02-03 23:16:53 </details> <!---TykTechnologies/jira-linter ends here--> [TT-16532]: https://tyktech.atlassian.net/browse/TT-16532?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ Co-authored-by: Burak Sezer <burak.sezer.developer@gmail.com> [TT-16532]: https://tyktech.atlassian.net/browse/TT-16532?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ [TT-16532]: https://tyktech.atlassian.net/browse/TT-16532?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ [TT-16532]: https://tyktech.atlassian.net/browse/TT-16532?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ Co-authored-by: probelabs[bot] <219682034+probelabs[bot]@users.noreply.github.com> Co-authored-by: Burak Sezer <burak.sezer.developer@gmail.com>
1 parent 927cc49 commit 4cb2e05

File tree

6 files changed

+248
-21
lines changed

6 files changed

+248
-21
lines changed

cli/linter/schema.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,9 @@
473473
"bundle_insecure_skip_verify": {
474474
"type": "boolean"
475475
},
476+
"skip_verify_existing_plugin_bundle": {
477+
"type": "boolean"
478+
},
476479
"enable_custom_domains": {
477480
"type": "boolean"
478481
},

config/config.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1098,6 +1098,9 @@ type Config struct {
10981098
// Disable TLS validation for bundle URLs
10991099
BundleInsecureSkipVerify bool `bson:"bundle_insecure_skip_verify" json:"bundle_insecure_skip_verify"`
11001100

1101+
// SkipVerifyExistingPluginBundle skips checksum verification for plugin bundles already on disk.
1102+
SkipVerifyExistingPluginBundle bool `bson:"skip_verify_existing_plugin_bundle" json:"skip_verify_existing_plugin_bundle"`
1103+
11011104
// Set to true if you are using JSVM custom middleware or virtual endpoints.
11021105
EnableJSVM bool `json:"enable_jsvm"`
11031106

gateway/coprocess_bundle.go

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"archive/zip"
55
"bytes"
66
"crypto/md5"
7+
"crypto/sha256"
78
"crypto/tls"
89
"encoding/base64"
910
"encoding/json"
@@ -16,6 +17,7 @@ import (
1617
"path"
1718
"path/filepath"
1819
"strings"
20+
"sync"
1921
"time"
2022

2123
"github.com/cenk/backoff"
@@ -44,6 +46,13 @@ type Bundle struct {
4446
Gw *Gateway `json:"-"`
4547
}
4648

49+
var bundleVerifyPool = sync.Pool{
50+
New: func() interface{} {
51+
buffer := make([]byte, 32*1024)
52+
return &buffer
53+
},
54+
}
55+
4756
// Verify performs signature verification on the bundle file.
4857
func (b *Bundle) Verify(bundleFs afero.Fs) error {
4958
log.WithFields(logrus.Fields{
@@ -69,23 +78,31 @@ func (b *Bundle) Verify(bundleFs afero.Fs) error {
6978
}
7079
}
7180

72-
var bundleData bytes.Buffer
81+
md5Hash := md5.New()
82+
sha256Hash := sha256.New()
83+
84+
var w io.Writer = md5Hash
85+
if useSignature {
86+
w = io.MultiWriter(md5Hash, sha256Hash)
87+
}
88+
89+
buf := bundleVerifyPool.Get().(*[]byte)
90+
defer bundleVerifyPool.Put(buf)
7391

7492
for _, f := range b.Manifest.FileList {
7593
extractedPath := filepath.Join(b.Path, f)
76-
77-
f, err := bundleFs.Open(extractedPath)
94+
file, err := bundleFs.Open(extractedPath)
7895
if err != nil {
7996
return err
8097
}
81-
_, err = io.Copy(&bundleData, f)
82-
f.Close()
98+
_, err = io.CopyBuffer(w, file, *buf)
99+
file.Close()
83100
if err != nil {
84101
return err
85102
}
86103
}
87104

88-
checksum := fmt.Sprintf("%x", md5.Sum(bundleData.Bytes()))
105+
checksum := fmt.Sprintf("%x", md5Hash.Sum(nil))
89106
if checksum != b.Manifest.Checksum {
90107
return errors.New("Invalid checksum")
91108
}
@@ -95,7 +112,7 @@ func (b *Bundle) Verify(bundleFs afero.Fs) error {
95112
if err != nil {
96113
return err
97114
}
98-
return verifier.Verify(bundleData.Bytes(), signed)
115+
return verifier.VerifyHash(sha256Hash.Sum(nil), signed)
99116
}
100117
return nil
101118
}
@@ -426,7 +443,7 @@ func (gw *Gateway) loadBundleWithFs(spec *APISpec, bundleFs afero.Fs) error {
426443
Gw: gw,
427444
}
428445

429-
err = loadBundleManifest(bundleFs, &bundle, spec, false)
446+
err = loadBundleManifest(bundleFs, &bundle, spec, gw.GetConfig().SkipVerifyExistingPluginBundle)
430447
if err != nil {
431448
log.WithFields(logrus.Fields{
432449
"prefix": "main",

gateway/coprocess_bundle_test.go

Lines changed: 214 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -498,18 +498,19 @@ func TestBundle_Pull(t *testing.T) {
498498
}
499499

500500
func TestBundle_Verify(t *testing.T) {
501-
502501
tests := []struct {
503-
name string
504-
bundle Bundle
505-
wantErr bool
502+
name string
503+
bundle Bundle
504+
setupFs func(afero.Fs, string)
505+
usePublicKey bool
506+
wantErr bool
507+
wantErrContain string
506508
}{
507509
{
508510
name: "bundle with invalid public key path",
509511
bundle: Bundle{
510512
Name: "test",
511513
Data: []byte("test"),
512-
Path: "/test/test.zip",
513514
Spec: &APISpec{
514515
APIDefinition: &apidef.APIDefinition{
515516
CustomMiddlewareBundle: "test-mw-bundle",
@@ -520,14 +521,14 @@ func TestBundle_Verify(t *testing.T) {
520521
},
521522
Gw: &Gateway{},
522523
},
523-
wantErr: true,
524+
usePublicKey: true,
525+
wantErr: true,
524526
},
525527
{
526528
name: "bundle without signature",
527529
bundle: Bundle{
528530
Name: "test",
529531
Data: []byte("test"),
530-
Path: "/test/test.zip",
531532
Spec: &APISpec{
532533
APIDefinition: &apidef.APIDefinition{
533534
CustomMiddlewareBundle: "test-mw-bundle",
@@ -538,20 +539,142 @@ func TestBundle_Verify(t *testing.T) {
538539
},
539540
Gw: &Gateway{},
540541
},
541-
wantErr: true,
542+
usePublicKey: true,
543+
wantErr: true,
544+
},
545+
{
546+
name: "valid checksum with empty file list",
547+
bundle: Bundle{
548+
Name: "test",
549+
Spec: &APISpec{
550+
APIDefinition: &apidef.APIDefinition{
551+
CustomMiddlewareBundle: "test-mw-bundle",
552+
},
553+
},
554+
Manifest: apidef.BundleManifest{
555+
Checksum: "d41d8cd98f00b204e9800998ecf8427e", // MD5 of the empty string
556+
FileList: []string{},
557+
},
558+
Gw: &Gateway{},
559+
},
560+
usePublicKey: false,
561+
wantErr: false,
562+
},
563+
{
564+
name: "invalid checksum returns error",
565+
bundle: Bundle{
566+
Name: "test",
567+
Spec: &APISpec{
568+
APIDefinition: &apidef.APIDefinition{
569+
CustomMiddlewareBundle: "test-mw-bundle",
570+
},
571+
},
572+
Manifest: apidef.BundleManifest{
573+
Checksum: "invalidchecksum123",
574+
FileList: []string{},
575+
},
576+
Gw: &Gateway{},
577+
},
578+
usePublicKey: false,
579+
wantErr: true,
580+
wantErrContain: "Invalid checksum",
581+
},
582+
{
583+
name: "file not found in file list",
584+
bundle: Bundle{
585+
Name: "test",
586+
Spec: &APISpec{
587+
APIDefinition: &apidef.APIDefinition{
588+
CustomMiddlewareBundle: "test-mw-bundle",
589+
},
590+
},
591+
Manifest: apidef.BundleManifest{
592+
Checksum: "d41d8cd98f00b204e9800998ecf8427e",
593+
FileList: []string{"nonexistent.py"},
594+
},
595+
Gw: &Gateway{},
596+
},
597+
setupFs: func(fs afero.Fs, bundlePath string) {},
598+
usePublicKey: false,
599+
wantErr: true,
600+
},
601+
{
602+
name: "valid checksum with multiple files",
603+
bundle: Bundle{
604+
Name: "test",
605+
Spec: &APISpec{
606+
APIDefinition: &apidef.APIDefinition{
607+
CustomMiddlewareBundle: "test-mw-bundle",
608+
},
609+
},
610+
Manifest: apidef.BundleManifest{
611+
// MD5 of "file1 contentfile2 content"
612+
Checksum: "1510d0e71b31de1c78fd9e823a7c6de9",
613+
FileList: []string{"file1.py", "file2.py"},
614+
},
615+
Gw: &Gateway{},
616+
},
617+
setupFs: func(fs afero.Fs, bundlePath string) {
618+
assert.NoError(t, afero.WriteFile(fs, filepath.Join(bundlePath, "file1.py"), []byte("file1 content"), 0644))
619+
assert.NoError(t, afero.WriteFile(fs, filepath.Join(bundlePath, "file2.py"), []byte("file2 content"), 0644))
620+
},
621+
usePublicKey: false,
622+
wantErr: false,
623+
},
624+
{
625+
name: "invalid base64 signature returns error",
626+
bundle: Bundle{
627+
Name: "test",
628+
Spec: &APISpec{
629+
APIDefinition: &apidef.APIDefinition{
630+
CustomMiddlewareBundle: "test-mw-bundle",
631+
},
632+
},
633+
Manifest: apidef.BundleManifest{
634+
Checksum: "d41d8cd98f00b204e9800998ecf8427e",
635+
FileList: []string{},
636+
Signature: "!!!invalid-base64!!!",
637+
},
638+
Gw: &Gateway{},
639+
},
640+
usePublicKey: true,
641+
wantErr: true,
542642
},
543643
}
544644
for _, tt := range tests {
545645
t.Run(tt.name, func(t *testing.T) {
546646
b := tt.bundle
547647

548648
globalConf := config.Config{}
549-
globalConf.PublicKeyPath = "test"
649+
if tt.usePublicKey {
650+
pemfile := createPEMFile(t)
651+
t.Cleanup(func() {
652+
_ = pemfile.Close()
653+
_ = os.Remove(pemfile.Name())
654+
})
655+
globalConf.PublicKeyPath = pemfile.Name()
656+
}
550657
b.Gw.SetConfig(globalConf)
551658

552-
if err := b.Verify(afero.NewOsFs()); (err != nil) != tt.wantErr {
659+
fs := afero.NewMemMapFs()
660+
bundlePath := "/test/bundles/test-bundle"
661+
b.Path = bundlePath
662+
663+
if err := fs.MkdirAll(bundlePath, 0755); err != nil {
664+
t.Fatalf("failed to create bundle directory: %v", err)
665+
}
666+
667+
if tt.setupFs != nil {
668+
tt.setupFs(fs, bundlePath)
669+
}
670+
671+
err := b.Verify(fs)
672+
if (err != nil) != tt.wantErr {
553673
t.Errorf("Bundle.Verify() error = %v, wantErr %v", err, tt.wantErr)
554674
}
675+
if tt.wantErrContain != "" && err != nil {
676+
assert.ErrorContains(t, err, tt.wantErrContain)
677+
}
555678
})
556679
}
557680
}
@@ -583,3 +706,84 @@ func createPEMFile(t *testing.T) *os.File {
583706

584707
return tmpfile
585708
}
709+
710+
func setupBenchmarkBundle(b *testing.B, fs afero.Fs, bundlePath string, fileSize, numFiles int) *Bundle {
711+
b.Helper()
712+
713+
if err := fs.MkdirAll(bundlePath, 0755); err != nil {
714+
b.Fatalf("failed to create bundle directory: %v", err)
715+
}
716+
717+
fileContent := make([]byte, fileSize)
718+
for i := range fileContent {
719+
fileContent[i] = 'A'
720+
}
721+
722+
fileList := make([]string, numFiles)
723+
md5Hash := md5.New()
724+
725+
for i := 0; i < numFiles; i++ {
726+
fileName := fmt.Sprintf("file%d.py", i)
727+
fileList[i] = fileName
728+
filePath := filepath.Join(bundlePath, fileName)
729+
730+
if err := afero.WriteFile(fs, filePath, fileContent, 0644); err != nil {
731+
b.Fatalf("failed to write file %s: %v", fileName, err)
732+
}
733+
734+
md5Hash.Write(fileContent)
735+
}
736+
737+
checksum := fmt.Sprintf("%x", md5Hash.Sum(nil))
738+
739+
if numFiles == 0 {
740+
checksum = "d41d8cd98f00b204e9800998ecf8427e"
741+
}
742+
743+
return &Bundle{
744+
Name: "benchmark-bundle",
745+
Path: bundlePath,
746+
Spec: &APISpec{
747+
APIDefinition: &apidef.APIDefinition{
748+
CustomMiddlewareBundle: "benchmark-bundle.zip",
749+
},
750+
},
751+
Manifest: apidef.BundleManifest{
752+
Checksum: checksum,
753+
FileList: fileList,
754+
},
755+
Gw: &Gateway{},
756+
}
757+
}
758+
759+
func BenchmarkBundle_Verify(b *testing.B) {
760+
benchmarks := []struct {
761+
name string
762+
fileSize int
763+
numFiles int
764+
}{
765+
{"empty_file_list", 0, 0},
766+
{"single_small_file_1KB", 1024, 1},
767+
{"single_large_file_1MB", 1024 * 1024, 1},
768+
{"multiple_files_10x10KB", 10 * 1024, 10},
769+
}
770+
771+
for _, bm := range benchmarks {
772+
b.Run(bm.name, func(b *testing.B) {
773+
// Setup bundle and filesystem
774+
fs := afero.NewMemMapFs()
775+
bundlePath := "/test/bundles/benchmark-bundle"
776+
bundle := setupBenchmarkBundle(b, fs, bundlePath, bm.fileSize, bm.numFiles)
777+
778+
// Configure GW with no public key (no signature verification)
779+
bundle.Gw.SetConfig(config.Config{})
780+
781+
b.ResetTimer()
782+
b.ReportAllocs()
783+
784+
for i := 0; i < b.N; i++ {
785+
_ = bundle.Verify(fs)
786+
}
787+
})
788+
}
789+
}

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ require (
2424
github.com/TykTechnologies/drl v0.0.0-20231218155806-88e4363884a2
2525
github.com/TykTechnologies/goautosocket v0.0.0-20190430121222-97bfa5e7e481
2626
github.com/TykTechnologies/gorpc v0.0.0-20250214161245-e9f3f088e8c6
27-
github.com/TykTechnologies/goverify v0.0.0-20220808203004-1486f89e7708
27+
github.com/TykTechnologies/goverify v0.0.0-20260203113354-7a104729566e
2828
github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20251104074758-abfe5c458f3b
2929
github.com/TykTechnologies/graphql-translator v0.0.0-20250602105400-41c2e7514a36
3030
github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -812,8 +812,8 @@ github.com/TykTechnologies/goautosocket v0.0.0-20190430121222-97bfa5e7e481 h1:fP
812812
github.com/TykTechnologies/goautosocket v0.0.0-20190430121222-97bfa5e7e481/go.mod h1:CtF8OunV123VfKa8Z9kKcIPHgcd67hSAwFMLlS7FvS4=
813813
github.com/TykTechnologies/gorpc v0.0.0-20250214161245-e9f3f088e8c6 h1:wwt23wdyiZg386taFYNx8NB0jNdtTe66yUcq7vKq9L8=
814814
github.com/TykTechnologies/gorpc v0.0.0-20250214161245-e9f3f088e8c6/go.mod h1:v6v7Mlj08+EmEcXOfpuTxGt2qYU9yhqqtv4QF9Wf50E=
815-
github.com/TykTechnologies/goverify v0.0.0-20220808203004-1486f89e7708 h1:cmXjlMzcexhc/Cg+QB/c2CPUVs1ux9xn6162qaf/LC4=
816-
github.com/TykTechnologies/goverify v0.0.0-20220808203004-1486f89e7708/go.mod h1:mkS8jKcz8otdfEXhJs1QQ/DKoIY1NFFsRPKS0RwQENI=
815+
github.com/TykTechnologies/goverify v0.0.0-20260203113354-7a104729566e h1:wbd35YYCywZbgc4Fk/G2pQHnvwYzjedrw8pNzUsVowg=
816+
github.com/TykTechnologies/goverify v0.0.0-20260203113354-7a104729566e/go.mod h1:WtiSLIlItUVsHyHQB1NG+de/L4/PTtmZWc9CnzTRBLs=
817817
github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20251104074758-abfe5c458f3b h1:Qepgu3gphc87YOx7lQ66m5QRfhXy8Xy5PmSZf1qSOoc=
818818
github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20251104074758-abfe5c458f3b/go.mod h1:bvVafmGebtdjIFG2bXkLd+O1jOjjU/3To+mQHcLr4KI=
819819
github.com/TykTechnologies/graphql-go-tools/v2 v2.0.0-20250926102005-c54e73aae17d h1:bK9T78hExbTuDK4UaBuGi9aL28hK68Uw3OyNZswdPcA=

0 commit comments

Comments
 (0)