From c5f251b83f98b75614b34208e19fb46c5539335e Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Sun, 31 Aug 2025 21:32:19 +0000 Subject: [PATCH 01/28] use experimental go json v2 library --- Makefile | 3 ++ go.mod | 2 +- modules/json/json.go | 54 ++++++++++++++++++++++++- modules/json/jsonv2.go | 72 +++++++++++++++++++++++++++++++++ modules/json/jsonv2_fallback.go | 33 +++++++++++++++ 5 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 modules/json/jsonv2.go create mode 100644 modules/json/jsonv2_fallback.go diff --git a/Makefile b/Makefile index 9cd32e4c33f9d..9815bfefed1f2 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ DIST := dist DIST_DIRS := $(DIST)/binaries $(DIST)/release IMPORT := code.gitea.io/gitea +# No experiment set by default, but you can set jsonv2 to use go 1.25.0 json v2 experimental changes +export GOEXPERIMENT ?= + GO ?= go SHASUM ?= shasum -a 256 HAS_GO := $(shell hash $(GO) > /dev/null 2>&1 && echo yes) diff --git a/go.mod b/go.mod index 164702c90e5f5..17e6d86eb3cf8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module code.gitea.io/gitea -go 1.24.6 +go 1.25 // rfc5280 said: "The serial number is an integer assigned by the CA to each certificate." // But some CAs use negative serial number, just relax the check. related: diff --git a/modules/json/json.go b/modules/json/json.go index 444dc8526aeab..185897648ebb7 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -32,13 +32,22 @@ type Interface interface { } var ( - // DefaultJSONHandler default json handler - DefaultJSONHandler Interface = JSONiter{jsoniter.ConfigCompatibleWithStandardLibrary} + // DefaultJSONHandler default json handler - uses JSON v2 if available, otherwise JSONiter + DefaultJSONHandler = getDefaultHandler() _ Interface = StdJSON{} _ Interface = JSONiter{} + _ Interface = JSONv2{} ) +// getDefaultHandler returns the expected JSON implementation +func getDefaultHandler() Interface { + if isJSONv2Available() { + return JSONv2{} + } + return JSONiter{jsoniter.ConfigCompatibleWithStandardLibrary} +} + // StdJSON implements Interface via encoding/json type StdJSON struct{} @@ -97,6 +106,47 @@ func (j JSONiter) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) e return json.Indent(dst, src, prefix, indent) } +// JSONv2 implements Interface via encoding/json/v2 +// Requires GOEXPERIMENT=jsonv2 to be set at build time +type JSONv2 struct{} + +// Marshal implements Interface using JSON v2 - fallback if v2 is not available +func (JSONv2) Marshal(v any) ([]byte, error) { + if !isJSONv2Available() { + return json.Marshal(v) + } + return marshalV2(v) +} + +// Unmarshal implements Interface using JSON v2 - fallback if v2 is not available +func (JSONv2) Unmarshal(data []byte, v any) error { + if !isJSONv2Available() { + return json.Unmarshal(data, v) + } + return unmarshalV2(data, v) +} + +// NewEncoder implements Interface using JSON v2 - fallback if v2 is not available +func (JSONv2) NewEncoder(writer io.Writer) Encoder { + if !isJSONv2Available() { + return json.NewEncoder(writer) + } + return newEncoderV2(writer) +} + +// NewDecoder implements Interface using JSON v2 - fallback if v2 is not available +func (JSONv2) NewDecoder(reader io.Reader) Decoder { + if !isJSONv2Available() { + return json.NewDecoder(reader) + } + return newDecoderV2(reader) +} + +// Indent implements Interface using standard library (JSON v2 doesn't have Indent yet) +func (JSONv2) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { + return json.Indent(dst, src, prefix, indent) +} + // Marshal converts object as bytes func Marshal(v any) ([]byte, error) { return DefaultJSONHandler.Marshal(v) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go new file mode 100644 index 0000000000000..93b618a26e425 --- /dev/null +++ b/modules/json/jsonv2.go @@ -0,0 +1,72 @@ +//go:build goexperiment.jsonv2 + +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package json + +import ( + jsonv2 "encoding/json/v2" + "io" +) + +// isJSONv2Available returns true when JSON v2 is available (compiled with GOEXPERIMENT=jsonv2) +func isJSONv2Available() bool { + return true +} + +// marshalV2 uses JSON v2 marshal with v1 compatibility options +func marshalV2(v any) ([]byte, error) { + opts := jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + jsonv2.FormatNilSliceAsNull(true), + jsonv2.FormatNilMapAsNull(true), + ) + return jsonv2.Marshal(v, opts) +} + +// unmarshalV2 uses JSON v2 unmarshal with v1 compatibility options +func unmarshalV2(data []byte, v any) error { + opts := jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + ) + return jsonv2.Unmarshal(data, v, opts) +} + +// encoderV2 wraps JSON v2 streaming encoder +type encoderV2 struct { + writer io.Writer + opts jsonv2.Options +} + +func (e *encoderV2) Encode(v any) error { + return jsonv2.MarshalWrite(e.writer, v, e.opts) +} + +// newEncoderV2 creates a new JSON v2 streaming encoder +func newEncoderV2(writer io.Writer) Encoder { + opts := jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + jsonv2.FormatNilSliceAsNull(true), + jsonv2.FormatNilMapAsNull(true), + ) + return &encoderV2{writer: writer, opts: opts} +} + +// decoderV2 wraps JSON v2 streaming decoder +type decoderV2 struct { + reader io.Reader + opts jsonv2.Options +} + +func (d *decoderV2) Decode(v any) error { + return jsonv2.UnmarshalRead(d.reader, v, d.opts) +} + +// newDecoderV2 creates a new JSON v2 streaming decoder +func newDecoderV2(reader io.Reader) Decoder { + opts := jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + ) + return &decoderV2{reader: reader, opts: opts} +} diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go new file mode 100644 index 0000000000000..326d2f8b4652c --- /dev/null +++ b/modules/json/jsonv2_fallback.go @@ -0,0 +1,33 @@ +//go:build !goexperiment.jsonv2 + +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package json + +import "io" + +// isJSONv2Available returns false when JSON v2 is not available (not compiled with GOEXPERIMENT=jsonv2) +func isJSONv2Available() bool { + return false +} + +// marshalV2 fallback - should not be called when JSON v2 is not available +func marshalV2(v any) ([]byte, error) { + panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") +} + +// unmarshalV2 fallback - should not be called when JSON v2 is not available +func unmarshalV2(data []byte, v any) error { + panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") +} + +// newEncoderV2 fallback - should not be called when JSON v2 is not available +func newEncoderV2(writer io.Writer) Encoder { + panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") +} + +// newDecoderV2 fallback - should not be called when JSON v2 is not available +func newDecoderV2(reader io.Reader) Decoder { + panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") +} From abc734cb13572d4ae287a81a623715c043401499 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:50:37 -0400 Subject: [PATCH 02/28] Update go.mod Signed-off-by: techknowlogick --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4cc2e8a925acf..38bd1e9b722b2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module code.gitea.io/gitea -go 1.25 +go 1.24.6 // rfc5280 said: "The serial number is an integer assigned by the CA to each certificate." // But some CAs use negative serial number, just relax the check. related: From f9c880570816d5549cd86925626665a6742f163b Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:53:20 -0400 Subject: [PATCH 03/28] Update jsonv2.go Signed-off-by: techknowlogick --- modules/json/jsonv2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 93b618a26e425..5ad7f8db1c936 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -1,4 +1,3 @@ -//go:build goexperiment.jsonv2 // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT From 9d83440d597ed498d3cbc72e9a4df5424ba60b07 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:54:22 -0400 Subject: [PATCH 04/28] Update jsonv2.go Signed-off-by: techknowlogick --- modules/json/jsonv2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 5ad7f8db1c936..6178ba79ff56b 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -1,4 +1,3 @@ - // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT From 31492c12fd310c8bf33825310bac83d87939efea Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:55:20 -0400 Subject: [PATCH 05/28] Update jsonv2.go Signed-off-by: techknowlogick --- modules/json/jsonv2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 6178ba79ff56b..99d8e7f67438d 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -1,6 +1,7 @@ // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT +//go:build goexperiment.jsonv2 package json import ( From 7b0832cd2f39d06a6f53779b688f40bd6df5e7a4 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:56:15 -0400 Subject: [PATCH 06/28] Update jsonv2.go Signed-off-by: techknowlogick --- modules/json/jsonv2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 99d8e7f67438d..e3faedda83a6a 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT //go:build goexperiment.jsonv2 + package json import ( From fe51938ef7293cad6fdfcdc181ad634fbfb63fcd Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:57:43 -0400 Subject: [PATCH 07/28] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index 326d2f8b4652c..d46f49bf4fbab 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -3,6 +3,7 @@ // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT +//go:build goexperiment.jsonv2 package json import "io" From 29d17705fb595ca6061084499d6ba63285e63f92 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:57:50 -0400 Subject: [PATCH 08/28] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index d46f49bf4fbab..b6ca8af1a662f 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -1,5 +1,4 @@ //go:build !goexperiment.jsonv2 - // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT From 3c3f77fe8e568f41b582f846a494d2b2b5301504 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:57:57 -0400 Subject: [PATCH 09/28] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index b6ca8af1a662f..73bbb957ab073 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -1,4 +1,3 @@ -//go:build !goexperiment.jsonv2 // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT From b8fb94decae58c2340851d2ddbc6483805bd84ae Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 20:01:20 -0400 Subject: [PATCH 10/28] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index 73bbb957ab073..985300dbd73b9 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -1,7 +1,7 @@ // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build goexperiment.jsonv2 +//go:build !goexperiment.jsonv2 package json import "io" From c748ca8cda652b043860cb0a304b4e50a7b54690 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 20:12:09 -0400 Subject: [PATCH 11/28] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index 985300dbd73b9..e78893ed45b88 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT //go:build !goexperiment.jsonv2 + package json import "io" From 7fec8381119f364ad9792b7284ae05c0585cfd17 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 11:08:25 -0400 Subject: [PATCH 12/28] Update Go version from 1.24.6 to 1.25.0 Signed-off-by: techknowlogick --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index cadb23f069644..09b150e3e5dde 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module code.gitea.io/gitea -go 1.24.6 +go 1.25.0 // rfc5280 said: "The serial number is an integer assigned by the CA to each certificate." // But some CAs use negative serial number, just relax the check. related: From 3c6d690dadb8667b900a437dbef82133ac4b7c88 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 11:56:18 -0400 Subject: [PATCH 13/28] bump go.mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 09b150e3e5dde..3661d051a9b29 100644 --- a/go.mod +++ b/go.mod @@ -278,7 +278,7 @@ require ( go.uber.org/zap v1.27.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect + golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b // indirect golang.org/x/mod v0.27.0 // indirect golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.36.0 // indirect diff --git a/go.sum b/go.sum index b69b380cfe2ab..3e59115f04de2 100644 --- a/go.sum +++ b/go.sum @@ -850,8 +850,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= -golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= +golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0= +golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.30.0 h1:jD5RhkmVAnjqaCUXfbGBrn3lpxbknfN9w2UhHHU+5B4= From b3969becbf2c2a78a89b903dc8bb49ea67d06323 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 11:57:39 -0400 Subject: [PATCH 14/28] use fixed go-swagger version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4b089821e1377..5651a01af470e 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ GOFUMPT_PACKAGE ?= mvdan.cc/gofumpt@v0.8.0 GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.4.0 GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.15 MISSPELL_PACKAGE ?= github.com/golangci/misspell/cmd/misspell@v0.7.0 -SWAGGER_PACKAGE ?= github.com/go-swagger/go-swagger/cmd/swagger@v0.32.3 +SWAGGER_PACKAGE ?= github.com/go-swagger/go-swagger/cmd/swagger@717e3cb29becaaf00e56953556c6d80f8a01b286 XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest GO_LICENSES_PACKAGE ?= github.com/google/go-licenses@v1 GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1 From 94d537c8450392bc6a77ce0c0d2627f6908d69ae Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 12:06:06 -0400 Subject: [PATCH 15/28] Update GOEXPERIMENT to use jsonv2 by default Signed-off-by: techknowlogick --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5651a01af470e..d5136db68fafb 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,9 @@ DIST := dist DIST_DIRS := $(DIST)/binaries $(DIST)/release IMPORT := code.gitea.io/gitea -# No experiment set by default, but you can set jsonv2 to use go 1.25.0 json v2 experimental changes -export GOEXPERIMENT ?= +# By default use go's 1.25 experimental json v2 library when building +# TODO: remove when no longer experimental +export GOEXPERIMENT ?= jsonv2 GO ?= go SHASUM ?= shasum -a 256 From 82b2d977e97456d1285b911d79f4364743ddafc8 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 12:26:35 -0400 Subject: [PATCH 16/28] fix lint --- modules/globallock/locker_test.go | 6 ++---- modules/json/jsonv2.go | 2 +- modules/log/event_writer_conn_test.go | 6 ++---- modules/queue/workergroup.go | 6 ++---- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/modules/globallock/locker_test.go b/modules/globallock/locker_test.go index c9e73c25d2e5b..14cb0ec388898 100644 --- a/modules/globallock/locker_test.go +++ b/modules/globallock/locker_test.go @@ -105,15 +105,13 @@ func testLocker(t *testing.T, locker Locker) { require.NoError(t, err) wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { started := time.Now() release, err := locker.Lock(t.Context(), "test") // should be blocked for seconds defer release() assert.Greater(t, time.Since(started), time.Second) assert.NoError(t, err) - }() + }) time.Sleep(2 * time.Second) release() diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index e3faedda83a6a..73ec03523b16c 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -6,7 +6,7 @@ package json import ( - jsonv2 "encoding/json/v2" + jsonv2 "encoding/json/v2" //nolint:depguard // this package wraps it "io" ) diff --git a/modules/log/event_writer_conn_test.go b/modules/log/event_writer_conn_test.go index 2aff37812d639..e7011da79cb8b 100644 --- a/modules/log/event_writer_conn_test.go +++ b/modules/log/event_writer_conn_test.go @@ -62,11 +62,9 @@ func TestConnLogger(t *testing.T) { } expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.Filename, event.Line, event.Caller, strings.ToUpper(event.Level.String())[0], event.MsgSimpleText) var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { listenReadAndClose(t, l, expected) - }() + }) logger.SendLogEvent(&event) wg.Wait() diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go index 82b0790d5a9c2..3910179a6bb42 100644 --- a/modules/queue/workergroup.go +++ b/modules/queue/workergroup.go @@ -153,10 +153,8 @@ func resetIdleTicker(t *time.Ticker, dur time.Duration) { // doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { - wp.wg.Add(1) - go func() { - defer wp.wg.Done() + &{wp wg}.Go(func() { log.Debug("Queue %q starts new worker", q.GetName()) defer log.Debug("Queue %q stops idle worker", q.GetName()) @@ -192,7 +190,7 @@ func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { q.workerNumMu.Unlock() } } - }() + }) } // doFlush flushes the queue: it tries to read all items from the queue and handles them. From 502b4fb4809f0affc899e1aab6d726b87bea280f Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 12:39:28 -0400 Subject: [PATCH 17/28] clean up modernizer fixes --- modules/queue/workergroup.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go index 3910179a6bb42..e76c1ebc1e2a9 100644 --- a/modules/queue/workergroup.go +++ b/modules/queue/workergroup.go @@ -153,9 +153,7 @@ func resetIdleTicker(t *time.Ticker, dur time.Duration) { // doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { - - &{wp wg}.Go(func() { - + &wp.wg.Go(func() { log.Debug("Queue %q starts new worker", q.GetName()) defer log.Debug("Queue %q stops idle worker", q.GetName()) From d4b1e10f3195dde2b39cfd584865cd83757c3561 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 12:50:11 -0400 Subject: [PATCH 18/28] try to fix lint --- modules/queue/workergroup.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go index e76c1ebc1e2a9..76435ba32ad42 100644 --- a/modules/queue/workergroup.go +++ b/modules/queue/workergroup.go @@ -153,7 +153,9 @@ func resetIdleTicker(t *time.Ticker, dur time.Duration) { // doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { - &wp.wg.Go(func() { + wp.wg.Add(1) + go func() { + defer wp.wg.Done() log.Debug("Queue %q starts new worker", q.GetName()) defer log.Debug("Queue %q stops idle worker", q.GetName()) @@ -188,7 +190,7 @@ func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { q.workerNumMu.Unlock() } } - }) + }() } // doFlush flushes the queue: it tries to read all items from the queue and handles them. From 8f3b21892e24e01c62101b95b5c1e2a8b79124f5 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 13:23:51 -0400 Subject: [PATCH 19/28] try to use go1.25 waitgroup logic --- modules/queue/workergroup.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go index 76435ba32ad42..c7e33497c6416 100644 --- a/modules/queue/workergroup.go +++ b/modules/queue/workergroup.go @@ -153,9 +153,7 @@ func resetIdleTicker(t *time.Ticker, dur time.Duration) { // doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { - wp.wg.Add(1) - go func() { - defer wp.wg.Done() + wp.wg.Go(func() { log.Debug("Queue %q starts new worker", q.GetName()) defer log.Debug("Queue %q stops idle worker", q.GetName()) @@ -190,7 +188,7 @@ func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { q.workerNumMu.Unlock() } } - }() + }) } // doFlush flushes the queue: it tries to read all items from the queue and handles them. From 34af20c731d6a06fec8c85037d97e94e781da673 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 13:50:35 -0400 Subject: [PATCH 20/28] fixup test fails --- modules/json/json.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/json/json.go b/modules/json/json.go index 185897648ebb7..656a1060e0eaf 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -200,7 +200,7 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { // To be consistent, we should treat all empty inputs as success return nil } - err := json.Unmarshal(bs, v) + err := DefaultJSONHandler.Unmarshal(bs, v) if err != nil { ok := true rs := []byte{} @@ -217,11 +217,11 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { if len(rs) > 1 && rs[0] == 0xff && rs[1] == 0xfe { rs = rs[2:] } - err = json.Unmarshal(rs, v) + err = DefaultJSONHandler.Unmarshal(rs, v) } } if err != nil && len(bs) > 2 && bs[0] == 0xff && bs[1] == 0xfe { - err = json.Unmarshal(bs[2:], v) + err = DefaultJSONHandler.Unmarshal(bs[2:], v) } return err } From cdcb9bda2efb2d6261ed41adcecf8209187a1eba Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Wed, 10 Sep 2025 23:37:27 +0000 Subject: [PATCH 21/28] adjust jsonv2 output to become similar to v1 --- models/webhook/webhook.go | 9 ++++++- modules/json/json.go | 8 ++++++- modules/json/jsonv2.go | 41 ++++++++++++++++++++++++++++---- services/webhook/deliver_test.go | 4 ++-- services/webhook/matrix_test.go | 2 +- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/models/webhook/webhook.go b/models/webhook/webhook.go index 7d4b2e2237db0..794608e970b39 100644 --- a/models/webhook/webhook.go +++ b/models/webhook/webhook.go @@ -150,7 +150,14 @@ func init() { // AfterLoad updates the webhook object upon setting a column func (w *Webhook) AfterLoad() { w.HookEvent = &webhook_module.HookEvent{} - if err := json.Unmarshal([]byte(w.Events), w.HookEvent); err != nil { + + events := w.Events + if events == "" { + // jsonv2 is unable to unmarshal an empty string + return + } + + if err := json.Unmarshal([]byte(events), w.HookEvent); err != nil { log.Error("Unmarshal[%d]: %v", w.ID, err) } } diff --git a/modules/json/json.go b/modules/json/json.go index 656a1060e0eaf..c736e5b7232bf 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -174,7 +174,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { // MarshalIndent copied from encoding/json func MarshalIndent(v any, prefix, indent string) ([]byte, error) { - b, err := Marshal(v) + b, err := DefaultJSONHandler.Marshal(v) if err != nil { return nil, err } @@ -200,6 +200,12 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { // To be consistent, we should treat all empty inputs as success return nil } + + trimmed := bytes.TrimSpace(bs) + if len(trimmed) == 0 { + return nil + } + err := DefaultJSONHandler.Unmarshal(bs, v) if err != nil { ok := true diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 73ec03523b16c..9b41af2e34cb6 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -6,6 +6,7 @@ package json import ( + "bytes" jsonv2 "encoding/json/v2" //nolint:depguard // this package wraps it "io" ) @@ -15,18 +16,38 @@ func isJSONv2Available() bool { return true } -// marshalV2 uses JSON v2 marshal with v1 compatibility options -func marshalV2(v any) ([]byte, error) { +// marshalV2Internal uses JSON v2 marshal with v1 compatibility options (no trailing newline) +func marshalV2Internal(v any) ([]byte, error) { opts := jsonv2.JoinOptions( jsonv2.MatchCaseInsensitiveNames(true), jsonv2.FormatNilSliceAsNull(true), jsonv2.FormatNilMapAsNull(true), + jsonv2.Deterministic(true), ) return jsonv2.Marshal(v, opts) } +// marshalV2 uses JSON v2 marshal with v1 compatibility options (with trailing newline for compatibility with standard library) +func marshalV2(v any) ([]byte, error) { + result, err := marshalV2Internal(v) + if err != nil { + return nil, err + } + + return append(result, '\n'), nil +} + // unmarshalV2 uses JSON v2 unmarshal with v1 compatibility options func unmarshalV2(data []byte, v any) error { + if len(data) == 0 { + return nil + } + + data = bytes.TrimSpace(data) + if len(data) == 0 { + return nil + } + opts := jsonv2.JoinOptions( jsonv2.MatchCaseInsensitiveNames(true), ) @@ -40,7 +61,13 @@ type encoderV2 struct { } func (e *encoderV2) Encode(v any) error { - return jsonv2.MarshalWrite(e.writer, v, e.opts) + err := jsonv2.MarshalWrite(e.writer, v, e.opts) + if err != nil { + return err + } + + _, err = e.writer.Write([]byte{'\n'}) + return err } // newEncoderV2 creates a new JSON v2 streaming encoder @@ -49,6 +76,7 @@ func newEncoderV2(writer io.Writer) Encoder { jsonv2.MatchCaseInsensitiveNames(true), jsonv2.FormatNilSliceAsNull(true), jsonv2.FormatNilMapAsNull(true), + jsonv2.Deterministic(true), ) return &encoderV2{writer: writer, opts: opts} } @@ -60,7 +88,12 @@ type decoderV2 struct { } func (d *decoderV2) Decode(v any) error { - return jsonv2.UnmarshalRead(d.reader, v, d.opts) + err := jsonv2.UnmarshalRead(d.reader, v, d.opts) + // Handle EOF more gracefully to match standard library behavior + if err != nil && err.Error() == "unexpected EOF" { + return io.EOF + } + return err } // newDecoderV2 creates a new JSON v2 streaming decoder diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index efbef1fc89444..b0e83175747fb 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -142,13 +142,13 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.NoError(t, err) assert.Equal(t, `{"data": 42}`, string(body)) - case "/webhook/6db5dc1e282529a8c162c7fe93dd2667494eeb51": + case "/webhook/4ddf3b1533e54f082ae6eadfc1b5530be36c8893": // Version 2 assert.Equal(t, "push", r.Header.Get("X-GitHub-Event")) assert.Equal(t, "application/json", r.Header.Get("Content-Type")) body, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.Len(t, body, 2147) + assert.Len(t, body, 2047) default: w.WriteHeader(http.StatusNotFound) diff --git a/services/webhook/matrix_test.go b/services/webhook/matrix_test.go index d36d93c5a7385..c39867b4816f0 100644 --- a/services/webhook/matrix_test.go +++ b/services/webhook/matrix_test.go @@ -216,7 +216,7 @@ func TestMatrixJSONPayload(t *testing.T) { require.NoError(t, err) assert.Equal(t, "PUT", req.Method) - assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/6db5dc1e282529a8c162c7fe93dd2667494eeb51", req.URL.Path) + assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/4ddf3b1533e54f082ae6eadfc1b5530be36c8893", req.URL.Path) assert.Equal(t, "sha256=", req.Header.Get("X-Hub-Signature-256")) assert.Equal(t, "application/json", req.Header.Get("Content-Type")) var body MatrixPayload From ecc304aa53293749f6beb156c32ac593a8f9af63 Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Thu, 11 Sep 2025 00:08:26 +0000 Subject: [PATCH 22/28] resolve vagrant test failure --- tests/integration/api_packages_vagrant_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_packages_vagrant_test.go b/tests/integration/api_packages_vagrant_test.go index 22412a8558e62..83e1431dd82c9 100644 --- a/tests/integration/api_packages_vagrant_test.go +++ b/tests/integration/api_packages_vagrant_test.go @@ -165,6 +165,6 @@ func TestPackageVagrant(t *testing.T) { provider := version.Providers[0] assert.Equal(t, packageProvider, provider.Name) assert.Equal(t, "sha512", provider.ChecksumType) - assert.Equal(t, "259bebd6160acad695016d22a45812e26f187aaf78e71a4c23ee3201528346293f991af3468a8c6c5d2a21d7d9e1bdc1bf79b87110b2fddfcc5a0d45963c7c30", provider.Checksum) + assert.Equal(t, "c9967d88db2888a74778b5c62dbc2508921c8b54aca0e2ba34ab3e95e655cdb182bb2989b28e7ab4cab696f2ac7193d7ba9f57dea5191aad0c6a1082991c1ab8", provider.Checksum) }) } From 748b590c89595e80547953b4bcb1d12bd38541c3 Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Thu, 11 Sep 2025 14:23:40 +0000 Subject: [PATCH 23/28] the security check has a panic when using the experimental go library --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0132becff1991..9a274f97e82a8 100644 --- a/Makefile +++ b/Makefile @@ -770,7 +770,7 @@ generate-go: $(TAGS_PREREQ) .PHONY: security-check security-check: - go run $(GOVULNCHECK_PACKAGE) -show color ./... + GOEXPERIMENT= go run $(GOVULNCHECK_PACKAGE) -show color ./... $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ) ifneq ($(and $(STATIC),$(findstring pam,$(TAGS))),) From 60b26b0368dba35e54537eb4d103f9b038324967 Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Thu, 11 Sep 2025 16:19:22 +0000 Subject: [PATCH 24/28] resolve panic in assetfs parsing of embeded data --- modules/assetfs/embed.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/modules/assetfs/embed.go b/modules/assetfs/embed.go index 95176372d10d1..6f2aeb9218e3c 100644 --- a/modules/assetfs/embed.go +++ b/modules/assetfs/embed.go @@ -102,12 +102,29 @@ func NewEmbeddedFS(data []byte) fs.ReadDirFS { efs := &embeddedFS{data: data, files: make(map[string]*embeddedFileInfo)} efs.meta = sync.OnceValue(func() *EmbeddedMeta { var meta EmbeddedMeta + + // look for the separator newline between binary data and JSON metadata + // jsonv2 may end with an extra newline p := bytes.LastIndexByte(data, '\n') if p < 0 { return &meta } - if err := json.Unmarshal(data[p+1:], &meta); err != nil { - panic("embedded file is not valid") + + // if the data ends with a newline, look for the previous newline + // to find the real separator + if p == len(data)-1 { + p = bytes.LastIndexByte(data[:p], '\n') + if p < 0 { + return &meta + } + } + + jsonData := data[p+1:] + if err := json.Unmarshal(jsonData, &meta); err != nil { + panic("embedded file is not valid: " + err.Error()) + } + if meta.Root == nil { + panic("embedded file metadata has nil root") } return &meta }) @@ -150,9 +167,15 @@ func (e *embeddedFS) getFileInfo(fullName string) (*embeddedFileInfo, error) { fields := strings.Split(fullName, "/") fi = e.meta().Root + if fi == nil { + return nil, fs.ErrNotExist + } if fullName != "." { found := true for _, field := range fields { + if fi.Children == nil { + return nil, fs.ErrNotExist + } for _, child := range fi.Children { if found = child.BaseName == field; found { fi = child From 07afc37a24d2b81aad909ccde5c08ad586fa472a Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 26 Sep 2025 23:38:06 +0800 Subject: [PATCH 25/28] fix --- models/webhook/webhook.go | 9 +- modules/assetfs/embed.go | 27 +----- modules/json/json.go | 61 +----------- modules/json/jsonlegacy.go | 9 ++ modules/json/jsonv2.go | 92 +++++++------------ modules/json/jsonv2_fallback.go | 33 ------- services/webhook/deliver_test.go | 4 +- services/webhook/matrix_test.go | 2 +- .../integration/api_packages_vagrant_test.go | 2 +- 9 files changed, 52 insertions(+), 187 deletions(-) create mode 100644 modules/json/jsonlegacy.go delete mode 100644 modules/json/jsonv2_fallback.go diff --git a/models/webhook/webhook.go b/models/webhook/webhook.go index 794608e970b39..7d4b2e2237db0 100644 --- a/models/webhook/webhook.go +++ b/models/webhook/webhook.go @@ -150,14 +150,7 @@ func init() { // AfterLoad updates the webhook object upon setting a column func (w *Webhook) AfterLoad() { w.HookEvent = &webhook_module.HookEvent{} - - events := w.Events - if events == "" { - // jsonv2 is unable to unmarshal an empty string - return - } - - if err := json.Unmarshal([]byte(events), w.HookEvent); err != nil { + if err := json.Unmarshal([]byte(w.Events), w.HookEvent); err != nil { log.Error("Unmarshal[%d]: %v", w.ID, err) } } diff --git a/modules/assetfs/embed.go b/modules/assetfs/embed.go index 6f2aeb9218e3c..95176372d10d1 100644 --- a/modules/assetfs/embed.go +++ b/modules/assetfs/embed.go @@ -102,29 +102,12 @@ func NewEmbeddedFS(data []byte) fs.ReadDirFS { efs := &embeddedFS{data: data, files: make(map[string]*embeddedFileInfo)} efs.meta = sync.OnceValue(func() *EmbeddedMeta { var meta EmbeddedMeta - - // look for the separator newline between binary data and JSON metadata - // jsonv2 may end with an extra newline p := bytes.LastIndexByte(data, '\n') if p < 0 { return &meta } - - // if the data ends with a newline, look for the previous newline - // to find the real separator - if p == len(data)-1 { - p = bytes.LastIndexByte(data[:p], '\n') - if p < 0 { - return &meta - } - } - - jsonData := data[p+1:] - if err := json.Unmarshal(jsonData, &meta); err != nil { - panic("embedded file is not valid: " + err.Error()) - } - if meta.Root == nil { - panic("embedded file metadata has nil root") + if err := json.Unmarshal(data[p+1:], &meta); err != nil { + panic("embedded file is not valid") } return &meta }) @@ -167,15 +150,9 @@ func (e *embeddedFS) getFileInfo(fullName string) (*embeddedFileInfo, error) { fields := strings.Split(fullName, "/") fi = e.meta().Root - if fi == nil { - return nil, fs.ErrNotExist - } if fullName != "." { found := true for _, field := range fields { - if fi.Children == nil { - return nil, fs.ErrNotExist - } for _, child := range fi.Children { if found = child.BaseName == field; found { fi = child diff --git a/modules/json/json.go b/modules/json/json.go index c736e5b7232bf..5d2deca60876b 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -32,22 +32,12 @@ type Interface interface { } var ( - // DefaultJSONHandler default json handler - uses JSON v2 if available, otherwise JSONiter - DefaultJSONHandler = getDefaultHandler() + DefaultJSONHandler = getDefaultJSONHandler() _ Interface = StdJSON{} _ Interface = JSONiter{} - _ Interface = JSONv2{} ) -// getDefaultHandler returns the expected JSON implementation -func getDefaultHandler() Interface { - if isJSONv2Available() { - return JSONv2{} - } - return JSONiter{jsoniter.ConfigCompatibleWithStandardLibrary} -} - // StdJSON implements Interface via encoding/json type StdJSON struct{} @@ -106,47 +96,6 @@ func (j JSONiter) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) e return json.Indent(dst, src, prefix, indent) } -// JSONv2 implements Interface via encoding/json/v2 -// Requires GOEXPERIMENT=jsonv2 to be set at build time -type JSONv2 struct{} - -// Marshal implements Interface using JSON v2 - fallback if v2 is not available -func (JSONv2) Marshal(v any) ([]byte, error) { - if !isJSONv2Available() { - return json.Marshal(v) - } - return marshalV2(v) -} - -// Unmarshal implements Interface using JSON v2 - fallback if v2 is not available -func (JSONv2) Unmarshal(data []byte, v any) error { - if !isJSONv2Available() { - return json.Unmarshal(data, v) - } - return unmarshalV2(data, v) -} - -// NewEncoder implements Interface using JSON v2 - fallback if v2 is not available -func (JSONv2) NewEncoder(writer io.Writer) Encoder { - if !isJSONv2Available() { - return json.NewEncoder(writer) - } - return newEncoderV2(writer) -} - -// NewDecoder implements Interface using JSON v2 - fallback if v2 is not available -func (JSONv2) NewDecoder(reader io.Reader) Decoder { - if !isJSONv2Available() { - return json.NewDecoder(reader) - } - return newDecoderV2(reader) -} - -// Indent implements Interface using standard library (JSON v2 doesn't have Indent yet) -func (JSONv2) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { - return json.Indent(dst, src, prefix, indent) -} - // Marshal converts object as bytes func Marshal(v any) ([]byte, error) { return DefaultJSONHandler.Marshal(v) @@ -174,7 +123,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { // MarshalIndent copied from encoding/json func MarshalIndent(v any, prefix, indent string) ([]byte, error) { - b, err := DefaultJSONHandler.Marshal(v) + b, err := Marshal(v) if err != nil { return nil, err } @@ -200,12 +149,6 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { // To be consistent, we should treat all empty inputs as success return nil } - - trimmed := bytes.TrimSpace(bs) - if len(trimmed) == 0 { - return nil - } - err := DefaultJSONHandler.Unmarshal(bs, v) if err != nil { ok := true diff --git a/modules/json/jsonlegacy.go b/modules/json/jsonlegacy.go new file mode 100644 index 0000000000000..695d6051f08ce --- /dev/null +++ b/modules/json/jsonlegacy.go @@ -0,0 +1,9 @@ +//go:build !goexperiment.jsonv2 + +package json + +import jsoniter "github.com/json-iterator/go" + +func getDefaultJSONHandler() Interface { + return JSONiter{jsoniter.ConfigCompatibleWithStandardLibrary} +} diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 9b41af2e34cb6..3977088aa4def 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -11,95 +11,71 @@ import ( "io" ) -// isJSONv2Available returns true when JSON v2 is available (compiled with GOEXPERIMENT=jsonv2) -func isJSONv2Available() bool { - return true +// JSONv2 implements Interface via encoding/json/v2 +// Requires GOEXPERIMENT=jsonv2 to be set at build time +type JSONv2 struct{} + +var _ Interface = JSONv2{} + +func getDefaultJSONHandler() Interface { + return JSONv2{} } -// marshalV2Internal uses JSON v2 marshal with v1 compatibility options (no trailing newline) -func marshalV2Internal(v any) ([]byte, error) { - opts := jsonv2.JoinOptions( +func jsonv2DefaultMarshalOptions() jsonv2.Options { + return jsonv2.JoinOptions( jsonv2.MatchCaseInsensitiveNames(true), jsonv2.FormatNilSliceAsNull(true), jsonv2.FormatNilMapAsNull(true), jsonv2.Deterministic(true), ) - return jsonv2.Marshal(v, opts) } -// marshalV2 uses JSON v2 marshal with v1 compatibility options (with trailing newline for compatibility with standard library) -func marshalV2(v any) ([]byte, error) { - result, err := marshalV2Internal(v) - if err != nil { - return nil, err - } - - return append(result, '\n'), nil +func jsonv2DefaultUnmarshalOptions() jsonv2.Options { + return jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + ) } -// unmarshalV2 uses JSON v2 unmarshal with v1 compatibility options -func unmarshalV2(data []byte, v any) error { - if len(data) == 0 { - return nil - } +func (JSONv2) Marshal(v any) ([]byte, error) { + return jsonv2.Marshal(v, jsonv2DefaultMarshalOptions()) +} +func (JSONv2) Unmarshal(data []byte, v any) error { + // legacy behavior: treat empty or whitespace-only input as no input, it should be safe data = bytes.TrimSpace(data) if len(data) == 0 { return nil } + return jsonv2.Unmarshal(data, v, jsonv2DefaultUnmarshalOptions()) +} - opts := jsonv2.JoinOptions( - jsonv2.MatchCaseInsensitiveNames(true), - ) - return jsonv2.Unmarshal(data, v, opts) +func (JSONv2) NewEncoder(writer io.Writer) Encoder { + return &encoderV2{writer: writer, opts: jsonv2DefaultMarshalOptions()} +} + +func (JSONv2) NewDecoder(reader io.Reader) Decoder { + return &decoderV2{reader: reader, opts: jsonv2DefaultMarshalOptions()} +} + +// Indent implements Interface using standard library (JSON v2 doesn't have Indent yet) +func (JSONv2) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { + return json.Indent(dst, src, prefix, indent) } -// encoderV2 wraps JSON v2 streaming encoder type encoderV2 struct { writer io.Writer opts jsonv2.Options } func (e *encoderV2) Encode(v any) error { - err := jsonv2.MarshalWrite(e.writer, v, e.opts) - if err != nil { - return err - } - - _, err = e.writer.Write([]byte{'\n'}) - return err + return jsonv2.MarshalWrite(e.writer, v, e.opts) } -// newEncoderV2 creates a new JSON v2 streaming encoder -func newEncoderV2(writer io.Writer) Encoder { - opts := jsonv2.JoinOptions( - jsonv2.MatchCaseInsensitiveNames(true), - jsonv2.FormatNilSliceAsNull(true), - jsonv2.FormatNilMapAsNull(true), - jsonv2.Deterministic(true), - ) - return &encoderV2{writer: writer, opts: opts} -} - -// decoderV2 wraps JSON v2 streaming decoder type decoderV2 struct { reader io.Reader opts jsonv2.Options } func (d *decoderV2) Decode(v any) error { - err := jsonv2.UnmarshalRead(d.reader, v, d.opts) - // Handle EOF more gracefully to match standard library behavior - if err != nil && err.Error() == "unexpected EOF" { - return io.EOF - } - return err -} - -// newDecoderV2 creates a new JSON v2 streaming decoder -func newDecoderV2(reader io.Reader) Decoder { - opts := jsonv2.JoinOptions( - jsonv2.MatchCaseInsensitiveNames(true), - ) - return &decoderV2{reader: reader, opts: opts} + return jsonv2.UnmarshalRead(d.reader, v, d.opts) } diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go deleted file mode 100644 index e78893ed45b88..0000000000000 --- a/modules/json/jsonv2_fallback.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2025 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !goexperiment.jsonv2 - -package json - -import "io" - -// isJSONv2Available returns false when JSON v2 is not available (not compiled with GOEXPERIMENT=jsonv2) -func isJSONv2Available() bool { - return false -} - -// marshalV2 fallback - should not be called when JSON v2 is not available -func marshalV2(v any) ([]byte, error) { - panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") -} - -// unmarshalV2 fallback - should not be called when JSON v2 is not available -func unmarshalV2(data []byte, v any) error { - panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") -} - -// newEncoderV2 fallback - should not be called when JSON v2 is not available -func newEncoderV2(writer io.Writer) Encoder { - panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") -} - -// newDecoderV2 fallback - should not be called when JSON v2 is not available -func newDecoderV2(reader io.Reader) Decoder { - panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") -} diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index b0e83175747fb..efbef1fc89444 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -142,13 +142,13 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.NoError(t, err) assert.Equal(t, `{"data": 42}`, string(body)) - case "/webhook/4ddf3b1533e54f082ae6eadfc1b5530be36c8893": + case "/webhook/6db5dc1e282529a8c162c7fe93dd2667494eeb51": // Version 2 assert.Equal(t, "push", r.Header.Get("X-GitHub-Event")) assert.Equal(t, "application/json", r.Header.Get("Content-Type")) body, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.Len(t, body, 2047) + assert.Len(t, body, 2147) default: w.WriteHeader(http.StatusNotFound) diff --git a/services/webhook/matrix_test.go b/services/webhook/matrix_test.go index c39867b4816f0..d36d93c5a7385 100644 --- a/services/webhook/matrix_test.go +++ b/services/webhook/matrix_test.go @@ -216,7 +216,7 @@ func TestMatrixJSONPayload(t *testing.T) { require.NoError(t, err) assert.Equal(t, "PUT", req.Method) - assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/4ddf3b1533e54f082ae6eadfc1b5530be36c8893", req.URL.Path) + assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/6db5dc1e282529a8c162c7fe93dd2667494eeb51", req.URL.Path) assert.Equal(t, "sha256=", req.Header.Get("X-Hub-Signature-256")) assert.Equal(t, "application/json", req.Header.Get("Content-Type")) var body MatrixPayload diff --git a/tests/integration/api_packages_vagrant_test.go b/tests/integration/api_packages_vagrant_test.go index 83e1431dd82c9..22412a8558e62 100644 --- a/tests/integration/api_packages_vagrant_test.go +++ b/tests/integration/api_packages_vagrant_test.go @@ -165,6 +165,6 @@ func TestPackageVagrant(t *testing.T) { provider := version.Providers[0] assert.Equal(t, packageProvider, provider.Name) assert.Equal(t, "sha512", provider.ChecksumType) - assert.Equal(t, "c9967d88db2888a74778b5c62dbc2508921c8b54aca0e2ba34ab3e95e655cdb182bb2989b28e7ab4cab696f2ac7193d7ba9f57dea5191aad0c6a1082991c1ab8", provider.Checksum) + assert.Equal(t, "259bebd6160acad695016d22a45812e26f187aaf78e71a4c23ee3201528346293f991af3468a8c6c5d2a21d7d9e1bdc1bf79b87110b2fddfcc5a0d45963c7c30", provider.Checksum) }) } From cb47618fb640869e5206dea3a6f036e109b682d1 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 27 Sep 2025 00:09:55 +0800 Subject: [PATCH 26/28] fix --- modules/assetfs/embed.go | 4 +- modules/json/json.go | 6 +-- modules/json/jsonlegacy.go | 7 +++ modules/json/jsonv2.go | 58 ++++++++++++----------- modules/lfs/http_client_test.go | 16 +++++-- modules/optional/serialization_test.go | 25 ++++++---- services/webhook/deliver_test.go | 52 +++++++++++++++++++- services/webhook/matrix.go | 1 + services/webhook/matrix_test.go | 2 +- tests/integration/api_repo_branch_test.go | 4 +- 10 files changed, 124 insertions(+), 51 deletions(-) diff --git a/modules/assetfs/embed.go b/modules/assetfs/embed.go index 95176372d10d1..0b544635db0f6 100644 --- a/modules/assetfs/embed.go +++ b/modules/assetfs/embed.go @@ -365,11 +365,11 @@ func GenerateEmbedBindata(fsRootPath, outputFile string) error { if err = embedFiles(meta.Root, fsRootPath, ""); err != nil { return err } - jsonBuf, err := json.Marshal(meta) // can't use json.NewEncoder here because it writes extra EOL + jsonBuf, err := json.Marshal(meta) if err != nil { return err } _, _ = output.Write([]byte{'\n'}) - _, err = output.Write(jsonBuf) + _, err = output.Write(bytes.TrimSpace(jsonBuf)) return err } diff --git a/modules/json/json.go b/modules/json/json.go index 5d2deca60876b..deb869619bc91 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -149,7 +149,7 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { // To be consistent, we should treat all empty inputs as success return nil } - err := DefaultJSONHandler.Unmarshal(bs, v) + err := json.Unmarshal(bs, v) if err != nil { ok := true rs := []byte{} @@ -166,11 +166,11 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { if len(rs) > 1 && rs[0] == 0xff && rs[1] == 0xfe { rs = rs[2:] } - err = DefaultJSONHandler.Unmarshal(rs, v) + err = json.Unmarshal(rs, v) } } if err != nil && len(bs) > 2 && bs[0] == 0xff && bs[1] == 0xfe { - err = DefaultJSONHandler.Unmarshal(bs[2:], v) + err = json.Unmarshal(bs[2:], v) } return err } diff --git a/modules/json/jsonlegacy.go b/modules/json/jsonlegacy.go index 695d6051f08ce..35bb31e27c5fb 100644 --- a/modules/json/jsonlegacy.go +++ b/modules/json/jsonlegacy.go @@ -1,3 +1,6 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + //go:build !goexperiment.jsonv2 package json @@ -7,3 +10,7 @@ import jsoniter "github.com/json-iterator/go" func getDefaultJSONHandler() Interface { return JSONiter{jsoniter.ConfigCompatibleWithStandardLibrary} } + +func MarshalKeepOptionalEmpty(v any) ([]byte, error) { + return DefaultJSONHandler.Marshal(v) +} diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 3977088aa4def..6fc7e968ed2a7 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -7,59 +7,63 @@ package json import ( "bytes" + jsonv1 "encoding/json" //nolint:depguard // this package wraps it jsonv2 "encoding/json/v2" //nolint:depguard // this package wraps it "io" ) // JSONv2 implements Interface via encoding/json/v2 // Requires GOEXPERIMENT=jsonv2 to be set at build time -type JSONv2 struct{} - -var _ Interface = JSONv2{} - -func getDefaultJSONHandler() Interface { - return JSONv2{} +type JSONv2 struct { + marshalOptions jsonv2.Options + marshalKeepOptionalEmptyOptions jsonv2.Options + unmarshalOptions jsonv2.Options } -func jsonv2DefaultMarshalOptions() jsonv2.Options { - return jsonv2.JoinOptions( +var jsonV2 JSONv2 + +func init() { + commonMarshalOptions := []jsonv2.Options{ jsonv2.MatchCaseInsensitiveNames(true), jsonv2.FormatNilSliceAsNull(true), jsonv2.FormatNilMapAsNull(true), jsonv2.Deterministic(true), - ) + } + jsonV2.marshalOptions = jsonv2.JoinOptions(commonMarshalOptions...) + jsonV2.unmarshalOptions = jsonv2.JoinOptions(jsonv2.MatchCaseInsensitiveNames(true)) + + // by default, "json/v2" omitempty removes all `""` empty strings, no matter where it comes from. + // v1 has a different behavior: if the `""` is from a null pointer, or a Marshal function, it is kept. + jsonV2.marshalKeepOptionalEmptyOptions = jsonv2.JoinOptions(append(commonMarshalOptions, jsonv1.OmitEmptyWithLegacySemantics(true))...) } -func jsonv2DefaultUnmarshalOptions() jsonv2.Options { - return jsonv2.JoinOptions( - jsonv2.MatchCaseInsensitiveNames(true), - ) +func getDefaultJSONHandler() Interface { + return jsonV2 } -func (JSONv2) Marshal(v any) ([]byte, error) { - return jsonv2.Marshal(v, jsonv2DefaultMarshalOptions()) +func MarshalKeepOptionalEmpty(v any) ([]byte, error) { + return jsonv2.Marshal(v, jsonV2.marshalKeepOptionalEmptyOptions) } -func (JSONv2) Unmarshal(data []byte, v any) error { - // legacy behavior: treat empty or whitespace-only input as no input, it should be safe - data = bytes.TrimSpace(data) - if len(data) == 0 { - return nil - } - return jsonv2.Unmarshal(data, v, jsonv2DefaultUnmarshalOptions()) +func (j JSONv2) Marshal(v any) ([]byte, error) { + return jsonv2.Marshal(v, j.marshalOptions) +} + +func (j JSONv2) Unmarshal(data []byte, v any) error { + return jsonv2.Unmarshal(data, v, j.unmarshalOptions) } -func (JSONv2) NewEncoder(writer io.Writer) Encoder { - return &encoderV2{writer: writer, opts: jsonv2DefaultMarshalOptions()} +func (j JSONv2) NewEncoder(writer io.Writer) Encoder { + return &encoderV2{writer: writer, opts: j.marshalOptions} } -func (JSONv2) NewDecoder(reader io.Reader) Decoder { - return &decoderV2{reader: reader, opts: jsonv2DefaultMarshalOptions()} +func (j JSONv2) NewDecoder(reader io.Reader) Decoder { + return &decoderV2{reader: reader, opts: j.unmarshalOptions} } // Indent implements Interface using standard library (JSON v2 doesn't have Indent yet) func (JSONv2) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { - return json.Indent(dst, src, prefix, indent) + return jsonv1.Indent(dst, src, prefix, indent) } type encoderV2 struct { diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index 179bcdb29a48a..90b563ce2db87 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -193,7 +193,7 @@ func TestHTTPClientDownload(t *testing.T) { }, { endpoint: "https://invalid-json-response.io", - expectedError: "invalid json", + expectedError: "/(invalid json|jsontext: invalid character)/", }, { endpoint: "https://valid-batch-request-download.io", @@ -258,7 +258,11 @@ func TestHTTPClientDownload(t *testing.T) { return nil }) if c.expectedError != "" { - assert.ErrorContains(t, err, c.expectedError) + if strings.HasPrefix(c.expectedError, "/") && strings.HasSuffix(c.expectedError, "/") { + assert.Regexp(t, strings.Trim(c.expectedError, "/"), err.Error()) + } else { + assert.ErrorContains(t, err, c.expectedError) + } } else { assert.NoError(t, err) } @@ -297,7 +301,7 @@ func TestHTTPClientUpload(t *testing.T) { }, { endpoint: "https://invalid-json-response.io", - expectedError: "invalid json", + expectedError: "/(invalid json|jsontext: invalid character)/", }, { endpoint: "https://valid-batch-request-upload.io", @@ -352,7 +356,11 @@ func TestHTTPClientUpload(t *testing.T) { return io.NopCloser(new(bytes.Buffer)), objectError }) if c.expectedError != "" { - assert.ErrorContains(t, err, c.expectedError) + if strings.HasPrefix(c.expectedError, "/") && strings.HasSuffix(c.expectedError, "/") { + assert.Regexp(t, strings.Trim(c.expectedError, "/"), err.Error()) + } else { + assert.ErrorContains(t, err, c.expectedError) + } } else { assert.NoError(t, err) } diff --git a/modules/optional/serialization_test.go b/modules/optional/serialization_test.go index cf81a94cfce93..c059294bbb99d 100644 --- a/modules/optional/serialization_test.go +++ b/modules/optional/serialization_test.go @@ -15,12 +15,17 @@ import ( ) type testSerializationStruct struct { - NormalString string `json:"normal_string" yaml:"normal_string"` - NormalBool bool `json:"normal_bool" yaml:"normal_bool"` - OptBool optional.Option[bool] `json:"optional_bool,omitempty" yaml:"optional_bool,omitempty"` - OptString optional.Option[string] `json:"optional_string,omitempty" yaml:"optional_string,omitempty"` + NormalString string `json:"normal_string" yaml:"normal_string"` + NormalBool bool `json:"normal_bool" yaml:"normal_bool"` + OptBool optional.Option[bool] `json:"optional_bool,omitempty" yaml:"optional_bool,omitempty"` + + // It causes an undefined behavior: should the "omitempty" tag only omit "null", or also the empty string? + // The behavior is inconsistent between json and v2 packages, and there is no such use case in Gitea. + // If anyone really needs it, they can use json.MarshalKeepOptionalEmpty to revert the v1 behavior + OptString optional.Option[string] `json:"optional_string,omitempty" yaml:"optional_string,omitempty"` + OptTwoBool optional.Option[bool] `json:"optional_two_bool" yaml:"optional_two_bool"` - OptTwoString optional.Option[string] `json:"optional_twostring" yaml:"optional_two_string"` + OptTwoString optional.Option[string] `json:"optional_two_string" yaml:"optional_two_string"` } func TestOptionalToJson(t *testing.T) { @@ -32,7 +37,7 @@ func TestOptionalToJson(t *testing.T) { { name: "empty", obj: new(testSerializationStruct), - want: `{"normal_string":"","normal_bool":false,"optional_two_bool":null,"optional_twostring":null}`, + want: `{"normal_string":"","normal_bool":false,"optional_two_bool":null,"optional_two_string":null}`, }, { name: "some", @@ -44,12 +49,12 @@ func TestOptionalToJson(t *testing.T) { OptTwoBool: optional.None[bool](), OptTwoString: optional.None[string](), }, - want: `{"normal_string":"a string","normal_bool":true,"optional_bool":false,"optional_string":"","optional_two_bool":null,"optional_twostring":null}`, + want: `{"normal_string":"a string","normal_bool":true,"optional_bool":false,"optional_string":"","optional_two_bool":null,"optional_two_string":null}`, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - b, err := json.Marshal(tc.obj) + b, err := json.MarshalKeepOptionalEmpty(tc.obj) assert.NoError(t, err) assert.Equal(t, tc.want, string(b), "gitea json module returned unexpected") @@ -75,7 +80,7 @@ func TestOptionalFromJson(t *testing.T) { }, { name: "some", - data: `{"normal_string":"a string","normal_bool":true,"optional_bool":false,"optional_string":"","optional_two_bool":null,"optional_twostring":null}`, + data: `{"normal_string":"a string","normal_bool":true,"optional_bool":false,"optional_string":"","optional_two_bool":null,"optional_two_string":null}`, want: testSerializationStruct{ NormalString: "a string", NormalBool: true, @@ -169,7 +174,7 @@ normal_bool: true optional_bool: false optional_string: "" optional_two_bool: null -optional_twostring: null +optional_two_string: null `, want: testSerializationStruct{ NormalString: "a string", diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index efbef1fc89444..8900d6daa202c 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -131,6 +131,54 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) done := make(chan struct{}, 1) + version2Body := `{ + "body": "[[test/repo](http://localhost:3000/test/repo)] user1 pushed 2 commits to [test](http://localhost:3000/test/repo/src/branch/test):\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778): commit message - user1\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778): commit message - user1", + "msgtype": "", + "format": "org.matrix.custom.html", + "formatted_body": "[test/repo] user1 pushed 2 commits to test:
2020558: commit message - user1
2020558: commit message - user1", + "io.gitea.commits": [ + { + "id": "2020558fe2e34debb818a514715839cabd25e778", + "message": "commit message", + "url": "http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778", + "author": { + "name": "user1", + "email": "user1@localhost", + "username": "user1" + }, + "committer": { + "name": "user1", + "email": "user1@localhost", + "username": "user1" + }, + "verification": null, + "timestamp": "0001-01-01T00:00:00Z", + "added": null, + "removed": null, + "modified": null + }, + { + "id": "2020558fe2e34debb818a514715839cabd25e778", + "message": "commit message", + "url": "http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778", + "author": { + "name": "user1", + "email": "user1@localhost", + "username": "user1" + }, + "committer": { + "name": "user1", + "email": "user1@localhost", + "username": "user1" + }, + "verification": null, + "timestamp": "0001-01-01T00:00:00Z", + "added": null, + "removed": null, + "modified": null + } + ] +}` s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "PUT", r.Method) switch r.URL.Path { @@ -142,13 +190,13 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.NoError(t, err) assert.Equal(t, `{"data": 42}`, string(body)) - case "/webhook/6db5dc1e282529a8c162c7fe93dd2667494eeb51": + case "/webhook/4ddf3b1533e54f082ae6eadfc1b5530be36c8893": // Version 2 assert.Equal(t, "push", r.Header.Get("X-GitHub-Event")) assert.Equal(t, "application/json", r.Header.Get("Content-Type")) body, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.Len(t, body, 2147) + assert.JSONEq(t, version2Body, string(body)) default: w.WriteHeader(http.StatusNotFound) diff --git a/services/webhook/matrix.go b/services/webhook/matrix.go index 3e9163f78c2f2..57b1ece26350f 100644 --- a/services/webhook/matrix.go +++ b/services/webhook/matrix.go @@ -274,6 +274,7 @@ func getMessageBody(htmlText string) string { // getMatrixTxnID computes the transaction ID to ensure idempotency func getMatrixTxnID(payload []byte) (string, error) { + payload = bytes.TrimSpace(payload) if len(payload) >= matrixPayloadSizeLimit { return "", fmt.Errorf("getMatrixTxnID: payload size %d > %d", len(payload), matrixPayloadSizeLimit) } diff --git a/services/webhook/matrix_test.go b/services/webhook/matrix_test.go index d36d93c5a7385..c39867b4816f0 100644 --- a/services/webhook/matrix_test.go +++ b/services/webhook/matrix_test.go @@ -216,7 +216,7 @@ func TestMatrixJSONPayload(t *testing.T) { require.NoError(t, err) assert.Equal(t, "PUT", req.Method) - assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/6db5dc1e282529a8c162c7fe93dd2667494eeb51", req.URL.Path) + assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/4ddf3b1533e54f082ae6eadfc1b5530be36c8893", req.URL.Path) assert.Equal(t, "sha256=", req.Header.Get("X-Hub-Signature-256")) assert.Equal(t, "application/json", req.Header.Get("Content-Type")) var body MatrixPayload diff --git a/tests/integration/api_repo_branch_test.go b/tests/integration/api_repo_branch_test.go index 066eb366b1005..2438db72c5a4a 100644 --- a/tests/integration/api_repo_branch_test.go +++ b/tests/integration/api_repo_branch_test.go @@ -121,10 +121,10 @@ func TestAPIRepoBranchesMirror(t *testing.T) { resp = MakeRequest(t, req, http.StatusForbidden) bs, err = io.ReadAll(resp.Body) assert.NoError(t, err) - assert.Equal(t, "{\"message\":\"Git Repository is a mirror.\",\"url\":\""+setting.AppURL+"api/swagger\"}\n", string(bs)) + assert.JSONEq(t, "{\"message\":\"Git Repository is a mirror.\",\"url\":\""+setting.AppURL+"api/swagger\"}", string(bs)) resp = MakeRequest(t, NewRequest(t, "DELETE", link2.String()).AddTokenAuth(token), http.StatusForbidden) bs, err = io.ReadAll(resp.Body) assert.NoError(t, err) - assert.Equal(t, "{\"message\":\"Git Repository is a mirror.\",\"url\":\""+setting.AppURL+"api/swagger\"}\n", string(bs)) + assert.JSONEq(t, "{\"message\":\"Git Repository is a mirror.\",\"url\":\""+setting.AppURL+"api/swagger\"}", string(bs)) } From f302ca810a781d5df64aa57cd25fe2910c6f0ee0 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 27 Sep 2025 03:01:50 +0800 Subject: [PATCH 27/28] fix --- .github/workflows/pull-db-tests.yml | 6 +++--- modules/json/jsonv2.go | 6 +++--- modules/packages/container/metadata_test.go | 2 ++ services/webhook/deliver_test.go | 14 +++++++++----- services/webhook/matrix_test.go | 5 ++++- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index faf6a90e1bc5b..a7ad7ed5c389e 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -72,13 +72,13 @@ jobs: go-version-file: go.mod check-latest: true - run: make deps-backend - - run: make backend + - run: GOEXPERIMENT='' make backend env: TAGS: bindata gogit sqlite sqlite_unlock_notify - name: run migration tests run: make test-sqlite-migration - name: run tests - run: make test-sqlite + run: GOEXPERIMENT='' make test-sqlite timeout-minutes: 50 env: TAGS: bindata gogit sqlite sqlite_unlock_notify @@ -142,7 +142,7 @@ jobs: RACE_ENABLED: true GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} - name: unit-tests-gogit - run: make unit-test-coverage test-check + run: GOEXPERIMENT='' make unit-test-coverage test-check env: TAGS: bindata gogit RACE_ENABLED: true diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 6fc7e968ed2a7..483350bbfaa60 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -24,12 +24,12 @@ var jsonV2 JSONv2 func init() { commonMarshalOptions := []jsonv2.Options{ - jsonv2.MatchCaseInsensitiveNames(true), jsonv2.FormatNilSliceAsNull(true), jsonv2.FormatNilMapAsNull(true), - jsonv2.Deterministic(true), } jsonV2.marshalOptions = jsonv2.JoinOptions(commonMarshalOptions...) + + // Some JSON structs like oci.ImageConfig uses case-insensitive matching jsonV2.unmarshalOptions = jsonv2.JoinOptions(jsonv2.MatchCaseInsensitiveNames(true)) // by default, "json/v2" omitempty removes all `""` empty strings, no matter where it comes from. @@ -38,7 +38,7 @@ func init() { } func getDefaultJSONHandler() Interface { - return jsonV2 + return &jsonV2 } func MarshalKeepOptionalEmpty(v any) ([]byte, error) { diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go index 0f2d702925c28..89589693ef6e2 100644 --- a/modules/packages/container/metadata_test.go +++ b/modules/packages/container/metadata_test.go @@ -22,6 +22,8 @@ func TestParseImageConfig(t *testing.T) { repositoryURL := "https://gitea.com/gitea" documentationURL := "https://docs.gitea.com" + // FIXME: the test case is not right, the config fields are capitalized in the spec + // https://github.com/opencontainers/image-spec/blob/main/schema/config-schema.json configOCI := `{"config": {"labels": {"` + labelAuthors + `": "` + author + `", "` + labelLicenses + `": "` + license + `", "` + labelURL + `": "` + projectURL + `", "` + labelSource + `": "` + repositoryURL + `", "` + labelDocumentation + `": "` + documentationURL + `", "` + labelDescription + `": "` + description + `"}}, "history": [{"created_by": "do it 1"}, {"created_by": "dummy #(nop) do it 2"}]}` metadata, err := ParseImageConfig(oci.MediaTypeImageManifest, strings.NewReader(configOCI)) diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index 8900d6daa202c..822184aa94009 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -179,19 +179,21 @@ func TestWebhookDeliverHookTask(t *testing.T) { } ] }` + + testVersion := 0 s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "PUT", r.Method) - switch r.URL.Path { - case "/webhook/66d222a5d6349e1311f551e50722d837e30fce98": - // Version 1 + assert.True(t, strings.HasPrefix(r.URL.Path, "/webhook/")) + assert.Len(t, r.URL.Path, len("/webhook/")+40) // +40 for txnID, a unique ID from payload's sha1 hash + switch testVersion { + case 1: // Version 1 assert.Equal(t, "push", r.Header.Get("X-GitHub-Event")) assert.Empty(t, r.Header.Get("Content-Type")) body, err := io.ReadAll(r.Body) assert.NoError(t, err) assert.Equal(t, `{"data": 42}`, string(body)) - case "/webhook/4ddf3b1533e54f082ae6eadfc1b5530be36c8893": - // Version 2 + case 2: // Version 2 assert.Equal(t, "push", r.Header.Get("X-GitHub-Event")) assert.Equal(t, "application/json", r.Header.Get("Content-Type")) body, err := io.ReadAll(r.Body) @@ -220,6 +222,7 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.NoError(t, webhook_model.CreateWebhook(t.Context(), hook)) t.Run("Version 1", func(t *testing.T) { + testVersion = 1 hookTask := &webhook_model.HookTask{ HookID: hook.ID, EventType: webhook_module.HookEventPush, @@ -246,6 +249,7 @@ func TestWebhookDeliverHookTask(t *testing.T) { data, err := p.JSONPayload() assert.NoError(t, err) + testVersion = 2 hookTask := &webhook_model.HookTask{ HookID: hook.ID, EventType: webhook_module.HookEventPush, diff --git a/services/webhook/matrix_test.go b/services/webhook/matrix_test.go index c39867b4816f0..59b78f426926f 100644 --- a/services/webhook/matrix_test.go +++ b/services/webhook/matrix_test.go @@ -4,6 +4,7 @@ package webhook import ( + "strings" "testing" webhook_model "code.gitea.io/gitea/models/webhook" @@ -216,7 +217,9 @@ func TestMatrixJSONPayload(t *testing.T) { require.NoError(t, err) assert.Equal(t, "PUT", req.Method) - assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/4ddf3b1533e54f082ae6eadfc1b5530be36c8893", req.URL.Path) + txnID, ok := strings.CutPrefix(req.URL.Path, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/") + assert.True(t, ok) + assert.Len(t, txnID, 40) // txnID is just a unique ID for a webhook request, it is a sha1 hash from the payload assert.Equal(t, "sha256=", req.Header.Get("X-Hub-Signature-256")) assert.Equal(t, "application/json", req.Header.Get("Content-Type")) var body MatrixPayload From ad938260c1843198341f20cb852d062049d9ed05 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 27 Sep 2025 12:29:10 +0800 Subject: [PATCH 28/28] fine tune --- modules/json/json_test.go | 10 ++++++ modules/json/jsonlegacy.go | 10 +++++- modules/json/jsonv2.go | 37 ++++++++++++-------- modules/packages/container/metadata.go | 4 ++- modules/packages/container/metadata_test.go | 2 +- tests/integration/api_packages_swift_test.go | 2 +- tests/integration/integration_test.go | 3 +- 7 files changed, 48 insertions(+), 20 deletions(-) diff --git a/modules/json/json_test.go b/modules/json/json_test.go index ace7167913457..2fa4da4cf7966 100644 --- a/modules/json/json_test.go +++ b/modules/json/json_test.go @@ -4,6 +4,7 @@ package json import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -16,3 +17,12 @@ func TestGiteaDBJSONUnmarshal(t *testing.T) { err = UnmarshalHandleDoubleEncode([]byte(""), &m) assert.NoError(t, err) } + +func TestIndent(t *testing.T) { + buf := &bytes.Buffer{} + err := Indent(buf, []byte(`{"a":1}`), ">", " ") + assert.NoError(t, err) + assert.Equal(t, `{ +> "a": 1 +>}`, buf.String()) +} diff --git a/modules/json/jsonlegacy.go b/modules/json/jsonlegacy.go index 35bb31e27c5fb..508e87b6b5444 100644 --- a/modules/json/jsonlegacy.go +++ b/modules/json/jsonlegacy.go @@ -5,7 +5,11 @@ package json -import jsoniter "github.com/json-iterator/go" +import ( + "io" + + jsoniter "github.com/json-iterator/go" +) func getDefaultJSONHandler() Interface { return JSONiter{jsoniter.ConfigCompatibleWithStandardLibrary} @@ -14,3 +18,7 @@ func getDefaultJSONHandler() Interface { func MarshalKeepOptionalEmpty(v any) ([]byte, error) { return DefaultJSONHandler.Marshal(v) } + +func NewDecoderCaseInsensitive(reader io.Reader) Decoder { + return DefaultJSONHandler.NewDecoder(reader) +} diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 483350bbfaa60..0bba2783bcb41 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -18,6 +18,7 @@ type JSONv2 struct { marshalOptions jsonv2.Options marshalKeepOptionalEmptyOptions jsonv2.Options unmarshalOptions jsonv2.Options + unmarshalCaseInsensitiveOptions jsonv2.Options } var jsonV2 JSONv2 @@ -28,13 +29,15 @@ func init() { jsonv2.FormatNilMapAsNull(true), } jsonV2.marshalOptions = jsonv2.JoinOptions(commonMarshalOptions...) + jsonV2.unmarshalOptions = jsonv2.DefaultOptionsV2() - // Some JSON structs like oci.ImageConfig uses case-insensitive matching - jsonV2.unmarshalOptions = jsonv2.JoinOptions(jsonv2.MatchCaseInsensitiveNames(true)) - - // by default, "json/v2" omitempty removes all `""` empty strings, no matter where it comes from. + // By default, "json/v2" omitempty removes all `""` empty strings, no matter where it comes from. // v1 has a different behavior: if the `""` is from a null pointer, or a Marshal function, it is kept. + // Golang issue: https://github.com/golang/go/issues/75623 encoding/json/v2: unable to make omitempty work with pointer or Optional type with goexperiment.jsonv2 jsonV2.marshalKeepOptionalEmptyOptions = jsonv2.JoinOptions(append(commonMarshalOptions, jsonv1.OmitEmptyWithLegacySemantics(true))...) + + // Some legacy code uses case-insensitive matching (for example: parsing oci.ImageConfig) + jsonV2.unmarshalCaseInsensitiveOptions = jsonv2.JoinOptions(jsonv2.MatchCaseInsensitiveNames(true)) } func getDefaultJSONHandler() Interface { @@ -45,41 +48,45 @@ func MarshalKeepOptionalEmpty(v any) ([]byte, error) { return jsonv2.Marshal(v, jsonV2.marshalKeepOptionalEmptyOptions) } -func (j JSONv2) Marshal(v any) ([]byte, error) { +func (j *JSONv2) Marshal(v any) ([]byte, error) { return jsonv2.Marshal(v, j.marshalOptions) } -func (j JSONv2) Unmarshal(data []byte, v any) error { +func (j *JSONv2) Unmarshal(data []byte, v any) error { return jsonv2.Unmarshal(data, v, j.unmarshalOptions) } -func (j JSONv2) NewEncoder(writer io.Writer) Encoder { - return &encoderV2{writer: writer, opts: j.marshalOptions} +func (j *JSONv2) NewEncoder(writer io.Writer) Encoder { + return &jsonV2Encoder{writer: writer, opts: j.marshalOptions} } -func (j JSONv2) NewDecoder(reader io.Reader) Decoder { - return &decoderV2{reader: reader, opts: j.unmarshalOptions} +func (j *JSONv2) NewDecoder(reader io.Reader) Decoder { + return &jsonV2Decoder{reader: reader, opts: j.unmarshalOptions} } // Indent implements Interface using standard library (JSON v2 doesn't have Indent yet) -func (JSONv2) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { +func (*JSONv2) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { return jsonv1.Indent(dst, src, prefix, indent) } -type encoderV2 struct { +type jsonV2Encoder struct { writer io.Writer opts jsonv2.Options } -func (e *encoderV2) Encode(v any) error { +func (e *jsonV2Encoder) Encode(v any) error { return jsonv2.MarshalWrite(e.writer, v, e.opts) } -type decoderV2 struct { +type jsonV2Decoder struct { reader io.Reader opts jsonv2.Options } -func (d *decoderV2) Decode(v any) error { +func (d *jsonV2Decoder) Decode(v any) error { return jsonv2.UnmarshalRead(d.reader, v, d.opts) } + +func NewDecoderCaseInsensitive(reader io.Reader) Decoder { + return &jsonV2Decoder{reader: reader, opts: jsonV2.unmarshalCaseInsensitiveOptions} +} diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go index 3ef0684d13f33..d8a48120afabd 100644 --- a/modules/packages/container/metadata.go +++ b/modules/packages/container/metadata.go @@ -103,7 +103,9 @@ func ParseImageConfig(mediaType string, r io.Reader) (*Metadata, error) { func parseOCIImageConfig(r io.Reader) (*Metadata, error) { var image oci.Image - if err := json.NewDecoder(r).Decode(&image); err != nil { + // FIXME: JSON-KEY-CASE: here seems a abuse of the case-insensitive decoding feature, spec is case-sensitive + // https://github.com/opencontainers/image-spec/blob/main/schema/config-schema.json + if err := json.NewDecoderCaseInsensitive(r).Decode(&image); err != nil { return nil, err } diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go index 89589693ef6e2..2a6389a8f629d 100644 --- a/modules/packages/container/metadata_test.go +++ b/modules/packages/container/metadata_test.go @@ -22,7 +22,7 @@ func TestParseImageConfig(t *testing.T) { repositoryURL := "https://gitea.com/gitea" documentationURL := "https://docs.gitea.com" - // FIXME: the test case is not right, the config fields are capitalized in the spec + // FIXME: JSON-KEY-CASE: the test case is not right, the config fields are capitalized in the spec // https://github.com/opencontainers/image-spec/blob/main/schema/config-schema.json configOCI := `{"config": {"labels": {"` + labelAuthors + `": "` + author + `", "` + labelLicenses + `": "` + license + `", "` + labelURL + `": "` + projectURL + `", "` + labelSource + `": "` + repositoryURL + `", "` + labelDocumentation + `": "` + documentationURL + `", "` + labelDescription + `": "` + description + `"}}, "history": [{"created_by": "do it 1"}, {"created_by": "dummy #(nop) do it 2"}]}` diff --git a/tests/integration/api_packages_swift_test.go b/tests/integration/api_packages_swift_test.go index ddc3cb63f3b56..71bb1befd1719 100644 --- a/tests/integration/api_packages_swift_test.go +++ b/tests/integration/api_packages_swift_test.go @@ -318,7 +318,7 @@ func TestPackageSwift(t *testing.T) { AddBasicAuth(user.Name) resp = MakeRequest(t, req, http.StatusOK) - assert.Equal(t, body, resp.Body.String()) + assert.JSONEq(t, body, resp.Body.String()) }) t.Run("PackageVersionMetadata", func(t *testing.T) { diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 5896a97ef1c75..9dfa0ccd5d07d 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -413,7 +413,8 @@ func logUnexpectedResponse(t testing.TB, recorder *httptest.ResponseRecorder) { func DecodeJSON(t testing.TB, resp *httptest.ResponseRecorder, v any) { t.Helper() - decoder := json.NewDecoder(resp.Body) + // FIXME: JSON-KEY-CASE: for testing purpose only, because many structs don't provide `json` tags, they just use capitalized field names + decoder := json.NewDecoderCaseInsensitive(resp.Body) require.NoError(t, decoder.Decode(v)) }