[+] switch to json-iterator/go from encoding/json (#776)

pashagolub · web-flow · commit e35caa6a700d · 2025-06-02T19:00:20.000+02:00
Replaced standard "encoding/json" with "github.com/json-iterator/go"
to improve serialization/deserialization speed and reduce memory
allocations.
This change is expected to lower CPU usage and heap pressure in
high-throughput scenarios. Behavior remains compatible with the standard
library.
diff --git a/go.mod b/go.mod
@@ -8,6 +8,7 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/jackc/pgx/v5 v5.7.5
 	github.com/jessevdk/go-flags v1.6.1
+	github.com/json-iterator/go v1.1.12
 	github.com/pashagolub/pgxmock/v4 v4.7.0
 	github.com/prometheus/client_golang v1.22.0
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
@@ -70,6 +71,8 @@ require (
 	github.com/moby/sys/user v0.4.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
diff --git a/go.sum b/go.sum
@@ -66,6 +66,7 @@ github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl76
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -89,6 +90,8 @@ github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bB
 github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
 github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
 github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -123,6 +126,11 @@ github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g
 github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
 github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
 github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
diff --git a/internal/db/conn.go b/internal/db/conn.go
@@ -2,9 +2,10 @@ package db
 
 import (
 	"context"
-	"encoding/json"
 	"reflect"
 
+	jsoniter "github.com/json-iterator/go"
+
 	pgx "github.com/jackc/pgx/v5"
 	pgconn "github.com/jackc/pgx/v5/pgconn"
 	pgxpool "github.com/jackc/pgx/v5/pgxpool"
@@ -57,7 +58,7 @@ func MarshallParamToJSONB(v any) any {
 			return nil
 		}
 	}
-	if b, err := json.Marshal(v); err == nil {
+	if b, err := jsoniter.ConfigFastest.Marshal(v); err == nil {
 		return string(b)
 	}
 	return nil
diff --git a/internal/reaper/reaper.go b/internal/reaper/reaper.go
@@ -490,9 +490,7 @@ func (r *Reaper) FetchMetric(ctx context.Context, md *sources.SourceConn, metric
 				data[0]["is_up"] = 0 // should be updated if the "instance_up" metric definition is changed
 				goto send_to_storageChannel
 			}
-			l.
-				WithFields(map[string]any{"source": md.Name, "metric": metricName}).
-				WithError(err).Error("failed to fetch metrics")
+			l.WithError(err).Error("failed to fetch metrics")
 
 			return nil, err
 		}
diff --git a/internal/sinks/cmdopts.go b/internal/sinks/cmdopts.go
@@ -5,7 +5,7 @@ import "time"
 // CmdOpts specifies the storage configuration to store metrics measurements
 type CmdOpts struct {
 	Sinks                 []string      `long:"sink" mapstructure:"sink" description:"URI where metrics will be stored, can be used multiple times" env:"PW_SINK"`
-	BatchingDelay         time.Duration `long:"batching-delay" mapstructure:"batching-delay" description:"Max milliseconds to wait for a batched metrics flush" default:"250ms" env:"PW_BATCHING_DELAY"`
+	BatchingDelay         time.Duration `long:"batching-delay" mapstructure:"batching-delay" description:"Timeout to wait for a batched metrics flush" default:"950ms" env:"PW_BATCHING_DELAY"`
 	Retention             int           `long:"retention" mapstructure:"retention" description:"If set, metrics older than that will be deleted" default:"14" env:"PW_RETENTION"`
 	RealDbnameField       string        `long:"real-dbname-field" mapstructure:"real-dbname-field" description:"Tag key for real database name" env:"PW_REAL_DBNAME_FIELD" default:"real_dbname"`
 	SystemIdentifierField string        `long:"system-identifier-field" mapstructure:"system-identifier-field" description:"Tag key for system identifier value" env:"PW_SYSTEM_IDENTIFIER_FIELD" default:"sys_id"`
diff --git a/internal/sinks/json.go b/internal/sinks/json.go
@@ -2,9 +2,10 @@ package sinks
 
 import (
 	"context"
-	"encoding/json"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
+
 	"github.com/cybertec-postgresql/pgwatch/v3/internal/log"
 	"github.com/cybertec-postgresql/pgwatch/v3/internal/metrics"
 	"gopkg.in/natefinch/lumberjack.v2"
@@ -17,7 +18,7 @@ import (
 type JSONWriter struct {
 	ctx context.Context
 	lw  *lumberjack.Logger
-	enc *json.Encoder
+	enc *jsoniter.Encoder
 }
 
 func NewJSONWriter(ctx context.Context, fname string) (*JSONWriter, error) {
@@ -27,7 +28,7 @@ func NewJSONWriter(ctx context.Context, fname string) (*JSONWriter, error) {
 		ctx: ctx,
 		lw:  &lumberjack.Logger{Filename: fname, Compress: true},
 	}
-	jw.enc = json.NewEncoder(jw.lw)
+	jw.enc = jsoniter.ConfigFastest.NewEncoder(jw.lw)
 	go jw.watchCtx()
 	return jw, nil
 }
diff --git a/internal/sinks/json_test.go b/internal/sinks/json_test.go
@@ -2,10 +2,11 @@ package sinks
 
 import (
 	"context"
-	"encoding/json"
 	"os"
 	"testing"
 
+	jsoniter "github.com/json-iterator/go"
+
 	"github.com/cybertec-postgresql/pgwatch/v3/internal/metrics"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -42,7 +43,7 @@ func TestJSONWriter_Write(t *testing.T) {
 	var data map[string]any
 	file, err := os.ReadFile(tempFile)
 	r.NoError(err)
-	err = json.Unmarshal(file, &data)
+	err = jsoniter.ConfigFastest.Unmarshal(file, &data)
 	r.NoError(err)
 	a.Equal(msg.MetricName, data["metric"])
 	a.Equal(len(msg.Data), len(data["data"].([]any)))
diff --git a/internal/sources/resolver.go b/internal/sources/resolver.go
@@ -9,14 +9,15 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"path"
 	"strings"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
+
 	"github.com/cybertec-postgresql/pgwatch/v3/internal/db"
 	"github.com/cybertec-postgresql/pgwatch/v3/internal/log"
 	pgx "github.com/jackc/pgx/v5"
@@ -75,7 +76,7 @@ func jsonTextToStringMap(jsonText string) (map[string]string, error) {
 		return retmap, nil
 	}
 	var iMap map[string]any
-	if err := json.Unmarshal([]byte(jsonText), &iMap); err != nil {
+	if err := jsoniter.ConfigFastest.Unmarshal([]byte(jsonText), &iMap); err != nil {
 		return nil, err
 	}
 	for k, v := range iMap {
diff --git a/internal/webserver/jwt.go b/internal/webserver/jwt.go
@@ -1,12 +1,13 @@
 package webserver
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
+
 	"github.com/golang-jwt/jwt/v5"
 )
 
@@ -35,7 +36,7 @@ func (Server *WebUIServer) handleLogin(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case "POST":
-		if err = json.NewDecoder(r.Body).Decode(&lr); err != nil {
+		if err = jsoniter.ConfigFastest.NewDecoder(r.Body).Decode(&lr); err != nil {
 			return
 		}
 		if !Server.IsCorrectPassword(lr) {
diff --git a/internal/webserver/jwt_test.go b/internal/webserver/jwt_test.go
@@ -2,17 +2,19 @@ package webserver
 
 import (
 	"bytes"
-	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 	"time"
 
 	"github.com/golang-jwt/jwt/v5"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/stretchr/testify/assert"
 )
 
+var json = jsoniter.ConfigFastest
+
 func TestIsCorrectPassword(t *testing.T) {
 	ts := &WebUIServer{CmdOpts: CmdOpts{WebUser: "user", WebPassword: "pass"}}
 	assert.True(t, ts.IsCorrectPassword(loginReq{Username: "user", Password: "pass"}))
diff --git a/internal/webserver/metric.go b/internal/webserver/metric.go
@@ -1,10 +1,11 @@
 package webserver
 
 import (
-	"encoding/json"
 	"io"
 	"net/http"
 
+	jsoniter "github.com/json-iterator/go"
+
 	"github.com/cybertec-postgresql/pgwatch/v3/internal/metrics"
 )
 
@@ -56,15 +57,15 @@ func (server *WebUIServer) GetMetrics() (res string, err error) {
 	if mr, err = server.metricsReaderWriter.GetMetrics(); err != nil {
 		return
 	}
-	b, _ := json.Marshal(mr.MetricDefs)
+	b, _ := jsoniter.ConfigFastest.Marshal(mr.MetricDefs)
 	res = string(b)
 	return
 }
 
 // UpdateMetric updates the stored metric information
 func (server *WebUIServer) UpdateMetric(name string, params []byte) error {
 	var m metrics.Metric
-	err := json.Unmarshal(params, &m)
+	err := jsoniter.ConfigFastest.Unmarshal(params, &m)
 	if err != nil {
 		return err
 	}
@@ -121,7 +122,7 @@ func (server *WebUIServer) handlePresets(w http.ResponseWriter, r *http.Request)
 // UpdatePreset updates the stored preset
 func (server *WebUIServer) UpdatePreset(name string, params []byte) error {
 	var p metrics.Preset
-	err := json.Unmarshal(params, &p)
+	err := jsoniter.ConfigFastest.Unmarshal(params, &p)
 	if err != nil {
 		return err
 	}
@@ -134,7 +135,7 @@ func (server *WebUIServer) GetPresets() (res string, err error) {
 	if mr, err = server.metricsReaderWriter.GetMetrics(); err != nil {
 		return
 	}
-	b, _ := json.Marshal(mr.PresetDefs)
+	b, _ := jsoniter.ConfigFastest.Marshal(mr.PresetDefs)
 	res = string(b)
 	return
 }
diff --git a/internal/webserver/metric_test.go b/internal/webserver/metric_test.go
@@ -2,7 +2,6 @@ package webserver
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"io"
 	"net/http"
diff --git a/internal/webserver/server_test.go b/internal/webserver/server_test.go
@@ -2,7 +2,6 @@ package webserver_test
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,6 +10,8 @@ import (
 	"strings"
 	"testing"
 
+	jsoniter "github.com/json-iterator/go"
+
 	"github.com/cybertec-postgresql/pgwatch/v3/internal/webserver"
 	"github.com/stretchr/testify/assert"
 )
@@ -123,7 +124,7 @@ func TestGetToken(t *testing.T) {
 		Password: "admin",
 	}
 
-	payload, err := json.Marshal(credentials)
+	payload, err := jsoniter.ConfigFastest.Marshal(credentials)
 	if err != nil {
 		fmt.Println("Error marshaling ", err)
 	}
diff --git a/internal/webserver/source.go b/internal/webserver/source.go
@@ -1,10 +1,11 @@
 package webserver
 
 import (
-	"encoding/json"
 	"io"
 	"net/http"
 
+	jsoniter "github.com/json-iterator/go"
+
 	"github.com/cybertec-postgresql/pgwatch/v3/internal/sources"
 )
 
@@ -56,7 +57,7 @@ func (server *WebUIServer) GetSources() (res string, err error) {
 	if dbs, err = server.sourcesReaderWriter.GetSources(); err != nil {
 		return
 	}
-	b, _ := json.Marshal(dbs)
+	b, _ := jsoniter.ConfigFastest.Marshal(dbs)
 	res = string(b)
 	return
 }
@@ -69,7 +70,7 @@ func (server *WebUIServer) DeleteSource(database string) error {
 // UpdateSource updates the configured source information
 func (server *WebUIServer) UpdateSource(params []byte) error {
 	var md sources.Source
-	err := json.Unmarshal(params, &md)
+	err := jsoniter.ConfigFastest.Unmarshal(params, &md)
 	if err != nil {
 		return err
 	}
diff --git a/internal/webserver/source_test.go b/internal/webserver/source_test.go
@@ -2,7 +2,6 @@ package webserver
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"io"
 	"net/http"
diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go
@@ -1,7 +1,6 @@
 package webserver
 
 import (
-	"encoding/json"
 	"io"
 	"io/fs"
 	"net/http"

Original file line number	Diff line number	Diff line change
`@@ -2,9 +2,10 @@ package db`
`2`	`2`
`3`	`3`	`import (`
`4`	`4`	`"context"`
`5`		`- "encoding/json"`
`6`	`5`	`"reflect"`
`7`	`6`
	`7`	`+ jsoniter "github.com/json-iterator/go"`
	`8`	`+`
`8`	`9`	`pgx "github.com/jackc/pgx/v5"`
`9`	`10`	`pgconn "github.com/jackc/pgx/v5/pgconn"`
`10`	`11`	`pgxpool "github.com/jackc/pgx/v5/pgxpool"`
`@@ -57,7 +58,7 @@ func MarshallParamToJSONB(v any) any {`
`57`	`58`	`return nil`
`58`	`59`	`}`
`59`	`60`	`}`
`60`		`- if b, err := json.Marshal(v); err == nil {`
	`61`	`+ if b, err := jsoniter.ConfigFastest.Marshal(v); err == nil {`
`61`	`62`	`return string(b)`
`62`	`63`	`}`
`63`	`64`	`return nil`
Original file line number	Diff line number	Diff line change
`@@ -490,9 +490,7 @@ func (r Reaper) FetchMetric(ctx context.Context, md sources.SourceConn, metric`
`490`	`490`	`data[0]["is_up"] = 0 // should be updated if the "instance_up" metric definition is changed`
`491`	`491`	`goto send_to_storageChannel`
`492`	`492`	`}`
`493`		`- l.`
`494`		`- WithFields(map[string]any{"source": md.Name, "metric": metricName}).`
`495`		`- WithError(err).Error("failed to fetch metrics")`
	`493`	`+ l.WithError(err).Error("failed to fetch metrics")`
`496`	`494`
`497`	`495`	`return nil, err`
`498`	`496`	`}`