Merge pull request #113 from PowerDNS/protobuf-2gb

snapshot: allow DBI dumps >2GB

Author: wojas

go.mod

@@ -129,7 +129,11 @@ require (
 	golang.org/x/term v0.37.0 // indirect
 	golang.org/x/text v0.31.0 // indirect
 	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
-	google.golang.org/protobuf v1.36.10 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	pluginrpc.com/pluginrpc v0.5.0 // indirect
 )
+
+// See https://github.com/PowerDNS/lightningstream/issues/112
+// Can be removed once https://github.com/CrowdStrike/csproto/pull/208 is merged and released
+replace github.com/CrowdStrike/csproto => github.com/wojas/csproto v0.0.0-20260107092112-0e013c7984a2

go.sum

@@ -39,8 +39,6 @@ dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/CrowdStrike/csproto v0.35.0 h1:9vEV5PwDHXqinbobsI1Doy6kF5uHJyo4znYqXaUKQNs=
-github.com/CrowdStrike/csproto v0.35.0/go.mod h1:KVvLr7dlTgOmX5oT4cw9mZ0rhMIBG2LHb8yt9DMg7ic=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -322,6 +320,8 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
 github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo=
 github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=
+github.com/wojas/csproto v0.0.0-20260107092112-0e013c7984a2 h1:T9S4tRljv07jn5KNFyKhecgU6rbsq0OTxxar8ATxP/U=
+github.com/wojas/csproto v0.0.0-20260107092112-0e013c7984a2/go.mod h1:s4KQGP1RDkm7AyniRi6BUAWySd8RtXZmk1ZBm/CF1F8=
 github.com/wojas/go-healthz v0.2.0 h1:Pm+V2mCkEMvLoppiOW6zV/VpnbOfnwIA7F8Cd+ucmyg=
 github.com/wojas/go-healthz v0.2.0/go.mod h1:uAGwtiPYhG63AgJoWM+EFzSfwgW3rj58ZalNogNFRRk=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -473,8 +473,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
-google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

snapshot/snapshot.go

@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"github.com/CrowdStrike/csproto"
+	"github.com/c2h5oh/datasize"
 )
 
 // Protobuf field numbers
@@ -14,6 +15,12 @@ const (
 	FieldSnapshotCompatVersion = 4
 )
 
+// MaxFieldLength overrides the default 2GB limit on variable length protobuf
+// fields to allow larger dumps with our current schema.
+const (
+	MaxFieldLength = 100 * uint64(datasize.GB)
+)
+
 // Snapshot is the root object in a snapshot protobuf
 type Snapshot struct {
 	FormatVersion uint32 // version of this snapshot format
@@ -25,6 +32,7 @@ type Snapshot struct {
 func (s *Snapshot) Unmarshal(data []byte) error {
 	d := csproto.NewDecoder(data)
 	d.SetMode(csproto.DecoderModeFast)
+	d.SetMaxFieldLength(MaxFieldLength) // allow DBI dumps larger than 2GB
 	for d.More() {
 		tag, wireType, err := d.DecodeTag()
 		if err != nil {