kvcoord: remove call to Verify on responses

This check consumes CPU for no good reason. Network protocol (TCP) has
a checksum, also, lower layers (storage) verifies the checksum as well.

Sysbench-settings shows ~2% throughput increase:

                                                         │ before.txt  │             after.txt              │
                                                         │ queries/sec │ queries/sec  vs base               │
SysbenchSettings/a=oltp_read_write/nodes=3/cpu=8/conc=64   29.97k ± 2%   30.61k ± 2%  +2.16% (p=0.007 n=20)

                                                         │ before.txt  │             after.txt              │
                                                         │  txns/sec   │  txns/sec    vs base               │
SysbenchSettings/a=oltp_read_write/nodes=3/cpu=8/conc=64   1.498k ± 2%   1.531k ± 2%  +2.16% (p=0.007 n=20)

                                                         │ before.txt │             after.txt             │
                                                         │   ms/min   │   ms/min    vs base               │
SysbenchSettings/a=oltp_read_write/nodes=3/cpu=8/conc=64   12.56 ± 6%   11.78 ± 1%  -6.21% (p=0.001 n=20)

                                                         │ before.txt │             after.txt             │
                                                         │   ms/avg   │   ms/avg    vs base               │
SysbenchSettings/a=oltp_read_write/nodes=3/cpu=8/conc=64   42.72 ± 2%   41.80 ± 2%  -2.13% (p=0.007 n=20)

                                                         │ before.txt │             after.txt             │
                                                         │   ms/p95   │   ms/p95    vs base               │
SysbenchSettings/a=oltp_read_write/nodes=3/cpu=8/conc=64   65.65 ± 2%   64.47 ± 2%  -1.80% (p=0.046 n=20)

                                                         │ before.txt  │           after.txt           │
                                                         │   ms/max    │   ms/max     vs base          │
SysbenchSettings/a=oltp_read_write/nodes=3/cpu=8/conc=64   238.4 ± 15%   228.2 ± 23%  ~ (p=0.738 n=20)

Fixes: #145541

Release note: None

Author: iskettaneh

pkg/kv/kvclient/kvcoord/transport.go

@@ -17,6 +17,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/rpc"
 	"github.com/cockroachdb/cockroach/pkg/rpc/nodedialer"
 	"github.com/cockroachdb/cockroach/pkg/util"
+	"github.com/cockroachdb/cockroach/pkg/util/buildutil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/tracing"
 	"github.com/cockroachdb/errors"
@@ -208,14 +209,23 @@ func (gt *grpcTransport) sendBatch(
 	log.VEvent(ctx, 2, "sending batch request")
 	reply, err := iface.Batch(ctx, ba)
 	log.VEvent(ctx, 2, "received batch response")
-	// If we queried a remote node, perform extra validation.
-	if reply != nil && !rpc.IsLocal(iface) {
-		if err == nil {
-			for i := range reply.Responses {
-				err = reply.Responses[i].GetInner().Verify(ba.Requests[i].GetInner())
-				if err != nil {
-					log.Errorf(ctx, "verification of response for %s failed: %v", ba.Requests[i].GetInner(), err)
-					break
+
+	// We don't have any strong reason to keep verifying the checksum of the
+	// response. However, since this check has historically caught some bugs, we
+	// are keeping it in Test builds for not.
+	// TODO(ibrahim): There is a path to remove Value checksum computations and
+	// verifications. More details are available in:
+	// https://github.com/cockroachdb/cockroach/issues/145541#issuecomment-2917225539
+	if buildutil.CrdbTestBuild {
+		// If we queried a remote node, perform extra validation.
+		if reply != nil && !rpc.IsLocal(iface) {
+			if err == nil {
+				for i := range reply.Responses {
+					err = reply.Responses[i].GetInner().Verify(ba.Requests[i].GetInner())
+					if err != nil {
+						log.Errorf(ctx, "verification of response for %s failed: %v", ba.Requests[i].GetInner(), err)
+						break
+					}
 				}
 			}
 		}