sql/inspect: run INSPECT against a single index

Previously, the INSPECT command was a no-op that performed no actual
checking. This change implements the first consistency check for
INSPECT: index consistency checking, currently supporting a single
index at a time.

The consistency checking uses a full outer join between the primary
index and secondary index access paths to identify rows that exist
in one index but not the other, detecting potential corruption or
inconsistency issues.

This new functionality is integrated into the INSPECT job and
triggered through SCRUB when the enable_scrub_job setting is enabled.
Future work will expose this through direct INSPECT SQL commands.

Note for reviewers:
- Some tests currently generate internal errors in the SQL query
  engine. These are not logged as INSPECT issues for now and will be
  resolved in subsequent work.
- I include a crdb_internal.void_func() predicate in the query to
  prevent the optimizer from optimizing the join away. There were
  a couple of other options (add barrier option to CTE, materialize
  the CTE inline), but this was the preferred method.

Release note: None

Epic: CRDB-30356

Author: spilchen

pkg/sql/exec_util.go

@@ -2098,6 +2098,8 @@ type InspectTestingKnobs struct {
 	// OnInspectJobStart is called just before the inspect job begins execution.
 	// If it returns an error, the job fails immediately.
 	OnInspectJobStart func() error
+	// InspectIssueLogger is an override to the default issue logger.
+	InspectIssueLogger interface{}
 }
 
 // ModuleTestingKnobs implements the base.ModuleTestingKnobs interface.

pkg/sql/inspect/BUILD.bazel

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "inspect",
     srcs = [
+        "index_consistency_check.go",
         "inspect_job.go",
         "inspect_processor.go",
         "issue.go",
@@ -16,20 +17,31 @@ go_library(
         "//pkg/jobs",
         "//pkg/jobs/jobspb",
         "//pkg/roachpb",
+        "//pkg/security/username",
         "//pkg/settings",
         "//pkg/settings/cluster",
         "//pkg/sql",
+        "//pkg/sql/catalog",
+        "//pkg/sql/catalog/catenumpb",
         "//pkg/sql/catalog/descpb",
         "//pkg/sql/catalog/descs",
         "//pkg/sql/execinfra",
         "//pkg/sql/execinfrapb",
         "//pkg/sql/isql",
+        "//pkg/sql/lexbase",
+        "//pkg/sql/pgwire/pgcode",
+        "//pkg/sql/pgwire/pgerror",
         "//pkg/sql/physicalplan",
         "//pkg/sql/rowexec",
+        "//pkg/sql/sem/idxtype",
         "//pkg/sql/sem/tree",
+        "//pkg/sql/sessiondata",
+        "//pkg/sql/sessiondatapb",
+        "//pkg/sql/spanutils",
         "//pkg/sql/types",
         "//pkg/util/ctxgroup",
         "//pkg/util/log",
+        "//pkg/util/timeutil",
         "//pkg/util/tracing",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_redact//:redact",
@@ -39,6 +51,7 @@ go_library(
 go_test(
     name = "inspect_test",
     srcs = [
+        "index_consistency_check_test.go",
         "inspect_job_test.go",
         "inspect_processor_test.go",
         "issue_test.go",
@@ -48,14 +61,21 @@ go_test(
     embed = [":inspect"],
     deps = [
         "//pkg/base",
+        "//pkg/keys",
+        "//pkg/kv",
         "//pkg/roachpb",
         "//pkg/security/securityassets",
         "//pkg/security/securitytest",
         "//pkg/server",
         "//pkg/settings/cluster",
         "//pkg/sql",
+        "//pkg/sql/catalog",
+        "//pkg/sql/catalog/desctestutils",
         "//pkg/sql/execinfra",
         "//pkg/sql/execinfrapb",
+        "//pkg/sql/rowenc",
+        "//pkg/sql/sem/tree",
+        "//pkg/sql/sessiondata",
         "//pkg/testutils",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/sqlutils",