Merge #156600

156600: sql: introduce a non-experimental version of SHOW FINGERPRINTS r=jeffswenson a=mw5h

In 2017, we implemented SHOW EXPERIMENTAL_FINGERPRINTS, which provides
FNV fingerprints of all the indexes in a table. The implementation of
this command involved casting all the columns in each index to either a
string or bytes value so that it could be consumed by fnv64(). This
meant that values that cast to the same string value were not
diffentiated by the hash function. For this reason, the feature was
deemed "experimental" and not for use by users.

SHOW EXPERIMENTAL_FINGERPRINTS was useful enough to find its way into
various DR and CDC tests to determine whether or not data was preserved
correctly.

This patch introduces SHOW FINGERPRINTS, which uses
crdb_internal.datums_to_bytes() to transform the input data into encoded
bytes for use by fnv64(). This version of fingerprinting should be more
sensitive to changes in data. Now, data that prints the same won't
necessarily produce the same hash. Additionally, adding NULL values will
impact the hash as well. FNV was originally chosen for SHOW
EXPERIMENTAL_FINGERPRINTS and not changed by this patch because it's
approximately four times as fast as SHA256/512 and cryptographic
security is not required by this particular use case.

This patch also changes the output to be a 0-padded hexidecimal
representation of the hash instead of an integer.

No changes to external references to SHOW EXPERIMENTAL_FINGERPRINTS are
made here so that other groups can migrate at their own pace.

Epic: [CRDB-49047](https://cockroachlabs.atlassian.net/browse/CRDB-49047)
Release note (sql change): We have introduced the "SHOW FINGERPRINTS FOR
TABLE" SQL command, which produces a FNV hash of each index a table. FNV
is used here for performance reasons.

Co-authored-by: Matt White <[email protected]>

Author: craig[bot]

docs/generated/sql/bnf/stmt_block.bnf

@@ -1232,6 +1232,7 @@ unreserved_keyword ::=
 	| 'FAILURE'
 	| 'FILES'
 	| 'FILTER'
+	| 'FINGERPRINTS'
 	| 'FIRST'
 	| 'FOLLOWING'
 	| 'FORMAT'
@@ -4174,6 +4175,7 @@ bare_label_keywords ::=
 	| 'FALSE'
 	| 'FAMILY'
 	| 'FILES'
+	| 'FINGERPRINTS'
 	| 'FIRST'
 	| 'FLOAT'
 	| 'FOLLOWING'

pkg/crosscluster/logical/logical_replication_job_test.go

@@ -1652,9 +1652,9 @@ func compareReplicatedTables(
 		indexB, err := catalog.MustFindIndexByName(descB, indexA.GetName())
 		require.NoError(t, err)
 
-		aFingerprintQuery, err := sql.BuildFingerprintQueryForIndex(descA, indexA, []string{})
+		aFingerprintQuery, err := sql.BuildExperimentalFingerprintQueryForIndex(descA, indexA, []string{})
 		require.NoError(t, err)
-		bFingerprintQuery, err := sql.BuildFingerprintQueryForIndex(descB, indexB, []string{})
+		bFingerprintQuery, err := sql.BuildExperimentalFingerprintQueryForIndex(descB, indexB, []string{})
 		require.NoError(t, err)
 		t.Logf("fingerprinting index %s", indexA.GetName())
 		runnerB.CheckQueryResults(t, bFingerprintQuery, runnerA.QueryStr(t, aFingerprintQuery))

pkg/sql/logictest/testdata/logic_test/show_fingerprints

@@ -8,6 +8,12 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 t_pkey   NULL
 t_b_idx  NULL
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey   NULL
+t_b_idx  NULL
+
 statement ok
 INSERT INTO t VALUES (1, 2, 3, 4), (5, 6, 7, 8), (9, 10, 11, 12)
 
@@ -18,28 +24,59 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 t_pkey   -7903300865687235210
 t_b_idx  -5073888452016928166
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey   461f727cc14973a5
+t_b_idx  d74ed1186a9d4682
+
+# Test that if the data doesn't change, neither do the hashes
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey   461f727cc14973a5
+t_b_idx  d74ed1186a9d4682
+
 # Test excluded columns
 query TT rowsort
 SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t WITH EXCLUDE COLUMNS = ('c')
 ----
 t_pkey   -2938394162542358272
 t_b_idx  -5073888452016928166
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t WITH EXCLUDE COLUMNS = ('c')
+----
+t_pkey   d74dbe186a97ae30
+t_b_idx  d74ed1186a9d4682
+
 # Test multiple excluded columns
 query TT rowsort
 SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t WITH EXCLUDE COLUMNS = ('a', 'b')
 ----
 t_pkey   -3539648437866042702
 t_b_idx  590700560494856532
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t WITH EXCLUDE COLUMNS = ('a', 'b')
+----
+t_pkey   0831f107b4ea89e0
+t_b_idx  af63bd4c8601b757
+
 # START TIMESTAMP is only for VIRTUAL CLUSTERS
 query error pgcode 22023 cannot use the START TIMESTAMP option when fingerprinting a table.
 SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t WITH START TIMESTAMP = '132412341234.000000'
 
+query error pgcode 22023 cannot use the START TIMESTAMP option when fingerprinting a table.
+SHOW FINGERPRINTS FROM TABLE t WITH START TIMESTAMP = '132412341234.000000'
+
 # EXCLUDE COLUMNS is only for tables
 query error pgcode 22023 cannot use the EXCLUDE COLUMNS option when fingerprinting a tenant.
 SHOW EXPERIMENTAL_FINGERPRINTS FROM VIRTUAL CLUSTER t WITH EXCLUDE COLUMNS = ('a', 'b')
 
+query error pgcode 22023 cannot use the EXCLUDE COLUMNS option when fingerprinting a tenant.
+SHOW FINGERPRINTS FROM VIRTUAL CLUSTER t WITH EXCLUDE COLUMNS = ('a', 'b')
+
 # Test a partial index. We expect this index to have the same fingerprint
 # as t_b_idx since the predicate covers all values.
 statement ok
@@ -52,6 +89,12 @@ t_pkey   -7903300865687235210
 t_b_idx  -5073888452016928166
 t_b_idx1  -5073888452016928166
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey    461f727cc14973a5
+t_b_idx   d74ed1186a9d4682
+t_b_idx1  d74ed1186a9d4682
 
 statement ok
 DROP INDEX t_b_idx1
@@ -66,6 +109,12 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 t_pkey   3722816579880544080
 t_b_idx  -8494698744159250398
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey   46bf827cc1222da6
+t_b_idx  d75fd1186a6bce95
+
 statement ok
 UPDATE t SET c = 10
 
@@ -76,6 +125,12 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 t_pkey   4547357529681250049
 t_b_idx  -8494698744159250398
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey   46bf977cc121fbd5
+t_b_idx  d75fd1186a6bce95
+
 statement ok
 UPDATE t SET d = 10
 
@@ -86,6 +141,12 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 t_pkey   492416650140211287
 t_b_idx  -8497500299788131628
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey   46bf977cc121fbcf
+t_b_idx  d75fd1186a6bce8f
+
 statement ok
 ALTER TABLE t ADD COLUMN e string;
 
@@ -97,6 +158,13 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 t_pkey   492416650140211287
 t_b_idx  -8497500299788131628
 
+# Because NULLs are encoded, the new FINGERPRINTS will pick up on the new column
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey   292198383001098d
+t_b_idx  d75fd1186a6bce8f
+
 statement ok
 UPDATE t SET e = 'foo' WHERE a = 1;
 
@@ -107,6 +175,12 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 t_pkey   1205834892498753533
 t_b_idx  -8497500299788131628
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey   7a7ee2be020b59bc
+t_b_idx  d75fd1186a6bce8f
+
 statement ok
 DROP INDEX t@t_b_idx
 
@@ -116,12 +190,22 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 ----
 t_pkey  1205834892498753533
 
+query TT
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey  7a7ee2be020b59bc
+
 # Make sure fully qualified table names work
 query TT
 SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE test.t
 ----
 t_pkey  1205834892498753533
 
+query TT
+SHOW FINGERPRINTS FROM TABLE test.t
+----
+t_pkey  7a7ee2be020b59bc
+
 statement ok
 CREATE TABLE "foo""bar" ("a""b" INT PRIMARY KEY, b INT, INDEX "id""x" (b))
 
@@ -136,6 +220,12 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE "foo""bar"
 foo"bar_pkey  590693963425091008
 id"x          590692863913460538
 
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE "foo""bar"
+----
+foo"bar_pkey  0831f907b4ea8440
+id"x          0831f807b4ea794a
+
 # BYTES is special cased so make sure tables with both BYTES and non-BYTES
 # columns work
 statement ok
@@ -149,6 +239,11 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE blocks
 ----
 blocks_pkey  -9087023432987872405
 
+query TT
+SHOW FINGERPRINTS FROM TABLE blocks
+----
+blocks_pkey  c0de5f63471cd038
+
 # Verify that we can show fingerprints from a read-only transaction (#39204).
 statement ok
 BEGIN TRANSACTION AS OF SYSTEM TIME '-1us'
@@ -158,6 +253,11 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t
 ----
 t_pkey  1205834892498753533
 
+query TT
+SHOW FINGERPRINTS FROM TABLE t
+----
+t_pkey  7a7ee2be020b59bc
+
 statement ok
 COMMIT
 
@@ -173,3 +273,10 @@ SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t_w_expr_index
 t_w_expr_index_pkey     -3347212893095932527
 t_w_expr_index_c_idx    -2777058403611971387
 t_w_expr_index_expr_idx -2777058403611971387
+
+query TT rowsort
+SHOW FINGERPRINTS FROM TABLE t_w_expr_index
+----
+t_w_expr_index_pkey      9842887cf772b8a5
+t_w_expr_index_c_idx     08321207b4eaafc3
+t_w_expr_index_expr_idx  08321207b4eaafc3

pkg/sql/logictest/testdata/logic_test/show_tenant_fingerprints

@@ -58,3 +58,53 @@ user testuser
 
 statement error user testuser does not have MANAGEVIRTUALCLUSTER system privilege
 SHOW EXPERIMENTAL_FINGERPRINTS FROM TENANT target
+
+
+# Test again with SHOW FINGERPRINTS
+user root
+
+let $fingerprint2
+SELECT fingerprint FROM [SHOW FINGERPRINTS FROM TENANT target] AS OF SYSTEM TIME $aost
+
+query T
+SELECT * FROM (SELECT 'fingerprint matches' FROM [ SHOW FINGERPRINTS FROM TENANT target] WHERE fingerprint = $fingerprint2) AS OF SYSTEM TIME $aost;
+----
+fingerprint matches
+
+# Test virtual cluster alias as well
+query T
+SELECT * FROM (SELECT 'fingerprint matches' FROM [ SHOW FINGERPRINTS FROM VIRTUAL CLUSTER target] WHERE fingerprint = $fingerprint2) AS OF SYSTEM TIME $aost;
+----
+fingerprint matches
+
+query TTTT
+SELECT * FROM (
+SELECT tenant_name, 'start_ts is NULL', 'end_ts is NOT NULL', 'fingerprint is non-zero'
+FROM [SHOW FINGERPRINTS FROM TENANT target]
+WHERE (start_ts IS NULL) AND (end_ts IS NOT NULL))
+AS OF SYSTEM TIME $aost;
+----
+target  start_ts is NULL  end_ts is NOT NULL  fingerprint is non-zero
+
+let $aost2
+SELECT cluster_logical_timestamp();
+
+
+query TTTT
+SELECT * FROM (
+SELECT tenant_name, 'start_ts is NOT NULL', 'end_ts is NOT NULL', 'fingerprint is non-zero'
+FROM [SHOW FINGERPRINTS FROM TENANT target WITH START TIMESTAMP = $aost]
+WHERE (start_ts IS NOT NULL) AND (end_ts IS NOT NULL))
+AS OF SYSTEM TIME $aost2;
+----
+target  start_ts is NOT NULL  end_ts is NOT NULL  fingerprint is non-zero
+
+# Test that the end timestamp has to be greater than equal to the start timestamp.
+statement error pgcode 22023 start timestamp.*is greater than the end timestamp.*
+SELECT * FROM [SHOW FINGERPRINTS FROM TENANT target WITH START TIMESTAMP = $aost2] AS OF SYSTEM TIME $aost;
+
+
+user testuser
+
+statement error user testuser does not have MANAGEVIRTUALCLUSTER system privilege
+SHOW FINGERPRINTS FROM TENANT target