[#28708] YSQL: fix tuple IN condition check logic

Summary:
Commit 71f5dcc fixes a bug in
YbIsTupleInRange, but it fails to apply the same changes to
YbNeedTupleRangeCheck.  This leaves a small, hard-to-hit hole in the
original bug #22800, leading to tserver crash.

Fix that.  Also clean up an unused variable from YbIsTupleInRange.

Jira: DB-18411

Test Plan:
On Almalinux 8:

    ./yb_build.sh fastdebug --gcc13 daemons initdb \
      --java-test 'org.yb.pgsql.TestPgRegressJoin' \
      --tp 1 -n 20 --stop-at-failure

Close: #28708

Reviewers: mtakahara

Reviewed By: mtakahara

Subscribers: yql

Differential Revision: https://phorge.dev.yugabyte.com/D46994

Author: jaki

src/postgres/src/backend/access/yb_access/yb_scan.c

@@ -1438,10 +1438,10 @@ YbShouldRecheckEquality(Oid column_typid, Oid value_typid)
 
 static bool
 YbNeedTupleRangeCheck(Datum value, TupleDesc bind_desc,
-					  int key_length, ScanKey keys[])
+					  int key_length, AttrNumber bind_key_attnums[])
 {
 	/* Move past header key. */
-	++keys;
+	++bind_key_attnums;
 	--key_length;
 
 	Oid			tupType = HeapTupleHeaderGetTypeId(DatumGetHeapTupleHeader(value));
@@ -1452,7 +1452,8 @@ YbNeedTupleRangeCheck(Datum value, TupleDesc bind_desc,
 	for (int i = 0; i < key_length; i++)
 	{
 		Oid			val_type = ybc_get_atttypid(val_tupdesc, i + 1);
-		Oid			column_type = ybc_get_atttypid(bind_desc, keys[i]->sk_attno);
+		Oid			column_type = ybc_get_atttypid(bind_desc,
+												   bind_key_attnums[i]);
 
 		if (YbShouldRecheckEquality(column_type, val_type))
 		{
@@ -1466,11 +1467,9 @@ YbNeedTupleRangeCheck(Datum value, TupleDesc bind_desc,
 
 static bool
 YbIsTupleInRange(Datum value, TupleDesc bind_desc,
-				 int key_length, ScanKey keys[],
-				 AttrNumber bind_key_attnums[])
+				 int key_length, AttrNumber bind_key_attnums[])
 {
 	/* Move past header key. */
-	++keys;
 	++bind_key_attnums;
 	--key_length;
 
@@ -1496,7 +1495,8 @@ YbIsTupleInRange(Datum value, TupleDesc bind_desc,
 	{
 		Datum		val = datum_values[i];
 		Oid			val_type = ybc_get_atttypid(val_tupdesc, i + 1);
-		Oid			column_type = ybc_get_atttypid(bind_desc, bind_key_attnums[i]);
+		Oid			column_type = ybc_get_atttypid(bind_desc,
+												   bind_key_attnums[i]);
 
 		if (!YbShouldRecheckEquality(column_type, val_type))
 			continue;
@@ -1840,7 +1840,7 @@ YbBindSearchArray(YbScanDesc ybScan, YbScanPlan scan_plan,
 										  YbNeedTupleRangeCheck(elem_values[0],
 																scan_plan->bind_desc,
 																length_of_key,
-																&ybScan->keys[i]));
+																&scan_plan->bind_key_attnums[i]));
 
 	bool		retain_nulls = YbSearchArrayRetainNulls(key);
 	bool		bind_to_null = false;
@@ -1883,7 +1883,6 @@ YbBindSearchArray(YbScanDesc ybScan, YbScanPlan scan_plan,
 				if (!YbIsTupleInRange(elem_values[j],
 									  scan_plan->bind_desc,
 									  length_of_key,
-									  &ybScan->keys[i],
 									  &scan_plan->bind_key_attnums[i]))
 					continue;
 			}

src/postgres/src/test/regress/expected/yb.orig.join_batching.out

@@ -904,7 +904,7 @@ explain (costs off) /*+Leading((f1 f2)) YbBatchedNL(f1 f2)*/ select * from float
 
 drop table floattable1;
 drop table floattable2;
-create table ss1(a bigint, b int, primary key(a asc, b asc));
+create table ss1(a bigint, b int, primary key(b asc, a asc));
 create table ss2(a int, b bigint, primary key(a asc, b asc));
 -- 0x8000000000000001 should not join with 1 and 0x8000000000000002 should not join with 2
 insert into ss1 values (1,2), (3,4), (x'8000000000000001'::bigint, 1);
@@ -933,7 +933,7 @@ explain (costs off) /*+Leading((ss2 ss1))*/ select * from ss1, ss2 where ss1.a =
    Join Filter: ((ss1.a = ss2.a) AND (ss1.b = ss2.b))
    ->  Index Scan using ss2_pkey on ss2
    ->  Index Scan using ss1_pkey on ss1
-         Index Cond: (ROW(a, b) = ANY (ARRAY[ROW(ss2.a, ss2.b), ROW($1, $4), ROW($2, $5)]))
+         Index Cond: (ROW(b, a) = ANY (ARRAY[ROW(ss2.b, ss2.a), ROW($1, $4), ROW($2, $5)]))
 (5 rows)
 
 /*+Leading((ss2 ss1))*/ select * from ss1, ss2 where ss1.a = ss2.a and ss1.b = ss2.b;
@@ -950,7 +950,7 @@ explain (costs off) /*+Leading((ss2 ss1))*/ select * from ss1, ss2 where ss1.a =
    Join Filter: ((ss1.a = ss2.a) AND (ss1.b = ss2.b))
    ->  Index Scan using ss2_pkey on ss2
    ->  Index Scan using ss1_pkey on ss1
-         Index Cond: ((ROW(a, b) = ANY (ARRAY[ROW(ss2.a, ss2.b), ROW($1, $4), ROW($2, $5)])) AND (b < 10))
+         Index Cond: ((ROW(b, a) = ANY (ARRAY[ROW(ss2.b, ss2.a), ROW($1, $4), ROW($2, $5)])) AND (b < 10))
 (5 rows)
 
 /*+Leading((ss2 ss1))*/ select * from ss1, ss2 where ss1.a = ss2.a and ss1.b = ss2.b and ss1.b < 10;

src/postgres/src/test/regress/sql/yb.orig.join_batching.sql

@@ -224,7 +224,7 @@ explain (costs off) /*+Leading((f1 f2)) YbBatchedNL(f1 f2)*/ select * from float
 drop table floattable1;
 drop table floattable2;
 
-create table ss1(a bigint, b int, primary key(a asc, b asc));
+create table ss1(a bigint, b int, primary key(b asc, a asc));
 create table ss2(a int, b bigint, primary key(a asc, b asc));
 -- 0x8000000000000001 should not join with 1 and 0x8000000000000002 should not join with 2
 insert into ss1 values (1,2), (3,4), (x'8000000000000001'::bigint, 1);