Bug fix issue 147: BUG with reading table that contains copied map

Issue with reading a table with a MAP column, but only when the table was created by copying an existing table.
The bug was caused by an indexing issue when reading map values from an arrow batch.  The occurrence of the bug was dependent on the configuration of batches the result set was broken down into.
Added a new test using the arrow batch returned in the error scenario.  Also fixed an existing test that had been masking this bug.

Signed-off-by: Raymond Cypher <[email protected]>

Author: rcypher-databricks

internal/rows/arrowbased/arrowRows_test.go

@@ -1221,7 +1221,7 @@ func TestArrowRowScanner(t *testing.T) {
 			"[\"2021-07-01 05:43:28 +0000 UTC\",\"-2022-08-13 14:01:01 +0000 UTC\",null]",
 			"[\"Gr8=\",\"D/8=\",null]",
 			"[[1,2,3],[4,5,6],null]",
-			"[{\"key1\":1,\"key2\":2},{\"key1\":1,\"key2\":2},null]",
+			"[{\"key1\":1,\"key2\":2},{\"key3\":3,\"key4\":4},null]",
 			"[{\"Field1\":77,\"Field2\":\"2020-12-31 00:00:00 +0000 UTC\"},{\"Field1\":13,\"Field2\":\"-2020-12-31 00:00:00 +0000 UTC\"},{\"Field1\":null,\"Field2\":null}]",
 			"[5.15,123.45,null]",
 			"[\"2020-12-31 00:00:00 +0000 UTC\",\"-2020-12-31 00:00:00 +0000 UTC\",null]",
@@ -1287,6 +1287,52 @@ func TestArrowRowScanner(t *testing.T) {
 
 	})
 
+	t.Run("Retrieve values - maps issue 147", func(t *testing.T) {
+		// This is a test for a bug reported as github issue 147
+		// After copying a table with a column of type 'MAP<STRING, STRING>' querying the copy
+		// would return the map value for the first row in all rows.
+		// This was caused by an indexing bug when retrieving map values that showed up based on
+		// how the result set was broken into arrow batches.
+		expected := [][]driver.Value{
+			{1, map[string]string{"name": "alice2"}},
+			{2, map[string]string{"name": "bob2"}},
+			{3, map[string]string{"name": "jon2"}},
+		}
+
+		executeStatementResp := cli_service.TExecuteStatementResp{}
+		loadTestData(t, "issue147.json", &executeStatementResp)
+
+		config := config.WithDefaults()
+		config.UseArrowNativeTimestamp = true
+		config.UseArrowNativeComplexTypes = true
+		config.UseArrowNativeDecimal = false
+		config.UseArrowNativeIntervalTypes = false
+		d, err := NewArrowRowScanner(executeStatementResp.DirectResults.ResultSetMetadata, executeStatementResp.DirectResults.ResultSet.Results, config, nil, context.Background())
+		assert.Nil(t, err)
+
+		ars := d.(*arrowRowScanner)
+
+		dest := make([]driver.Value, len(executeStatementResp.DirectResults.ResultSetMetadata.Schema.Columns))
+
+		for i := range expected {
+			err = ars.ScanRow(dest, int64(i))
+			assert.Nil(t, err)
+
+			var id int
+			s := dest[0].(string)
+			err := json.Unmarshal([]byte(s), &id)
+			assert.Nil(t, err)
+			assert.Equal(t, expected[i][0], id)
+
+			var foo map[string]string
+			s = dest[1].(string)
+			err = json.Unmarshal([]byte(s), &foo)
+			assert.Nil(t, err)
+			assert.Equal(t, expected[i][1], foo)
+		}
+
+	})
+
 	t.Run("Retrieve values - structs", func(t *testing.T) {
 		expected := []driver.Value{
 			"{\"f1\":1,\"f2\":\"-0450-11-13 00:00:00 +0000 UTC\",\"f3\":\"-2022-08-13 14:01:01 +0000 UTC\",\"f4\":{\"5\":5,\"6\":7},\"f5\":{\"ield1\":7,\"Field2\":\"-0450-11-13 00:00:00 +0000 UTC\"}}",

internal/rows/arrowbased/columnValues.go

@@ -204,7 +204,7 @@ func (mvc *mapValueContainer) Value(i int) (any, error) {
 		len := e - s
 		r := "{"
 		for i := int64(0); i < len; i++ {
-			k, err := mvc.keys.Value(int(i))
+			k, err := mvc.keys.Value(int(i + s))
 			if err != nil {
 				return nil, err
 			}
@@ -214,13 +214,13 @@ func (mvc *mapValueContainer) Value(i int) (any, error) {
 				return nil, err
 			}
 
-			v, err := mvc.values.Value(int(i))
+			v, err := mvc.values.Value(int(i + s))
 			if err != nil {
 				return nil, err
 			}
 
 			var b string
-			if mvc.values.IsNull(int(i)) {
+			if mvc.values.IsNull(int(i + s)) {
 				b = "null"
 			} else if mvc.complexValue {
 				b = v.(string)

internal/rows/arrowbased/testdata/issue147.json

@@ -0,0 +1,92 @@
+{
+ "status": {
+  "statusCode": "SUCCESS_STATUS"
+ },
+ "operationHandle": {
+  "operationId": {
+   "guid": "Ae4qWOAZGPWte+TeoMZC2w==",
+   "secret": "M41SnYJyRuuEgstBlGaDnQ=="
+  },
+  "operationType": "EXECUTE_STATEMENT",
+  "hasResultSet": true
+ },
+ "directResults": {
+  "operationStatus": {
+   "status": {
+    "statusCode": "SUCCESS_STATUS"
+   },
+   "operationState": "FINISHED_STATE",
+   "operationStarted": 1690227170051,
+   "operationCompleted": 1690227170417
+  },
+  "resultSetMetadata": {
+   "status": {
+    "statusCode": "SUCCESS_STATUS"
+   },
+   "schema": {
+    "columns": [
+     {
+      "columnName": "id",
+      "typeDesc": {
+       "types": [
+        {
+         "primitiveEntry": {
+          "type": "STRING_TYPE"
+         }
+        }
+       ]
+      },
+      "position": 1,
+      "comment": ""
+     },
+     {
+      "columnName": "myMap",
+      "typeDesc": {
+       "types": [
+        {
+         "primitiveEntry": {
+          "type": "MAP_TYPE"
+         }
+        }
+       ]
+      },
+      "position": 2,
+      "comment": ""
+     }
+    ]
+   },
+   "resultFormat": "ARROW_BASED_SET",
+   "lz4Compressed": false,
+   "arrowSchema": "/////6ACAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAIAAAC0AQAABAAAAGb+//8UAAAA2AAAAIABAAAAABEBfAEAAAIAAACEAAAABAAAABz+//8IAAAAWAAAAE8AAAB7InR5cGUiOiJtYXAiLCJrZXlUeXBlIjoic3RyaW5nIiwidmFsdWVUeXBlIjoic3RyaW5nIiwidmFsdWVDb250YWluc051bGwiOnRydWV9ABcAAABTcGFyazpEYXRhVHlwZTpKc29uVHlwZQCY/v//CAAAABwAAAATAAAATUFQPFNUUklORywgU1RSSU5HPgAWAAAAU3Bhcms6RGF0YVR5cGU6U3FsTmFtZQAAAQAAAAQAAACq////FAAAABQAAACIAAAAAAAADYQAAAAAAAAAAgAAAEgAAAAEAAAAdv///xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAADo/v//BQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAAFAAAABQAAAAAAAAFEAAAAAAAAAAAAAAAKP///wMAAABrZXkANP///wcAAABlbnRyaWVzAET///8FAAAAbXlNYXAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAACQAAAAlAAAAAAABQGQAAAAAgAAAEgAAAAEAAAAyP///wgAAAAUAAAACAAAACJzdHJpbmciAAAAABcAAABTcGFyazpEYXRhVHlwZTpKc29uVHlwZQAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABTVFJJTkcAABYAAABTcGFyazpEYXRhVHlwZTpTcWxOYW1lAAAAAAAABAAEAAQAAAACAAAAaWQAAAAAAAA=",
+   "cacheLookupResult": "LOCAL_CACHE_HIT",
+   "uncompressedBytes": 512,
+   "compressedBytes": 512
+  },
+  "resultSet": {
+   "status": {
+    "statusCode": "SUCCESS_STATUS"
+   },
+   "hasMoreRows": false,
+   "results": {
+    "startRowOffset": 0,
+    "rows": [],
+    "arrowBatches": [
+     {
+      "batch": "/////2gBAAAUAAAAAAAAAAwAFgAOABUAEAAEAAwAAACQAAAAAAAAAAAABAAQAAAAAAMKABgADAAIAAQACgAAABQAAADYAAAAAwAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAQAAAAAAAAAIAAAAAAAAABAAAAAAAAAAGAAAAAAAAAADAAAAAAAAACAAAAAAAAAAAQAAAAAAAAAoAAAAAAAAABAAAAAAAAAAOAAAAAAAAAABAAAAAAAAAEAAAAAAAAAAAQAAAAAAAABIAAAAAAAAABAAAAAAAAAAWAAAAAAAAAAMAAAAAAAAAGgAAAAAAAAAAQAAAAAAAABwAAAAAAAAABAAAAAAAAAAgAAAAAAAAAAOAAAAAAAAAAAAAAAFAAAAAwAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAABAAAAAgAAAAMAAAAxMjMAAAAAAAcAAAAAAAAAAAAAAAEAAAACAAAAAwAAAAcAAAAAAAAABwAAAAAAAAAAAAAABAAAAAgAAAAMAAAAbmFtZW5hbWVuYW1lAAAAAAcAAAAAAAAAAAAAAAYAAAAKAAAADgAAAGFsaWNlMmJvYjJqb24yAAA=",
+      "rowCount": 3
+     }
+    ]
+   }
+  },
+  "closeOperation": {
+   "status": {
+    "statusCode": "SUCCESS_STATUS"
+   }
+  }
+ },
+ "executionRejected": false,
+ "maxClusterCapacity": 10,
+ "queryCost": 0.5,
+ "currentClusterLoad": 1,
+ "idempotencyType": "IDEMPOTENT"
+}