Skip to content

Commit e7098bd

Browse files
authored
fix: correct spanner column schema type parser (#34868)
Regex for parsing array types did not take into account sized primitive types, eg ARRAY<STRING(MAX)> Fixes #34863
1 parent ca11207 commit e7098bd

File tree

2 files changed

+24
-15
lines changed

2 files changed

+24
-15
lines changed

sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchema.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,10 @@ private static Type parseSpannerType(String spannerType, Dialect dialect) {
208208
return Type.bytes();
209209
}
210210
if (spannerType.startsWith("ARRAY")) {
211-
// Substring "ARRAY<xxx>"
212-
Pattern pattern = Pattern.compile("ARRAY<([^>(]+)>");
211+
// find 'xxx' in string ARRAY<xxxxx>
212+
// Graph DBs may have suffixes, eg ARRAY<FLOAT32>(vector_length=>256)
213+
//
214+
Pattern pattern = Pattern.compile("ARRAY<([^>]+)>");
213215
Matcher matcher = pattern.matcher(originalSpannerType);
214216

215217
if (matcher.find()) {

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerSchemaTest.java

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,22 +33,26 @@ public class SpannerSchemaTest {
3333
public void testSingleTable() throws Exception {
3434
SpannerSchema schema =
3535
SpannerSchema.builder()
36-
.addColumn("test", "pk", "STRING(48)")
37-
.addKeyPart("test", "pk", false)
38-
.addColumn("test", "maxKey", "STRING(MAX)")
39-
.addColumn("test", "numericVal", "NUMERIC")
40-
.addColumn("test", "jsonVal", "JSON")
41-
.addColumn("test", "protoVal", "PROTO<customer.app.TestMessage>")
42-
.addColumn("test", "enumVal", "ENUM<customer.app.TestEnum>")
43-
.addColumn("test", "tokens", "TOKENLIST")
44-
.addColumn("test", "uuidCol", "UUID")
45-
.addColumn("test", "arrayVal", "ARRAY<FLOAT32>(vector_length=>256)")
46-
.addColumn("test", "arrayValue", "ARRAY<FLOAT32>")
36+
.addColumn("test", "pk_0", "STRING(48)")
37+
.addKeyPart("test", "pk_0", false)
38+
.addColumn("test", "maxKey_1", "STRING(MAX)")
39+
.addColumn("test", "numericVal_2", "NUMERIC")
40+
.addColumn("test", "jsonVal_3", "JSON")
41+
.addColumn("test", "protoVal_4", "PROTO<customer.app.TestMessage>")
42+
.addColumn("test", "enumVal_5", "ENUM<customer.app.TestEnum>")
43+
.addColumn("test", "tokens_6", "TOKENLIST")
44+
.addColumn("test", "uuidCol_7", "UUID")
45+
.addColumn("test", "arrayVal_8", "ARRAY<FLOAT32>(vector_length=>256)")
46+
.addColumn("test", "sizedArrayVal_9", "ARRAY<STRING(MAX)>")
47+
.addColumn("test", "sizedByteVal_10", "ARRAY<BYTES(1024)>")
48+
.addColumn("test", "hexSizedByteVal_11", "ARRAY<BYTES(0x400)>")
49+
.addColumn("test", "arrayValue_12", "ARRAY<FLOAT32>")
4750
.build();
4851

4952
assertEquals(1, schema.getTables().size());
50-
assertEquals(10, schema.getColumns("test").size());
53+
assertEquals(13, schema.getColumns("test").size());
5154
assertEquals(1, schema.getKeyParts("test").size());
55+
assertEquals(Type.numeric(), schema.getColumns("test").get(2).getType());
5256
assertEquals(Type.json(), schema.getColumns("test").get(3).getType());
5357
assertEquals(
5458
Type.proto("customer.app.TestMessage"), schema.getColumns("test").get(4).getType());
@@ -57,7 +61,10 @@ public void testSingleTable() throws Exception {
5761
assertEquals(Type.bytes(), schema.getColumns("test").get(6).getType());
5862
assertEquals(Type.string(), schema.getColumns("test").get(7).getType());
5963
assertEquals(Type.array(Type.float32()), schema.getColumns("test").get(8).getType());
60-
assertEquals(Type.array(Type.float32()), schema.getColumns("test").get(9).getType());
64+
assertEquals(Type.array(Type.string()), schema.getColumns("test").get(9).getType());
65+
assertEquals(Type.array(Type.bytes()), schema.getColumns("test").get(10).getType());
66+
assertEquals(Type.array(Type.bytes()), schema.getColumns("test").get(11).getType());
67+
assertEquals(Type.array(Type.float32()), schema.getColumns("test").get(12).getType());
6168
}
6269

6370
@Test

0 commit comments

Comments
 (0)