Fixed type mappings when using the Storage Read API

fernst · fernst · commit 8f9d5de587e7 · 2022-05-04T19:55:21.000-04:00
diff --git a/src/main/java/io/cdap/plugin/gcp/bigquery/sqlengine/BigQuerySparkDatasetProducer.java b/src/main/java/io/cdap/plugin/gcp/bigquery/sqlengine/BigQuerySparkDatasetProducer.java
@@ -123,13 +123,13 @@ private Dataset<Row> convertFieldTypes(Dataset<Row> ds) {
       }
 
       // Handle Int types
-      if (fieldSchema.getType() == Schema.Type.INT && fieldSchema.getLogicalType() == null) {
+      if (fieldSchema.getType() == Schema.Type.INT) {
         LOG.trace("Converting field {} to Integer", fieldName);
         ds = ds.withColumn(fieldName, ds.col(fieldName).cast(DataTypes.IntegerType));
       }
 
       // Handle float types
-      if (fieldSchema.getType() == Schema.Type.FLOAT && fieldSchema.getLogicalType() == null) {
+      if (fieldSchema.getType() == Schema.Type.FLOAT) {
         LOG.trace("Converting field {} to Float", fieldName);
         ds = ds.withColumn(fieldName, ds.col(fieldName).cast(DataTypes.FloatType));
       }

Original file line number	Diff line number	Diff line change
`@@ -123,13 +123,13 @@ private Dataset<Row> convertFieldTypes(Dataset<Row> ds) {`
`123`	`123`	`}`
`124`	`124`
`125`	`125`	`// Handle Int types`
`126`		`- if (fieldSchema.getType() == Schema.Type.INT && fieldSchema.getLogicalType() == null) {`
	`126`	`+ if (fieldSchema.getType() == Schema.Type.INT) {`
`127`	`127`	`LOG.trace("Converting field {} to Integer", fieldName);`
`128`	`128`	`ds = ds.withColumn(fieldName, ds.col(fieldName).cast(DataTypes.IntegerType));`
`129`	`129`	`}`
`130`	`130`
`131`	`131`	`// Handle float types`
`132`		`- if (fieldSchema.getType() == Schema.Type.FLOAT && fieldSchema.getLogicalType() == null) {`
	`132`	`+ if (fieldSchema.getType() == Schema.Type.FLOAT) {`
`133`	`133`	`LOG.trace("Converting field {} to Float", fieldName);`
`134`	`134`	`ds = ds.withColumn(fieldName, ds.col(fieldName).cast(DataTypes.FloatType));`
`135`	`135`	`}`