Skip to content

Commit b42cd5d

Browse files
vikasrathee-csvikasrathee-cs
committed
[PLUGIN-1938] Fix Google sheets blank row fix
1 parent a3b9e07 commit b42cd5d

File tree

2 files changed

+227
-8
lines changed

2 files changed

+227
-8
lines changed

‎src/main/java/io/cdap/plugin/google/sheets/source/SheetTransformer.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,13 @@ public class SheetTransformer {
4444
/**
4545
* Returns the StructuredRecord.
4646
*
47-
* @param rowRecord The rowRecord with
48-
* @param schema The schema with
49-
* @param extractMetadata The extractMetadata with
50-
* @param metadataRecordName The metadataRecordName with
51-
* @param addNames The addNames with
47+
* @param rowRecord The rowRecord with
48+
* @param schema The schema with
49+
* @param extractMetadata The extractMetadata with
50+
* @param metadataRecordName The metadataRecordName with
51+
* @param addNames The addNames with
5252
* @param spreadsheetFieldName The spreadsheetFieldName with
53-
* @param sheetFieldName The sheetFieldName
53+
* @param sheetFieldName The sheetFieldName
5454
* @return The StructuredRecord
5555
*/
5656
public static StructuredRecord transform(RowRecord rowRecord, Schema schema, boolean extractMetadata,
@@ -69,8 +69,9 @@ public static StructuredRecord transform(RowRecord rowRecord, Schema schema, boo
6969
builder.set(metadataRecordName, rowRecord.getMetadata());
7070
} else {
7171
ComplexSingleValueColumn complexSingleValueColumn = rowRecord.getHeaderedCells().get(name);
72-
if (complexSingleValueColumn == null || complexSingleValueColumn.getData() == null
73-
|| complexSingleValueColumn.getSubColumns() == null || complexSingleValueColumn.getSubColumns().isEmpty()) {
72+
if (complexSingleValueColumn == null ||
73+
(!field.getSchema().getNonNullable().getType()
74+
.equals(Schema.Type.RECORD) && complexSingleValueColumn.getData() == null)) {
7475
builder.set(name, null);
7576
} else {
7677
processCellData(builder, field, complexSingleValueColumn);
@@ -120,6 +121,9 @@ private static void processCellData(StructuredRecord.Builder builder, Schema.Fie
120121
builder.set(fieldName, effectiveValue.getNumberValue());
121122
}
122123
} else if (Schema.Type.RECORD.equals(fieldType)) {
124+
if (complexSingleValueColumn.getSubColumns() == null || complexSingleValueColumn.getSubColumns().isEmpty()) {
125+
throw new IllegalArgumentException("Columns are not present in sheet which are defined in the Record schema");
126+
}
123127
builder.set(fieldName, processRecord(fieldSchema.getNonNullable(), complexSingleValueColumn));
124128
}
125129
}

‎src/test/java/io/cdap/plugin/google/sheets/source/SheetTransformerTest.java‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
/*
2+
* Copyright © 2025 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.google.sheets.source;
18+
19+
import com.google.api.services.sheets.v4.model.CellData;
20+
import com.google.api.services.sheets.v4.model.ExtendedValue;
21+
import io.cdap.cdap.api.data.format.StructuredRecord;
22+
import io.cdap.cdap.api.data.schema.Schema;
23+
import io.cdap.plugin.google.sheets.source.utils.ComplexSingleValueColumn;
24+
import io.cdap.plugin.google.sheets.source.utils.RowRecord;
25+
import org.junit.Assert;
26+
import org.junit.Test;
27+
28+
import java.time.LocalDate;
29+
import java.time.ZoneId;
30+
import java.time.ZoneOffset;
31+
import java.time.ZonedDateTime;
32+
import java.time.temporal.ChronoField;
33+
import java.time.temporal.ChronoUnit;
34+
import java.util.HashMap;
35+
import java.util.Map;
36+
37+
public class SheetTransformerTest {
38+
39+
private static final String SPREADSHEET_NAME = "TestSpreadsheet";
40+
private static final String SHEET_TITLE = "TestSheet";
41+
private static final String METADATA_RECORD_NAME = "metadata";
42+
private static final String SPREADSHEET_FIELD_NAME = "spreadsheetName";
43+
private static final String SHEET_FIELD_NAME = "sheetTitle";
44+
45+
@Test
46+
public void transform_testAllDataTypes_runSuccessfully() {
47+
Schema schema =
48+
Schema.recordOf("record",
49+
Schema.Field.of("string_field", Schema.nullableOf(Schema.of(Schema.Type.STRING))),
50+
Schema.Field.of("double_field", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))),
51+
Schema.Field.of("boolean_field", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))),
52+
Schema.Field.of("date_field",
53+
Schema.nullableOf(Schema.of(Schema.LogicalType.DATE))),
54+
Schema.Field.of("ts_field",
55+
Schema.nullableOf(Schema.of(Schema.LogicalType.TIMESTAMP_MILLIS))),
56+
Schema.Field.of("long_field", Schema.nullableOf(Schema.of(Schema.Type.LONG)))
57+
);
58+
59+
Map<String, ComplexSingleValueColumn> headeredCells = new HashMap<>();
60+
ComplexSingleValueColumn stringColumn = new ComplexSingleValueColumn();
61+
stringColumn.setData(new CellData()
62+
.setFormattedValue("test_string")
63+
.setEffectiveValue(new ExtendedValue().setStringValue("test_string")));
64+
headeredCells.put("string_field", stringColumn);
65+
66+
ComplexSingleValueColumn doubleColumn = new ComplexSingleValueColumn();
67+
doubleColumn.setData(new CellData()
68+
.setFormattedValue("123.45")
69+
.setEffectiveValue(new ExtendedValue().setNumberValue(123.45)));
70+
headeredCells.put("double_field", doubleColumn);
71+
72+
ComplexSingleValueColumn booleanColumn = new ComplexSingleValueColumn();
73+
booleanColumn.setData(new CellData()
74+
.setFormattedValue("TRUE")
75+
.setEffectiveValue(new ExtendedValue().setBoolValue(true)));
76+
headeredCells.put("boolean_field", booleanColumn);
77+
78+
// Corresponds to 2023-10-27
79+
ComplexSingleValueColumn dateColumn = new ComplexSingleValueColumn();
80+
dateColumn.setData(new CellData()
81+
.setUserEnteredValue(new ExtendedValue().setNumberValue(45226.0)));
82+
headeredCells.put("date_field", dateColumn);
83+
84+
// Corresponds to 2023-10-26 09:59:59 UTC
85+
double tsValue = 45225.4166666667;
86+
ComplexSingleValueColumn tsColumn = new ComplexSingleValueColumn();
87+
tsColumn.setData(new CellData()
88+
.setUserEnteredValue(new ExtendedValue().setNumberValue(tsValue)));
89+
headeredCells.put("ts_field", tsColumn);
90+
91+
// Corresponds to 1 day
92+
double longValue = 1.0;
93+
ComplexSingleValueColumn longColumn = new ComplexSingleValueColumn();
94+
longColumn.setData(new CellData()
95+
.setUserEnteredValue(new ExtendedValue().setNumberValue(longValue)));
96+
headeredCells.put("long_field", longColumn);
97+
98+
RowRecord rowRecord = new RowRecord(SPREADSHEET_NAME, SHEET_TITLE, null, headeredCells, false);
99+
100+
StructuredRecord transformed = SheetTransformer.transform(rowRecord, schema, false, null, false, null, null);
101+
102+
Assert.assertEquals("test_string", transformed.get("string_field"));
103+
Assert.assertEquals(123.45, transformed.get("double_field"), 0.001);
104+
Assert.assertEquals(true, transformed.get("boolean_field"));
105+
Assert.assertEquals(LocalDate.of(2023, 10, 27), transformed.getDate("date_field"));
106+
107+
long dayMicros = ChronoField.MICRO_OF_DAY.range().getMaximum();
108+
ZonedDateTime expectedTimestamp = ZonedDateTime.of(1899, 12, 30, 0, 0, 0, 0,
109+
ZoneId.ofOffset("UTC", ZoneOffset.UTC))
110+
.plus((long) (tsValue * dayMicros), ChronoUnit.MICROS);
111+
Assert.assertEquals(expectedTimestamp.toInstant().toEpochMilli(),
112+
transformed.getTimestamp("ts_field").toInstant().toEpochMilli());
113+
114+
long expectedLong = (long) (longValue * dayMicros / 1000);
115+
Assert.assertEquals(expectedLong, (long) transformed.get("long_field"));
116+
}
117+
118+
@Test
119+
public void transform_addMetadata_metadataColumnAdded() {
120+
Schema schema = Schema.recordOf("record",
121+
Schema.Field.of("string_field", Schema.nullableOf(Schema.of(Schema.Type.STRING))),
122+
Schema.Field.of(METADATA_RECORD_NAME, Schema.nullableOf(Schema.mapOf(
123+
Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.STRING))))
124+
);
125+
126+
Map<String, ComplexSingleValueColumn> headeredCells = new HashMap<>();
127+
ComplexSingleValueColumn stringColumn = new ComplexSingleValueColumn();
128+
stringColumn.setData(new CellData()
129+
.setFormattedValue("test_string"));
130+
headeredCells.put("string_field", stringColumn);
131+
132+
Map<String, String> metadataMap = new HashMap<>();
133+
metadataMap.put("spreadsheetName", SPREADSHEET_NAME);
134+
metadataMap.put("sheetTitle", SHEET_TITLE);
135+
136+
RowRecord rowRecord = new RowRecord(SPREADSHEET_NAME, SHEET_TITLE, metadataMap, headeredCells, false);
137+
138+
StructuredRecord transformed = SheetTransformer.transform(rowRecord, schema, true, METADATA_RECORD_NAME,
139+
false, null, null);
140+
141+
Assert.assertEquals("test_string", transformed.get("string_field"));
142+
Assert.assertEquals(metadataMap, transformed.get(METADATA_RECORD_NAME));
143+
}
144+
145+
@Test
146+
public void transform_WithAddedNames_sheetAndSpreadsheetFieldsAdded() {
147+
Schema schema = Schema.recordOf("record",
148+
Schema.Field.of("string_field", Schema.nullableOf(Schema.of(Schema.Type.STRING))),
149+
Schema.Field.of(SPREADSHEET_FIELD_NAME,
150+
Schema.nullableOf(Schema.of(Schema.Type.STRING))),
151+
Schema.Field.of(SHEET_FIELD_NAME, Schema.nullableOf(Schema.of(Schema.Type.STRING)))
152+
);
153+
154+
Map<String, ComplexSingleValueColumn> headeredCells = new HashMap<>();
155+
ComplexSingleValueColumn stringColumn = new ComplexSingleValueColumn();
156+
stringColumn.setData(new CellData()
157+
.setFormattedValue("test_string"));
158+
headeredCells.put("string_field", stringColumn);
159+
160+
RowRecord rowRecord = new RowRecord(SPREADSHEET_NAME, SHEET_TITLE, null, headeredCells, false);
161+
162+
StructuredRecord transformed = SheetTransformer.transform(rowRecord, schema, false, null,
163+
true, SPREADSHEET_FIELD_NAME, SHEET_FIELD_NAME);
164+
165+
Assert.assertEquals("test_string", transformed.get("string_field"));
166+
Assert.assertEquals(SPREADSHEET_NAME, transformed.get(SPREADSHEET_FIELD_NAME));
167+
Assert.assertEquals(SHEET_TITLE, transformed.get(SHEET_FIELD_NAME));
168+
}
169+
170+
@Test
171+
public void transform_WithNullValues_runSuccessfully() {
172+
Schema schema = Schema.recordOf("record",
173+
Schema.Field.of("null_field", Schema.nullableOf(Schema.of(Schema.Type.STRING)))
174+
);
175+
176+
Map<String, ComplexSingleValueColumn> headeredCells = new HashMap<>();
177+
ComplexSingleValueColumn nullColumn = new ComplexSingleValueColumn();
178+
nullColumn.setData(null);
179+
headeredCells.put("null_field", nullColumn);
180+
181+
RowRecord rowRecord = new RowRecord(SPREADSHEET_NAME, SHEET_TITLE, null, headeredCells, true);
182+
183+
StructuredRecord transformed = SheetTransformer.transform(rowRecord, schema, false, null, false, null, null);
184+
185+
Assert.assertNull(transformed.get("null_field"));
186+
}
187+
188+
@Test
189+
public void transform_WithNestedRecord_runSuccessfully() {
190+
Schema nestedSchema = Schema.recordOf("nested",
191+
Schema.Field.of("sub_field",
192+
Schema.nullableOf(Schema.of(Schema.Type.STRING))));
193+
Schema schema = Schema.recordOf("record",
194+
Schema.Field.of("record_field", Schema.nullableOf(nestedSchema))
195+
);
196+
197+
Map<String, ComplexSingleValueColumn> subColumns = new HashMap<>();
198+
ComplexSingleValueColumn subColumn = new ComplexSingleValueColumn();
199+
subColumn.setData(new CellData().setFormattedValue("nested_value"));
200+
subColumns.put("sub_field", subColumn);
201+
202+
Map<String, ComplexSingleValueColumn> headeredCells = new HashMap<>();
203+
ComplexSingleValueColumn recordColumn = new ComplexSingleValueColumn();
204+
recordColumn.setSubColumns(subColumns);
205+
headeredCells.put("record_field", recordColumn);
206+
207+
RowRecord rowRecord = new RowRecord(SPREADSHEET_NAME, SHEET_TITLE, null, headeredCells, false);
208+
209+
StructuredRecord transformed = SheetTransformer.transform(rowRecord, schema, false, null, false, null, null);
210+
211+
StructuredRecord nestedRecord = transformed.get("record_field");
212+
Assert.assertNotNull(nestedRecord);
213+
Assert.assertEquals("nested_value", nestedRecord.get("sub_field"));
214+
}
215+
}

0 commit comments

Comments
 (0)