Skip to content

Commit 5fc1690

Browse files
MayurGubreleMayurGubrele
authored
refactor: improve error logging and code coverage (#69)
1 parent bf01e16 commit 5fc1690

File tree

3 files changed

+70
-6
lines changed

3 files changed

+70
-6
lines changed

‎src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,9 @@ public void validateBigTableSchema() throws BigTableInvalidSchemaException {
101101

102102
private void checkIfTableExists(String tableId) throws BigTableInvalidSchemaException {
103103
if (!bigtableTableAdminClient.exists(tableId)) {
104-
throw new BigTableInvalidSchemaException(String.format("Table: %s does not exist!", tableId));
105-
}
104+
throw new BigTableInvalidSchemaException(String.format("Table not found on the path: projects/%s/instances/%s/tables/%s",
105+
bigtableTableAdminClient.getProjectId(), bigtableTableAdminClient.getInstanceId(), tableId));
106+
}
106107
}
107108

108109
private void checkIfColumnFamiliesExist(String tableId) throws BigTableInvalidSchemaException {

‎src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,10 +114,12 @@ public void shouldReturnBigTableResponseWithFailedMutationsWhenBulkMutateRowsThr
114114
@Test
115115
public void shouldThrowInvalidSchemaExceptionIfTableDoesNotExist() {
116116
when(bigtableTableAdminClient.exists(sinkConfig.getTableId())).thenReturn(false);
117+
when(bigtableTableAdminClient.getProjectId()).thenReturn(sinkConfig.getGCloudProjectID());
118+
when(bigtableTableAdminClient.getInstanceId()).thenReturn(sinkConfig.getInstanceId());
117119
try {
118120
bigTableClient.validateBigTableSchema();
119121
} catch (BigTableInvalidSchemaException e) {
120-
Assert.assertEquals("Table: " + sinkConfig.getTableId() + " does not exist!", e.getMessage());
122+
Assert.assertEquals("Table not found on the path: projects/test-gcloud-project/instances/test-instance/tables/test-table", e.getMessage());
121123
}
122124
}
123125

‎src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package io.odpf.depot.bigtable.parser;
22

3+
import com.google.protobuf.Timestamp;
34
import io.odpf.depot.TestBookingLogKey;
45
import io.odpf.depot.TestBookingLogMessage;
6+
import io.odpf.depot.TestLocation;
57
import io.odpf.depot.TestServiceType;
68
import io.odpf.depot.bigtable.model.BigTableRecord;
79
import io.odpf.depot.bigtable.model.BigTableSchema;
@@ -60,14 +62,22 @@ public void setUp() throws IOException, InvalidTemplateException {
6062
MockitoAnnotations.openMocks(this);
6163
System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestBookingLogMessage");
6264
System.setProperty("SINK_CONNECTOR_SCHEMA_MESSAGE_MODE", String.valueOf(SinkConnectorSchemaMessageMode.LOG_MESSAGE));
63-
System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{}");
65+
System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{ \"cf1\" : { \"q1\" : \"order_number\", \"q2\" : \"service_type\"} }");
6466
System.setProperty("SINK_BIGTABLE_ROW_KEY_TEMPLATE", "row-key-constant-string");
6567

6668

6769
TestBookingLogKey bookingLogKey1 = TestBookingLogKey.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").build();
68-
TestBookingLogMessage bookingLogMessage1 = TestBookingLogMessage.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").setServiceType(TestServiceType.Enum.GO_SEND).build();
70+
TestBookingLogMessage bookingLogMessage1 = TestBookingLogMessage.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1")
71+
.setEventTimestamp(Timestamp.newBuilder().setSeconds(100L).setNanos(200).build())
72+
.setServiceType(TestServiceType.Enum.GO_SEND)
73+
.setDriverPickupLocation(TestLocation.newBuilder().setLatitude(100D).setLongitude(200D).build())
74+
.build();
6975
TestBookingLogKey bookingLogKey2 = TestBookingLogKey.newBuilder().setOrderNumber("order#2").setOrderUrl("order-url#2").build();
70-
TestBookingLogMessage bookingLogMessage2 = TestBookingLogMessage.newBuilder().setOrderNumber("order#2").setOrderUrl("order-url#2").setServiceType(TestServiceType.Enum.GO_SHOP).build();
76+
TestBookingLogMessage bookingLogMessage2 = TestBookingLogMessage.newBuilder().setOrderNumber("order#2").setOrderUrl("order-url#2")
77+
.setEventTimestamp(Timestamp.newBuilder().setSeconds(101L).setNanos(202).build())
78+
.setServiceType(TestServiceType.Enum.GO_SHOP)
79+
.setDriverPickupLocation(TestLocation.newBuilder().setLatitude(300D).setLongitude(400D).build())
80+
.build();
7181

7282
OdpfMessage message1 = new OdpfMessage(bookingLogKey1.toByteArray(), bookingLogMessage1.toByteArray());
7383
OdpfMessage message2 = new OdpfMessage(bookingLogKey2.toByteArray(), bookingLogMessage2.toByteArray());
@@ -92,6 +102,57 @@ public void shouldReturnValidRecordsForListOfValidOdpfMessages() {
92102
assertNull(records.get(1).getErrorInfo());
93103
}
94104

105+
@Test
106+
public void shouldReturnValidRecordsForListOfValidOdpfMessagesForComplexFieldsInColumnsMapping() throws InvalidTemplateException {
107+
System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{ \"cf1\" : { \"q1\" : \"order_number\", \"q2\" : \"service_type\", \"q3\" : \"driver_pickup_location\"} }");
108+
ProtoOdpfMessageParser protoOdpfMessageParser = new ProtoOdpfMessageParser(stencilClient);
109+
sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties());
110+
Tuple<SinkConnectorSchemaMessageMode, String> modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig);
111+
BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema);
112+
BigTableSchema bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping());
113+
bigTableRecordParser = new BigTableRecordParser(protoOdpfMessageParser, bigTableRowKeyParser, modeAndSchema, schema, bigtableSchema);
114+
115+
List<BigTableRecord> records = bigTableRecordParser.convert(messages);
116+
assertTrue(records.get(0).isValid());
117+
assertTrue(records.get(1).isValid());
118+
assertNull(records.get(0).getErrorInfo());
119+
assertNull(records.get(1).getErrorInfo());
120+
}
121+
122+
@Test
123+
public void shouldReturnValidRecordsForListOfValidOdpfMessagesForNestedTimestampFieldsInColumnsMapping() throws InvalidTemplateException {
124+
System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{ \"cf1\" : { \"q1\" : \"order_number\", \"q2\" : \"service_type\", \"q3\" : \"event_timestamp.nanos\"} }");
125+
ProtoOdpfMessageParser protoOdpfMessageParser = new ProtoOdpfMessageParser(stencilClient);
126+
sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties());
127+
Tuple<SinkConnectorSchemaMessageMode, String> modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig);
128+
BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema);
129+
BigTableSchema bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping());
130+
bigTableRecordParser = new BigTableRecordParser(protoOdpfMessageParser, bigTableRowKeyParser, modeAndSchema, schema, bigtableSchema);
131+
132+
List<BigTableRecord> records = bigTableRecordParser.convert(messages);
133+
assertTrue(records.get(0).isValid());
134+
assertTrue(records.get(1).isValid());
135+
assertNull(records.get(0).getErrorInfo());
136+
assertNull(records.get(1).getErrorInfo());
137+
}
138+
139+
@Test
140+
public void shouldReturnValidRecordsForListOfValidOdpfMessagesForNestedFieldsInColumnsMapping() throws InvalidTemplateException {
141+
System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{ \"cf1\" : { \"q1\" : \"order_number\", \"q2\" : \"service_type\", \"q3\" : \"driver_pickup_location.latitude\"} }");
142+
ProtoOdpfMessageParser protoOdpfMessageParser = new ProtoOdpfMessageParser(stencilClient);
143+
sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties());
144+
Tuple<SinkConnectorSchemaMessageMode, String> modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig);
145+
BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema);
146+
BigTableSchema bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping());
147+
bigTableRecordParser = new BigTableRecordParser(protoOdpfMessageParser, bigTableRowKeyParser, modeAndSchema, schema, bigtableSchema);
148+
149+
List<BigTableRecord> records = bigTableRecordParser.convert(messages);
150+
assertTrue(records.get(0).isValid());
151+
assertTrue(records.get(1).isValid());
152+
assertNull(records.get(0).getErrorInfo());
153+
assertNull(records.get(1).getErrorInfo());
154+
}
155+
95156
@Test
96157
public void shouldReturnInvalidRecordForAnyNullOdpfMessage() {
97158
List<BigTableRecord> records = bigTableRecordParser.convert(Collections.list(new OdpfMessage(null, null)));

0 commit comments

Comments
 (0)