Skip to content

Commit ad6822c

Browse files
gowagowa
committed
run unit tests in different codegen modes
1 parent bd3724e commit ad6822c

File tree

5 files changed

+192
-43
lines changed

5 files changed

+192
-43
lines changed

‎parquet-protobuf/src/main/java/org/apache/parquet/proto/ByteBuddyCodeGen.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ static boolean isByteBuddyAvailable(boolean failIfNot) {
174174
return true;
175175
} catch (ClassNotFoundException e) {
176176
if (failIfNot) {
177-
throw new IllegalStateException("ByteBuddy is not available", e);
177+
throw new UnsupportedOperationException("ByteBuddy is not available", e);
178178
}
179179
return false;
180180
}
@@ -1197,6 +1197,10 @@ private Field resolveField(
11971197
fieldDescriptor,
11981198
fieldWriter);
11991199
} else {
1200+
if (!Objects.equals(parquetFieldInfo.fieldName, fieldDescriptor.getName())) {
1201+
throw new CodeGenException("fields mismatch: parquetFieldInfo: " + parquetFieldInfo.fieldName
1202+
+ ", fieldDescriptor: " + fieldDescriptor);
1203+
}
12001204
return new Field(
12011205
fieldScanner,
12021206
this,

‎parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoWriteSupport.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public class ProtoWriteSupport<T extends MessageOrBuilder> extends WriteSupport<
8787

8888
private boolean writeSpecsCompliant = false;
8989
private boolean unwrapProtoWrappers = false;
90-
private CodegenMode codegenMode = CodegenMode.SUPPORT_COMPATIBLE;
90+
private CodegenMode codegenMode = CodegenMode.DEFAULT;
9191
private RecordConsumer recordConsumer;
9292
private Class<? extends Message> protoMessage;
9393
private Descriptor descriptor;
@@ -209,12 +209,18 @@ public boolean protobufReflectionForExtensions() {
209209
@Override
210210
public boolean tryCodeGen(Class<? extends Message> protoClass) {
211211
if (!ByteBuddyCodeGen.isGeneratedMessage(protoClass)) {
212-
throw new IllegalStateException("protoClass is a GeneratedMessage: " + protoClass);
212+
throw new UnsupportedOperationException("protoClass is not a GeneratedMessage: " + protoClass);
213213
}
214214
return ByteBuddyCodeGen.isByteBuddyAvailable(true);
215215
}
216216
};
217217

218+
public static final CodegenMode DEFAULT = CodegenMode.SUPPORT_COMPATIBLE;
219+
220+
public static CodegenMode orDefault(CodegenMode codegenMode) {
221+
return codegenMode == null ? DEFAULT : codegenMode;
222+
}
223+
218224
public abstract boolean ignoreCodeGenException();
219225

220226
public abstract boolean protobufReflectionForExtensions();

‎parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoInputOutputFormatTest.java‎

Lines changed: 48 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
import com.google.protobuf.Message;
2929
import com.google.protobuf.Timestamp;
3030
import com.google.protobuf.util.Timestamps;
31+
import java.util.ArrayList;
32+
import java.util.Arrays;
33+
import java.util.Collection;
3134
import java.util.List;
3235
import org.apache.hadoop.conf.Configuration;
3336
import org.apache.hadoop.fs.Path;
@@ -38,9 +41,33 @@
3841
import org.apache.parquet.proto.utils.ReadUsingMR;
3942
import org.apache.parquet.proto.utils.WriteUsingMR;
4043
import org.junit.Test;
44+
import org.junit.runner.RunWith;
45+
import org.junit.runners.Parameterized;
4146

47+
@RunWith(Parameterized.class)
4248
public class ProtoInputOutputFormatTest {
4349

50+
@Parameterized.Parameters(name = "codegenMode: {0}")
51+
public static Collection<Object[]> data() {
52+
List<Object[]> data = new ArrayList<>();
53+
54+
List<ProtoWriteSupport.CodegenMode> codegenModes =
55+
new ArrayList<>(Arrays.asList(ProtoWriteSupport.CodegenMode.values()));
56+
codegenModes.add(null);
57+
58+
for (ProtoWriteSupport.CodegenMode codegenMode : codegenModes) {
59+
data.add(new Object[] {codegenMode});
60+
}
61+
62+
return data;
63+
}
64+
65+
private final ProtoWriteSupport.CodegenMode codegenMode;
66+
67+
public ProtoInputOutputFormatTest(ProtoWriteSupport.CodegenMode codegenMode) {
68+
this.codegenMode = codegenMode;
69+
}
70+
4471
/**
4572
* Writes Protocol Buffer using first MR job, reads written file using
4673
* second job and compares input and output.
@@ -241,7 +268,7 @@ public void testRepeatedIntMessageClassSchemaCompliant() throws Exception {
241268
.addRepeatedInt(2)
242269
.build();
243270

244-
Configuration conf = new Configuration();
271+
Configuration conf = updateConfiguration(new Configuration());
245272
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
246273

247274
Path outputPath = new WriteUsingMR(conf).write(msgEmpty, msgNonEmpty);
@@ -264,7 +291,7 @@ public void testProto3RepeatedIntMessageClassSchemaCompliant() throws Exception
264291
.addRepeatedInt(2)
265292
.build();
266293

267-
Configuration conf = new Configuration();
294+
Configuration conf = updateConfiguration(new Configuration());
268295
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
269296

270297
Path outputPath = new WriteUsingMR(conf).write(msgEmpty, msgNonEmpty);
@@ -327,7 +354,7 @@ public void testMapIntMessageClassSchemaCompliant() throws Exception {
327354
.putMapInt(2, 234)
328355
.build();
329356

330-
Configuration conf = new Configuration();
357+
Configuration conf = updateConfiguration(new Configuration());
331358
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
332359

333360
Path outputPath = new WriteUsingMR(conf).write(msgEmpty, msgNonEmpty);
@@ -350,7 +377,7 @@ public void testProto3MapIntMessageClassSchemaCompliant() throws Exception {
350377
.putMapInt(2, 234)
351378
.build();
352379

353-
Configuration conf = new Configuration();
380+
Configuration conf = updateConfiguration(new Configuration());
354381
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
355382

356383
Path outputPath = new WriteUsingMR(conf).write(msgEmpty, msgNonEmpty);
@@ -419,7 +446,7 @@ public void testRepeatedInnerMessageClassSchemaCompliant() throws Exception {
419446
TestProtobuf.InnerMessage.newBuilder().setTwo("two").build())
420447
.build();
421448

422-
Configuration conf = new Configuration();
449+
Configuration conf = updateConfiguration(new Configuration());
423450
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
424451

425452
Path outputPath = new WriteUsingMR(conf).write(msgEmpty, msgNonEmpty);
@@ -444,7 +471,7 @@ public void testProto3RepeatedInnerMessageClassSchemaCompliant() throws Exceptio
444471
TestProto3.InnerMessage.newBuilder().setTwo("two").build())
445472
.build();
446473

447-
Configuration conf = new Configuration();
474+
Configuration conf = updateConfiguration(new Configuration());
448475
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
449476

450477
Path outputPath = new WriteUsingMR(conf).write(msgEmpty, msgNonEmpty);
@@ -463,7 +490,7 @@ public void testProto3Defaults() throws Exception {
463490
TestProto3.SchemaConverterAllDatatypes msgEmpty =
464491
TestProto3.SchemaConverterAllDatatypes.newBuilder().build();
465492

466-
Configuration conf = new Configuration();
493+
Configuration conf = updateConfiguration(new Configuration());
467494
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
468495

469496
Path outputPath = new WriteUsingMR(conf).write(msgEmpty);
@@ -508,7 +535,7 @@ public void testProto3AllTypes() throws Exception {
508535

509536
TestProto3.SchemaConverterAllDatatypes dataBuilt = data.build();
510537

511-
Configuration conf = new Configuration();
538+
Configuration conf = updateConfiguration(new Configuration());
512539
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
513540

514541
Path outputPath = new WriteUsingMR(conf).write(dataBuilt);
@@ -574,7 +601,7 @@ public void testProto3AllTypesMultiple() throws Exception {
574601
input[i] = d.build();
575602
}
576603

577-
Configuration conf = new Configuration();
604+
Configuration conf = updateConfiguration(new Configuration());
578605
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
579606

580607
Path outputPath = new WriteUsingMR(conf).write(input);
@@ -606,7 +633,7 @@ public void testProto3RepeatedMessages() throws Exception {
606633
top.addInnerBuilder().setTwo("Second inner");
607634
top.addInnerBuilder().setThree("Third inner");
608635

609-
Configuration conf = new Configuration();
636+
Configuration conf = updateConfiguration(new Configuration());
610637
ProtoWriteSupport.setWriteSpecsCompliant(conf, true);
611638

612639
Path outputPath = new WriteUsingMR(conf).write(top.build());
@@ -643,7 +670,7 @@ public void testProto3TimestampMessageClass() throws Exception {
643670
TestProto3.DateTimeMessage msgNonEmpty =
644671
TestProto3.DateTimeMessage.newBuilder().setTimestamp(timestamp).build();
645672

646-
Configuration conf = new Configuration();
673+
Configuration conf = updateConfiguration(new Configuration());
647674
conf.setBoolean(ProtoWriteSupport.PB_UNWRAP_PROTO_WRAPPERS, true);
648675
Path outputPath = new WriteUsingMR(conf).write(msgEmpty, msgNonEmpty);
649676
ReadUsingMR readUsingMR = new ReadUsingMR();
@@ -665,7 +692,7 @@ public void testProto3WrappedMessageClass() throws Exception {
665692
.setWrappedBool(BoolValue.of(true))
666693
.build();
667694

668-
Configuration conf = new Configuration();
695+
Configuration conf = updateConfiguration(new Configuration());
669696
conf.setBoolean(ProtoWriteSupport.PB_UNWRAP_PROTO_WRAPPERS, true);
670697
Path outputPath = new WriteUsingMR(conf).write(msgEmpty, msgNonEmpty);
671698
ReadUsingMR readUsingMR = new ReadUsingMR();
@@ -681,9 +708,16 @@ public void testProto3WrappedMessageClass() throws Exception {
681708
/**
682709
* Runs job that writes input to file and then job reading data back.
683710
*/
684-
public static List<Message> runMRJobs(Message... messages) throws Exception {
685-
Path outputPath = new WriteUsingMR().write(messages);
711+
public List<Message> runMRJobs(Message... messages) throws Exception {
712+
Path outputPath = new WriteUsingMR(updateConfiguration(new Configuration())).write(messages);
686713
List<Message> result = new ReadUsingMR().read(outputPath);
687714
return result;
688715
}
716+
717+
private Configuration updateConfiguration(Configuration configuration) {
718+
if (codegenMode != null) {
719+
ProtoWriteSupport.setCodegenMode(configuration, codegenMode);
720+
}
721+
return configuration;
722+
}
689723
}

‎parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoParquetWriterTest.java‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,49 @@
2121
import static org.apache.parquet.proto.TestUtils.readMessages;
2222
import static org.apache.parquet.proto.TestUtils.someTemporaryFilePath;
2323
import static org.junit.Assert.assertEquals;
24+
import static org.junit.Assert.assertFalse;
25+
import static org.junit.Assert.assertTrue;
2426

2527
import com.google.protobuf.Descriptors;
2628
import com.google.protobuf.DynamicMessage;
29+
import java.util.ArrayList;
30+
import java.util.Arrays;
31+
import java.util.Collection;
32+
import java.util.EnumSet;
2733
import java.util.List;
2834
import org.apache.hadoop.conf.Configuration;
2935
import org.apache.hadoop.fs.Path;
3036
import org.apache.parquet.hadoop.ParquetFileWriter;
3137
import org.apache.parquet.hadoop.ParquetWriter;
3238
import org.apache.parquet.proto.test.TestProto3;
3339
import org.junit.Test;
40+
import org.junit.runner.RunWith;
41+
import org.junit.runners.Parameterized;
3442

43+
@RunWith(Parameterized.class)
3544
public class ProtoParquetWriterTest {
45+
46+
@Parameterized.Parameters(name = "codegenMode: {0}")
47+
public static Collection<Object[]> data() {
48+
List<Object[]> data = new ArrayList<>();
49+
50+
List<ProtoWriteSupport.CodegenMode> codegenModes =
51+
new ArrayList<>(Arrays.asList(ProtoWriteSupport.CodegenMode.values()));
52+
codegenModes.add(null);
53+
54+
for (ProtoWriteSupport.CodegenMode codegenMode : codegenModes) {
55+
data.add(new Object[] {codegenMode});
56+
}
57+
58+
return data;
59+
}
60+
61+
private final ProtoWriteSupport.CodegenMode codegenMode;
62+
63+
public ProtoParquetWriterTest(ProtoWriteSupport.CodegenMode codegenMode) {
64+
this.codegenMode = codegenMode;
65+
}
66+
3667
@Test
3768
public void testProtoParquetWriterWithDynamicMessage() throws Exception {
3869
Path file = someTemporaryFilePath();
@@ -41,12 +72,23 @@ public void testProtoParquetWriterWithDynamicMessage() throws Exception {
4172
msg.setOne("oneValue");
4273
DynamicMessage dynamicMessage = DynamicMessage.newBuilder(msg.build()).build();
4374

44-
Configuration conf = new Configuration();
45-
ParquetWriter<DynamicMessage> writer = ProtoParquetWriter.<DynamicMessage>builder(file)
46-
.withDescriptor(descriptor)
47-
.withConf(conf)
48-
.withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
49-
.build();
75+
Configuration conf = updateConfiguration(new Configuration());
76+
77+
ProtoWriteSupport.CodegenMode codegenModeOrDefault = ProtoWriteSupport.CodegenMode.orDefault(codegenMode);
78+
EnumSet<ProtoWriteSupport.CodegenMode> failingModes = EnumSet.of(ProtoWriteSupport.CodegenMode.REQUIRED_ALL);
79+
80+
ParquetWriter<DynamicMessage> writer;
81+
try {
82+
writer = ProtoParquetWriter.<DynamicMessage>builder(file)
83+
.withDescriptor(descriptor)
84+
.withConf(conf)
85+
.withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
86+
.build();
87+
} catch (UnsupportedOperationException e) {
88+
assertTrue("codegenMode: " + codegenMode, failingModes.contains(codegenModeOrDefault));
89+
return;
90+
}
91+
assertFalse("codegenMode: " + codegenMode, failingModes.contains(codegenModeOrDefault));
5092
writer.write(dynamicMessage);
5193
writer.close();
5294

@@ -58,4 +100,11 @@ public void testProtoParquetWriterWithDynamicMessage() throws Exception {
58100
assertEquals(getFirst.getTwo(), "");
59101
assertEquals(getFirst.getThree(), "");
60102
}
103+
104+
private Configuration updateConfiguration(Configuration configuration) {
105+
if (codegenMode != null) {
106+
ProtoWriteSupport.setCodegenMode(configuration, codegenMode);
107+
}
108+
return configuration;
109+
}
61110
}

0 commit comments

Comments
 (0)