Skip to content

Commit c3fbbf8

Browse files
panos-kakospanos-kakos
authored
Merge pull request #38 from panos-kakos/master
Update from master
2 parents 5f37895 + 3e87a5d commit c3fbbf8

File tree

250 files changed

+4811
-780
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

250 files changed

+4811
-780
lines changed

‎algorithms-modules/algorithms-miscellaneous-5/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
<configuration>
4848
<source>17</source>
4949
<target>17</target>
50+
<release>17</release>
5051
</configuration>
5152
</plugin>
5253
</plugins>

‎apache-libraries-3/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
## Relevant Articles
1+
## Relevant Articles
2+
- [Storing Null Values in Avro Files](https://www.baeldung.com/avro-storing-null-values-files)

apache-libraries/src/main/java/com/baeldung/apache/avro/storingnullvaluesinavrofile/AvroUser.java renamed to apache-libraries-3/src/main/java/com/baeldung/apache/avro/storingnullvaluesinavrofile/AvroUser.java

File renamed without changes.

‎apache-libraries-3/src/test/java/com/baeldung/apache/avro/SerializeAndDeserializeDateUnitTest.java‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
package com.baeldung.apache.avro;
2+
3+
import org.apache.avro.Conversion;
4+
import org.apache.avro.LogicalTypes;
5+
import org.apache.avro.Schema;
6+
import org.apache.avro.data.TimeConversions;
7+
import org.apache.avro.generic.GenericData;
8+
import org.apache.avro.generic.GenericDatumReader;
9+
import org.apache.avro.generic.GenericDatumWriter;
10+
import org.apache.avro.generic.GenericRecord;
11+
import org.apache.avro.io.DatumReader;
12+
import org.apache.avro.io.DatumWriter;
13+
import org.apache.avro.io.Decoder;
14+
import org.apache.avro.io.DecoderFactory;
15+
import org.apache.avro.io.Encoder;
16+
import org.apache.avro.io.EncoderFactory;
17+
import org.apache.commons.lang3.tuple.Pair;
18+
import org.junit.jupiter.api.Test;
19+
20+
import java.io.ByteArrayOutputStream;
21+
import java.io.IOException;
22+
import java.time.Instant;
23+
import java.time.LocalDate;
24+
import java.time.ZoneId;
25+
import java.util.Date;
26+
27+
import static org.junit.jupiter.api.Assertions.assertEquals;
28+
29+
public class SerializeAndDeserializeDateUnitTest {
30+
31+
@Test
32+
void whenSerializingDateWithLogicalType_thenDeserializesCorrectly() throws IOException {
33+
34+
LocalDate expectedDate = LocalDate.now();
35+
Instant expectedTimestamp = Instant.now();
36+
37+
byte[] serialized = serializeDateWithLogicalType(expectedDate, expectedTimestamp);
38+
Pair<LocalDate, Instant> deserialized = deserializeDateWithLogicalType(serialized);
39+
40+
assertEquals(expectedDate, deserialized.getLeft());
41+
42+
// This is perfectly valid when using logical types
43+
assertEquals(expectedTimestamp.toEpochMilli(), deserialized.getRight().toEpochMilli(),
44+
"Timestamps should match exactly at millisecond precision");
45+
}
46+
47+
@Test
48+
void whenSerializingWithConversionApi_thenDeserializesCorrectly() throws IOException {
49+
50+
LocalDate expectedDate = LocalDate.now();
51+
Instant expectedTimestamp = Instant.now();
52+
53+
byte[] serialized = serializeWithConversionApi(expectedDate, expectedTimestamp);
54+
Pair<LocalDate, Instant> deserialized = deserializeWithConversionApi(serialized);
55+
56+
assertEquals(expectedDate, deserialized.getLeft());
57+
assertEquals(expectedTimestamp.toEpochMilli(), deserialized.getRight().toEpochMilli(),
58+
"Timestamps should match at millisecond precision");
59+
}
60+
61+
@Test
62+
void whenSerializingLegacyDate_thenConvertsCorrectly() throws IOException {
63+
64+
Date legacyDate = new Date();
65+
LocalDate expectedLocalDate = legacyDate.toInstant()
66+
.atZone(ZoneId.systemDefault())
67+
.toLocalDate();
68+
69+
byte[] serialized = serializeLegacyDateAsModern(legacyDate);
70+
LocalDate deserialized = deserializeDateWithLogicalType(serialized).getKey();
71+
72+
assertEquals(expectedLocalDate, deserialized);
73+
}
74+
75+
public static Schema createDateSchema() {
76+
String schemaJson =
77+
"{"
78+
+ "\"type\": \"record\","
79+
+ "\"name\": \"DateRecord\","
80+
+ "\"fields\": ["
81+
+ " {\"name\": \"date\", \"type\": {\"type\": \"int\", \"logicalType\": \"date\"}},"
82+
+ " {\"name\": \"timestamp\", \"type\": {\"type\": \"long\", \"logicalType\": \"timestamp-millis\"}}"
83+
+ "]"
84+
+ "}";
85+
return new Schema.Parser().parse(schemaJson);
86+
}
87+
88+
public static byte[] serializeDateWithLogicalType(LocalDate date, Instant timestamp) throws IOException {
89+
Schema schema = createDateSchema();
90+
GenericRecord record = new GenericData.Record(schema);
91+
92+
// Convert LocalDate to days since epoch
93+
record.put("date", (int) date.toEpochDay());
94+
95+
// Convert Instant to milliseconds since epoch
96+
record.put("timestamp", timestamp.toEpochMilli());
97+
98+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
99+
DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
100+
Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null);
101+
102+
datumWriter.write(record, encoder);
103+
encoder.flush();
104+
105+
return baos.toByteArray();
106+
}
107+
108+
public static Pair<LocalDate, Instant> deserializeDateWithLogicalType(byte[] bytes) throws IOException {
109+
Schema schema = createDateSchema();
110+
DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(schema);
111+
Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
112+
113+
GenericRecord record = datumReader.read(null, decoder);
114+
115+
// Convert days since epoch back to LocalDate
116+
LocalDate date = LocalDate.ofEpochDay((int) record.get("date"));
117+
118+
// Convert milliseconds since epoch back to Instant
119+
Instant timestamp = Instant.ofEpochMilli((long) record.get("timestamp"));
120+
121+
return Pair.of(date, timestamp);
122+
}
123+
124+
public static byte[] serializeWithConversionApi(LocalDate date, Instant timestamp) throws IOException {
125+
Schema schema = createDateSchema();
126+
GenericRecord record = new GenericData.Record(schema);
127+
128+
// Use LogicalTypes.date() for conversion
129+
Conversion<LocalDate> dateConversion = new org.apache.avro.data.TimeConversions.DateConversion();
130+
LogicalTypes.date().addToSchema(schema.getField("date").schema());
131+
132+
// Use LogicalTypes.timestampMillis() for conversion
133+
Conversion<Instant> timestampConversion = new org.apache.avro.data.TimeConversions.TimestampMillisConversion();
134+
LogicalTypes.timestampMillis().addToSchema(schema.getField("timestamp").schema());
135+
136+
record.put("date", dateConversion.toInt(date, schema.getField("date").schema(), LogicalTypes.date()));
137+
record.put("timestamp", timestampConversion.toLong(timestamp, schema.getField("timestamp").schema(), LogicalTypes.timestampMillis()));
138+
139+
// Serialize as before
140+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
141+
DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
142+
Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null);
143+
144+
datumWriter.write(record, encoder);
145+
encoder.flush();
146+
147+
return baos.toByteArray();
148+
}
149+
150+
public static Pair<LocalDate, Instant> deserializeWithConversionApi(byte[] bytes) throws IOException {
151+
Schema schema = createDateSchema();
152+
DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(schema);
153+
Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
154+
155+
GenericRecord record = datumReader.read(null, decoder);
156+
157+
// Use LogicalTypes.date() for conversion
158+
Conversion<LocalDate> dateConversion = new TimeConversions.DateConversion();
159+
LogicalTypes.date().addToSchema(schema.getField("date").schema());
160+
161+
// Use LogicalTypes.timestampMillis() for conversion
162+
Conversion<Instant> timestampConversion = new TimeConversions.TimestampMillisConversion();
163+
LogicalTypes.timestampMillis().addToSchema(schema.getField("timestamp").schema());
164+
165+
// Get the primitive values from the record
166+
int daysSinceEpoch = (int) record.get("date");
167+
long millisSinceEpoch = (long) record.get("timestamp");
168+
169+
// Convert back to Java types using the conversion API
170+
LocalDate date = dateConversion.fromInt(
171+
daysSinceEpoch,
172+
schema.getField("date").schema(),
173+
LogicalTypes.date()
174+
);
175+
176+
Instant timestamp = timestampConversion.fromLong(
177+
millisSinceEpoch,
178+
schema.getField("timestamp").schema(),
179+
LogicalTypes.timestampMillis()
180+
);
181+
182+
return Pair.of(date, timestamp);
183+
}
184+
185+
public static byte[] serializeLegacyDateAsModern(Date legacyDate) throws IOException {
186+
// Convert java.util.Date to java.time.Instant
187+
Instant instant = legacyDate.toInstant();
188+
189+
// Convert to LocalDate if you need date-only information
190+
LocalDate localDate = instant.atZone(ZoneId.systemDefault()).toLocalDate();
191+
192+
// Then use one of our modern date serialization methods
193+
return serializeDateWithLogicalType(localDate, instant);
194+
}
195+
}

‎apache-libraries-3/src/test/java/com/baeldung/apache/avro/SerializeEnumValueInAvroUnitTest.java‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
package com.baeldung.apache.avro;
2+
3+
import org.apache.avro.Schema;
4+
import org.apache.avro.SchemaBuilder;
5+
import org.apache.avro.file.DataFileReader;
6+
import org.apache.avro.file.DataFileWriter;
7+
import org.apache.avro.generic.GenericData;
8+
import org.apache.avro.generic.GenericDatumReader;
9+
import org.apache.avro.generic.GenericDatumWriter;
10+
import org.apache.avro.generic.GenericRecord;
11+
import org.apache.avro.io.DatumReader;
12+
import org.apache.avro.io.DatumWriter;
13+
import org.junit.jupiter.api.BeforeEach;
14+
import org.junit.jupiter.api.Test;
15+
import org.junit.jupiter.api.io.TempDir;
16+
17+
import java.io.File;
18+
import java.io.IOException;
19+
import java.nio.file.Path;
20+
21+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
24+
public class SerializeEnumValueInAvroUnitTest {
25+
26+
@TempDir
27+
Path tempDir;
28+
29+
private Schema colorEnum;
30+
private Schema recordSchema;
31+
private Schema unionSchema;
32+
private Schema recordWithUnionSchema;
33+
34+
@BeforeEach
35+
void setUp() {
36+
// Create enum schema
37+
colorEnum = SchemaBuilder.enumeration("Color")
38+
.namespace("com.baeldung.apache.avro")
39+
.symbols("UNKNOWN", "GREEN", "RED", "BLUE");
40+
41+
// Create record schema with enum field
42+
recordSchema = SchemaBuilder.record("ColorRecord")
43+
.namespace("com.baeldung.apache.avro")
44+
.fields()
45+
.name("color")
46+
.type(colorEnum)
47+
.noDefault()
48+
.endRecord();
49+
50+
// Create union schema with enum and null
51+
unionSchema = SchemaBuilder.unionOf()
52+
.type(colorEnum)
53+
.and()
54+
.nullType()
55+
.endUnion();
56+
57+
// Create record schema with union field
58+
recordWithUnionSchema = SchemaBuilder.record("ColorRecordWithUnion")
59+
.namespace("com.baeldung.apache.avro")
60+
.fields()
61+
.name("color")
62+
.type(unionSchema)
63+
.noDefault()
64+
.endRecord();
65+
}
66+
67+
@Test
68+
void whenSerializingEnum_thenSuccess() throws IOException {
69+
File file = tempDir.resolve("color.avro").toFile();
70+
71+
// Create record with enum value
72+
GenericRecord record = new GenericData.Record(recordSchema);
73+
GenericData.EnumSymbol colorSymbol = new GenericData.EnumSymbol(colorEnum, "RED");
74+
record.put("color", colorSymbol);
75+
76+
// Write to file
77+
DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(recordSchema);
78+
try (DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter)) {
79+
dataFileWriter.create(recordSchema, file);
80+
dataFileWriter.append(record);
81+
}
82+
83+
// Read from file
84+
DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(recordSchema);
85+
try (DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file, datumReader)) {
86+
GenericRecord result = dataFileReader.next();
87+
assertEquals("RED", result.get("color").toString());
88+
}
89+
}
90+
91+
@Test
92+
void whenSerializingEnumInUnion_thenSuccess() throws IOException {
93+
File file = tempDir.resolve("colorUnion.avro").toFile();
94+
95+
// Create record with enum in union
96+
GenericRecord record = new GenericData.Record(recordWithUnionSchema);
97+
GenericData.EnumSymbol colorSymbol = new GenericData.EnumSymbol(colorEnum, "GREEN");
98+
record.put("color", colorSymbol);
99+
100+
// Write to file
101+
DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(recordWithUnionSchema);
102+
try (DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter)) {
103+
dataFileWriter.create(recordWithUnionSchema, file);
104+
dataFileWriter.append(record);
105+
}
106+
107+
// Read from file
108+
DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(recordWithUnionSchema);
109+
try (DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file, datumReader)) {
110+
GenericRecord result = dataFileReader.next();
111+
assertEquals("GREEN", result.get("color").toString());
112+
}
113+
}
114+
115+
@Test
116+
void whenSerializingNullInUnion_thenSuccess() throws IOException {
117+
File file = tempDir.resolve("colorNull.avro").toFile();
118+
119+
// Create record with null in union
120+
GenericRecord record = new GenericData.Record(recordWithUnionSchema);
121+
record.put("color", null);
122+
123+
// Write to file
124+
DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(recordWithUnionSchema);
125+
assertDoesNotThrow(() -> {
126+
try (DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter)) {
127+
dataFileWriter.create(recordWithUnionSchema, file);
128+
dataFileWriter.append(record);
129+
}
130+
});
131+
}
132+
133+
@Test
134+
void whenSchemaEvolution_thenDefaultValueUsed() throws IOException {
135+
// Create schema with new enum value and default at schema level
136+
String evolvedSchemaJson = "{\"type\":\"record\"," +
137+
"\"name\":\"ColorRecord\"," +
138+
"\"namespace\":\"com.baeldung.apache.avro\"," +
139+
"\"fields\":[{\"name\":\"color\"," +
140+
"\"type\":{\"type\":\"enum\"," +
141+
"\"name\":\"Color\"," +
142+
"\"symbols\":[\"UNKNOWN\",\"GREEN\",\"RED\",\"BLUE\",\"YELLOW\"]," +
143+
"\"default\":\"UNKNOWN\"}}]}";
144+
145+
Schema evolvedRecordSchema = new Schema.Parser().parse(evolvedSchemaJson);
146+
Schema evolvedEnum = evolvedRecordSchema.getField("color").schema();
147+
148+
File file = tempDir.resolve("colorEvolved.avro").toFile();
149+
150+
// Create record with new enum value
151+
GenericRecord record = new GenericData.Record(evolvedRecordSchema);
152+
GenericData.EnumSymbol colorSymbol = new GenericData.EnumSymbol(evolvedEnum, "YELLOW");
153+
record.put("color", colorSymbol);
154+
155+
// Write with evolved schema
156+
DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(evolvedRecordSchema);
157+
try (DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter)) {
158+
dataFileWriter.create(evolvedRecordSchema, file);
159+
dataFileWriter.append(record);
160+
}
161+
162+
// Create old schema without YELLOW but WITH default
163+
String originalSchemaJson = "{\"type\":\"record\"," +
164+
"\"name\":\"ColorRecord\"," +
165+
"\"namespace\":\"com.baeldung.apache.avro\"," +
166+
"\"fields\":[{\"name\":\"color\",\"type\":{\"type\":\"enum\",\"name\":\"Color\"," +
167+
"\"symbols\":[\"UNKNOWN\",\"GREEN\",\"RED\",\"BLUE\"]," +
168+
"\"default\":\"UNKNOWN\"}}]}";
169+
170+
Schema originalRecordSchema = new Schema.Parser().parse(originalSchemaJson);
171+
172+
// Read with original schema
173+
DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(evolvedRecordSchema, originalRecordSchema);
174+
try (DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file, datumReader)) {
175+
GenericRecord result = dataFileReader.next();
176+
assertEquals("UNKNOWN", result.get("color").toString());
177+
}
178+
}
179+
}

apache-libraries/src/test/java/com/baeldung/apache/avro/storingnullvaluesinavrofile/AvroUserUnitTest.java renamed to apache-libraries-3/src/test/java/com/baeldung/apache/avro/storingnullvaluesinavrofile/AvroUserUnitTest.java

File renamed without changes.

0 commit comments

Comments
 (0)