Skip to content

Commit 670fb4c

Browse files
davidmartinezbaruadavidmartinezbarua
authored
Merge pull request #16732 from thiagohora/BAEL-7871/serialization_using_apache_fury
BAEL-7871: Serialization using apache fury
2 parents eb26abe + 50fad55 commit 670fb4c

File tree

9 files changed

+458
-0
lines changed

9 files changed

+458
-0
lines changed

‎libraries-data-io-2/pom.xml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@
1717
<maven.compiler.target>19</maven.compiler.target>
1818
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
1919
<google-flatbuffers.version>23.5.26</google-flatbuffers.version>
20+
21+
<apache-fury.version>0.5.0</apache-fury.version>
22+
<avro.version>1.11.3</avro.version>
23+
<protobuff.version>4.27.0</protobuff.version>
24+
<protobuf.plugin.version>2.1.1</protobuf.plugin.version>
25+
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
2026
</properties>
2127

2228
<dependencies>
@@ -25,6 +31,63 @@
2531
<artifactId>flatbuffers-java</artifactId>
2632
<version>${google-flatbuffers.version}</version>
2733
</dependency>
34+
35+
<dependency>
36+
<groupId>org.apache.fury</groupId>
37+
<artifactId>fury-core</artifactId>
38+
<version>${apache-fury.version}</version>
39+
</dependency>
40+
<dependency>
41+
<groupId>org.apache.avro</groupId>
42+
<artifactId>avro</artifactId>
43+
<version>${avro.version}</version>
44+
</dependency>
45+
<dependency>
46+
<groupId>com.google.protobuf</groupId>
47+
<artifactId>protobuf-java</artifactId>
48+
<version>${protobuff.version}</version>
49+
</dependency>
2850
</dependencies>
2951

52+
53+
<build>
54+
<plugins>
55+
<plugin>
56+
<groupId>io.github.ascopes</groupId>
57+
<artifactId>protobuf-maven-plugin</artifactId>
58+
<version>${protobuf.plugin.version}</version>
59+
<configuration>
60+
<protocVersion>${protobuff.version}</protocVersion>
61+
<sourceDirectories>
62+
<sourceDirectory>src/main/resources</sourceDirectory>
63+
64+
</sourceDirectories>
65+
</configuration>
66+
<executions>
67+
<execution>
68+
<goals>
69+
<goal>generate</goal>
70+
</goals>
71+
</execution>
72+
</executions>
73+
</plugin>
74+
<!-- Avro Plugin -->
75+
<plugin>
76+
<groupId>org.apache.avro</groupId>
77+
<artifactId>avro-maven-plugin</artifactId>
78+
<version>${avro.version}</version>
79+
<executions>
80+
<execution>
81+
<phase>generate-sources</phase>
82+
<goals>
83+
<goal>schema</goal>
84+
</goals>
85+
<configuration>
86+
<sourceDirectory>${project.basedir}/src/main/resources</sourceDirectory>
87+
</configuration>
88+
</execution>
89+
</executions>
90+
</plugin>
91+
</plugins>
92+
</build>
3093
</project>

‎libraries-data-io-2/src/main/java/com/baeldung/apachefury/event/Address.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package com.baeldung.apachefury.event;
2+
3+
import java.io.Serializable;
4+
5+
public class Address implements Serializable {
6+
7+
private final String street;
8+
private final String city;
9+
private final String zipCode;
10+
11+
public Address(String street, String city, String zipCode) {
12+
this.street = street;
13+
this.city = city;
14+
this.zipCode = zipCode;
15+
}
16+
17+
public String getStreet() {
18+
return street;
19+
}
20+
21+
public String getCity() {
22+
return city;
23+
}
24+
25+
public String getZipCode() {
26+
return zipCode;
27+
}
28+
29+
@Override
30+
public boolean equals(Object o) {
31+
if (this == o) return true;
32+
if (!(o instanceof Address)) return false;
33+
Address address = (Address) o;
34+
return street.equals(address.street) && city.equals(address.city) && zipCode.equals(address.zipCode);
35+
}
36+
37+
@Override
38+
public int hashCode() {
39+
return street.hashCode() + city.hashCode() + zipCode.hashCode();
40+
}
41+
42+
}

‎libraries-data-io-2/src/main/java/com/baeldung/apachefury/event/UserEvent.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package com.baeldung.apachefury.event;
2+
3+
import java.io.Serializable;
4+
import java.util.Objects;
5+
6+
public class UserEvent implements Serializable {
7+
private final String userId;
8+
private final String eventType;
9+
private final long timestamp;
10+
private final Address address;
11+
12+
public UserEvent(String userId, String eventType, long timestamp, Address address) {
13+
this.userId = userId;
14+
this.eventType = eventType;
15+
this.timestamp = timestamp;
16+
this.address = address;
17+
}
18+
19+
// Getters and setters
20+
public String getUserId() {
21+
return userId;
22+
}
23+
24+
public String getEventType() {
25+
return eventType;
26+
}
27+
28+
public long getTimestamp() {
29+
return timestamp;
30+
}
31+
32+
public Address getAddress() {
33+
return address;
34+
}
35+
36+
@Override
37+
public boolean equals(Object o) {
38+
if (this == o) return true;
39+
if (!(o instanceof UserEvent)) return false;
40+
UserEvent userEvent = (UserEvent) o;
41+
return timestamp == userEvent.timestamp && userId.equals(userEvent.userId) && eventType.equals(userEvent.eventType) && address.equals(userEvent.address);
42+
}
43+
44+
@Override
45+
public int hashCode() {
46+
return Objects.hash(userId, eventType, timestamp, address);
47+
}
48+
}

‎libraries-data-io-2/src/main/resources/UserEvent.avsc‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"namespace": "com.baeldung.apachefury.event.avro",
3+
"type": "record",
4+
"name": "UserEvent",
5+
"fields": [
6+
{"name": "userId", "type": "string"},
7+
{"name": "eventType", "type": "string"},
8+
{"name": "timestamp", "type": "long"},
9+
{"name": "address", "type": {
10+
"type": "record",
11+
"name": "Address",
12+
"fields": [
13+
{"name": "street", "type": "string"},
14+
{"name": "city", "type": "string"},
15+
{"name": "zipCode", "type": "string"}
16+
]
17+
}}
18+
]
19+
}

‎libraries-data-io-2/src/main/resources/user.proto‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
syntax = "proto3";
2+
3+
option java_package = "com.baeldung.apachefury.event";
4+
option java_outer_classname = "UserEventProto";
5+
6+
message UserEvent {
7+
string userId = 1;
8+
string eventType = 2;
9+
int64 timestamp = 3;
10+
Address address = 4;
11+
}
12+
13+
message Address {
14+
string street = 1;
15+
string city = 2;
16+
string zipCode = 3;
17+
}

‎libraries-data-io-2/src/test/java/com/baeldung/apachefury/serialization/AvroSerializationUnitTest.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package com.baeldung.apachefury.serialization;
2+
3+
import com.baeldung.apachefury.event.avro.Address;
4+
import com.baeldung.apachefury.event.avro.UserEvent;
5+
import org.apache.avro.io.DatumReader;
6+
import org.apache.avro.io.DatumWriter;
7+
import org.apache.avro.io.Decoder;
8+
import org.apache.avro.io.DecoderFactory;
9+
import org.apache.avro.io.Encoder;
10+
import org.apache.avro.io.EncoderFactory;
11+
import org.apache.avro.specific.SpecificDatumReader;
12+
import org.apache.avro.specific.SpecificDatumWriter;
13+
import org.junit.jupiter.api.Assertions;
14+
import org.junit.jupiter.api.Test;
15+
16+
import java.io.ByteArrayOutputStream;
17+
import java.io.IOException;
18+
import java.util.ArrayList;
19+
import java.util.List;
20+
import java.util.UUID;
21+
import java.util.logging.Logger;
22+
23+
class AvroSerializationUnitTest {
24+
25+
static final Logger LOG = Logger.getLogger(AvroSerializationUnitTest.class.getName());
26+
27+
@Test
28+
void whenUsingAvroSerialization_thenGenerateByteOutput() throws IOException {
29+
30+
List<UserEvent> events = new ArrayList<>(100000);
31+
List<UserEvent> parserEvents = new ArrayList<>(100000);
32+
33+
for (int i = 0; i < 100000; i++) {
34+
events.add(UserEvent.newBuilder()
35+
.setUserId(UUID.randomUUID()+"-"+i)
36+
.setEventType("login")
37+
.setTimestamp(System.currentTimeMillis())
38+
.setAddress(Address.newBuilder()
39+
.setStreet(i + " Main St")
40+
.setCity("Spring field " + i)
41+
.setZipCode(UUID.randomUUID().toString())
42+
.build())
43+
.build());
44+
}
45+
46+
long startTime = System.currentTimeMillis();
47+
for (int i = 0; i < 100000; i++) {
48+
byte[] serializedData = serializeAvro(events.get(i));
49+
UserEvent temp = deSerializeAvro(serializedData);
50+
parserEvents.add(temp);
51+
}
52+
long endTime = System.currentTimeMillis();
53+
54+
long totalBytes = 0;
55+
for (int i = 0; i < 100000; i++) {
56+
byte[] serializedData = serializeAvro(events.get(i));
57+
totalBytes += serializedData.length;
58+
}
59+
60+
LOG.info("Avro serialization time: " + (endTime - startTime) + " ms");
61+
LOG.info("Total bytes: " + totalBytes / (1024 * 1024) + " MB");
62+
63+
Assertions.assertEquals(events, parserEvents);
64+
}
65+
66+
public byte[] serializeAvro(UserEvent request) {
67+
DatumWriter<UserEvent> writer = new SpecificDatumWriter<>(UserEvent.class);;
68+
try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {
69+
final Encoder jsonEncoder = EncoderFactory.get().jsonEncoder(UserEvent.getClassSchema(), stream);
70+
writer.write(request, jsonEncoder);
71+
jsonEncoder.flush();
72+
return stream.toByteArray();
73+
} catch (IOException e) {
74+
// handle exception
75+
return new byte[0];
76+
}
77+
}
78+
79+
public UserEvent deSerializeAvro(byte[] data) {
80+
DatumReader<UserEvent> reader = new SpecificDatumReader<>(UserEvent.class);
81+
try {
82+
Decoder decoder = DecoderFactory.get().jsonDecoder(UserEvent.getClassSchema(), new String(data));
83+
return reader.read(null, decoder);
84+
} catch (IOException e) {
85+
// handle exception
86+
return null;
87+
}
88+
}
89+
}

‎libraries-data-io-2/src/test/java/com/baeldung/apachefury/serialization/FurySerializationUnitTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package com.baeldung.apachefury.serialization;
2+
3+
import com.baeldung.apachefury.event.Address;
4+
import com.baeldung.apachefury.event.UserEvent;
5+
import org.apache.fury.Fury;
6+
import org.apache.fury.config.Language;
7+
import org.junit.jupiter.api.Assertions;
8+
import org.junit.jupiter.api.Test;
9+
10+
import java.util.ArrayList;
11+
import java.util.List;
12+
import java.util.UUID;
13+
import java.util.logging.Logger;
14+
15+
class FurySerializationUnitTest {
16+
17+
static final Logger LOG = Logger.getLogger(FurySerializationUnitTest.class.getName());
18+
19+
@Test
20+
void whenUsingFurySerialization_thenGenerateByteOutput() {
21+
22+
Fury fury = Fury.builder()
23+
.withLanguage(Language.JAVA)
24+
.withAsyncCompilation(true)
25+
.build();
26+
27+
fury.register(UserEvent.class);
28+
fury.register(Address.class);
29+
30+
List<UserEvent> events = new ArrayList<>(100000);
31+
List<UserEvent> parserEvents = new ArrayList<>(100000);
32+
33+
for (int i = 0; i < 100000; i++) {
34+
final Address address = new Address(i+" Main St", "Spring field "+i, UUID.randomUUID().toString());
35+
events.add(new UserEvent(UUID.randomUUID()+"-"+i, "login", System.currentTimeMillis(), address));
36+
}
37+
38+
long startTime = System.currentTimeMillis();
39+
for (int i = 0; i < 100000; i++) {
40+
byte[] serializedData = fury.serialize(events.get(i));
41+
UserEvent temp = (UserEvent) fury.deserialize(serializedData);
42+
parserEvents.add(temp);
43+
}
44+
long endTime = System.currentTimeMillis();
45+
46+
long totalBytes = 0;
47+
for (int i = 0; i < 100000; i++) {
48+
byte[] serializedData = fury.serialize(events.get(i));
49+
totalBytes += serializedData.length;
50+
}
51+
52+
LOG.info("Apache Fury serialization time: " + (endTime - startTime) + " ms");
53+
LOG.info("Total bytes: " + totalBytes / (1024 * 1024) + " MB");
54+
55+
Assertions.assertEquals(events, parserEvents);
56+
}
57+
}

0 commit comments

Comments
 (0)