Skip to content

Commit 0e6213b

Browse files
heremaps-botheremaps-bot
heremaps-bot
authored and
heremaps-bot
committed
HERE Data SDK for Java & Scala Release 2.48.3
1 parent 45ba8cb commit 0e6213b

File tree

171 files changed

+5411
-7541
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

171 files changed

+5411
-7541
lines changed

‎README.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,6 @@ For more information, see [Data Archiving Library Developer Guide](https://devel
4444
| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------- | --------------------------------------------------------- |
4545
| Archiving SDII stream data in Avro | The example shows how to use the Data Archiving Library to quickly develop an archiving solution that archives data in Avro format. | [Java](data-archive/java/avro-example) | Data Archiving Library, Flink, Stream, Avro, SDII |
4646
| Archiving SDII stream data in Parquet | The example shows how to use the Data Archiving Library to quickly develop an archiving solution that archives data in Parquet format. | [Java](data-archive/java/parquet-example) | Data Archiving Library, Flink, Stream, Parquet, SDII |
47-
| Archiving SDII stream data in Protobuf | The example shows how to use the Data Archiving Library to quickly develop an archiving solution that archives data in Protobuf format. | [Java](data-archive/java/protobuf-example) | Data Archiving Library, Flink, Stream, Protobuf, SDII |
48-
| Archiving SENSORIS stream data in Parquet | The example shows how to use the Data Archiving Library to quickly develop an archiving solution that archives SENSORIS data in Parquet format. | [Java](data-archive/java/sensoris-parquet-example) | Data Archiving Library, Flink, Stream, SENSORIS, Parquet |
4947
| Archiving SENSORIS stream data in Protobuf | The example shows how to use the Data Archiving Library to quickly develop an archiving solution that archives SENSORIS data in Protobuf format. | [Java](data-archive/java/sensoris-protobuf-example) | Data Archiving Library, Flink, Stream, SENSORIS, Protobuf |
5048

5149
### Compacting Index Data

‎data-archive/java/avro-example/README.md‎

Lines changed: 490 additions & 184 deletions
Large diffs are not rendered by default.

‎data-archive/java/avro-example/pom.xml‎

Lines changed: 66 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
<groupId>com.here.platform.data.archive</groupId>
66
<artifactId>data-archive-avro-example</artifactId>
7-
<version>0.0.818</version>
7+
<version>0.0.824</version>
88
<packaging>jar</packaging>
99

1010
<!-- Meta information section start -->
@@ -23,17 +23,17 @@
2323
<connection>scm:git:https://github.com/heremaps/here-workspace-examples-java-scala.git</connection>
2424
<developerConnection>scm:git:[email protected]:heremaps/here-workspace-examples-java-scala.git</developerConnection>
2525
<url>https://github.com/heremaps/here-workspace-examples-java-scala</url>
26-
<tag>2.47.2</tag>
26+
<tag>2.48.3</tag>
2727
</scm>
2828

2929
<!-- Meta information section end -->
3030
<properties>
3131
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
3232
<java.version>1.8</java.version>
3333

34-
<sdk-bom.version>2.47.2</sdk-bom.version>
34+
<sdk-bom.version>2.48.3</sdk-bom.version>
3535
<scala.compat.version>2.12</scala.compat.version>
36-
<artifact.wagon.version>2.0.13</artifact.wagon.version>
36+
<artifact.wagon.version>2.0.14</artifact.wagon.version>
3737

3838
<avro.version>1.10.0</avro.version>
3939
<commons-logging.version>1.2</commons-logging.version>
@@ -125,6 +125,16 @@
125125
</dependency>
126126
</dependencies>
127127

128+
<repositories>
129+
<repository>
130+
<id>HERE_PLATFORM_ARTIFACT</id>
131+
<layout>here</layout>
132+
<!-- The 'here+' prefix serves to invoke the HERE wagon extention that handles OAUTH
133+
and re-writes Maven URIs for the HERE secure repository.-->
134+
<url>here+artifact-service://artifact-service</url>
135+
</repository>
136+
</repositories>
137+
128138
<build>
129139
<plugins>
130140
<!-- Java Compiler -->
@@ -199,18 +209,61 @@
199209
</extensions>
200210
</build>
201211

202-
<!-- This profile helps to make things run out of the box in IntelliJ -->
203-
<!-- It adds Flink's core classes to the runtime class path. -->
204-
<!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->
212+
<!-- This profile helps to run this example via the `mvn compile exec:java` command -->
205213
<profiles>
206214
<profile>
207-
<id>add-dependencies-for-IDEA</id>
208-
<activation>
209-
<property>
210-
<name>idea.version</name>
211-
</property>
212-
</activation>
215+
<id>add-dependencies-for-local-run</id>
213216
<dependencies>
217+
<dependency>
218+
<groupId>org.apache.flink</groupId>
219+
<artifactId>flink-runtime_${scala.compat.version}</artifactId>
220+
<scope>compile</scope>
221+
</dependency>
222+
<dependency>
223+
<groupId>org.apache.flink</groupId>
224+
<artifactId>flink-shaded-asm-7</artifactId>
225+
<scope>compile</scope>
226+
</dependency>
227+
<dependency>
228+
<groupId>com.esotericsoftware.minlog</groupId>
229+
<artifactId>minlog</artifactId>
230+
<scope>compile</scope>
231+
</dependency>
232+
<dependency>
233+
<groupId>org.apache.flink</groupId>
234+
<artifactId>flink-shaded-guava</artifactId>
235+
<scope>compile</scope>
236+
</dependency>
237+
<dependency>
238+
<groupId>org.apache.flink</groupId>
239+
<artifactId>flink-optimizer_${scala.compat.version}</artifactId>
240+
<scope>compile</scope>
241+
</dependency>
242+
<dependency>
243+
<groupId>org.apache.flink</groupId>
244+
<artifactId>flink-shaded-netty</artifactId>
245+
<scope>compile</scope>
246+
</dependency>
247+
<dependency>
248+
<groupId>org.apache.flink</groupId>
249+
<artifactId>flink-queryable-state-client-java</artifactId>
250+
<scope>compile</scope>
251+
</dependency>
252+
<dependency>
253+
<groupId>org.apache.flink</groupId>
254+
<artifactId>flink-shaded-jackson</artifactId>
255+
<scope>compile</scope>
256+
</dependency>
257+
<dependency>
258+
<groupId>com.here.platform.data.client</groupId>
259+
<artifactId>local-support_${scala.compat.version}</artifactId>
260+
<exclusions>
261+
<exclusion>
262+
<groupId>com.here.platform.data.client</groupId>
263+
<artifactId>client-core_${scala.compat.version}</artifactId>
264+
</exclusion>
265+
</exclusions>
266+
</dependency>
214267
<dependency>
215268
<groupId>org.apache.flink</groupId>
216269
<artifactId>flink-core</artifactId>
@@ -268,18 +321,5 @@
268321
</dependency>
269322
</dependencies>
270323
</profile>
271-
<!-- HERE Artifact Service repository -->
272-
<profile>
273-
<activation>
274-
<activeByDefault>true</activeByDefault>
275-
</activation>
276-
<id>here-artifact-service</id>
277-
<repositories>
278-
<repository>
279-
<id>HERE_PLATFORM_ARTIFACT</id>
280-
<url>here+artifact-service://artifact-service</url>
281-
</repository>
282-
</repositories>
283-
</profile>
284324
</profiles>
285325
</project>

‎data-archive/java/avro-example/src/main/java/com/here/platform/data/archive/example/AvroHelper.java‎

Lines changed: 0 additions & 99 deletions
This file was deleted.

‎data-archive/java/avro-example/src/main/java/com/here/platform/data/archive/example/AvroMultiKeysExample.java‎

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,18 @@
2222
import com.here.platform.dal.custom.MultiKeysUDF;
2323
import com.here.sdii.v3.SdiiCommon;
2424
import com.here.sdii.v3.SdiiMessage;
25-
import java.util.ArrayList;
26-
import java.util.Collections;
27-
import java.util.HashMap;
28-
import java.util.HashSet;
29-
import java.util.Iterator;
30-
import java.util.List;
31-
import java.util.Map;
32-
import java.util.Set;
33-
import java.util.Spliterator;
34-
import java.util.Spliterators;
25+
import java.io.ByteArrayOutputStream;
26+
import java.io.File;
27+
import java.io.IOException;
28+
import java.io.OutputStream;
29+
import java.util.*;
3530
import java.util.stream.StreamSupport;
31+
import org.apache.avro.Schema;
32+
import org.apache.avro.file.DataFileReader;
33+
import org.apache.avro.file.DataFileWriter;
34+
import org.apache.avro.protobuf.ProtobufData;
35+
import org.apache.avro.protobuf.ProtobufDatumReader;
36+
import org.apache.avro.protobuf.ProtobufDatumWriter;
3637
import org.slf4j.Logger;
3738
import org.slf4j.LoggerFactory;
3839

@@ -118,10 +119,41 @@ public byte[] aggregate(Map<String, Object> keys, Iterator<byte[]> messages) {
118119
}
119120
})
120121
.iterator();
121-
return AvroHelper.aggregateProtobufMessagesAsAvro(sdiiMessages, SdiiMessage.Message.class);
122+
return aggregateProtobufMessagesAsAvro(sdiiMessages, SdiiMessage.Message.class);
122123
} catch (Exception e) {
123124
LOG.error("Aggregation errors....", e);
124125
}
125126
return null;
126127
}
128+
129+
public static <V> byte[] aggregateProtobufMessagesAsAvro(Iterator<V> iterator, Class<V> cls)
130+
throws Exception {
131+
ByteArrayOutputStream os = new ByteArrayOutputStream();
132+
asAvro(iterator, cls, os);
133+
os.close();
134+
return os.toByteArray();
135+
}
136+
137+
public static <V> void asAvro(Iterator<V> iterator, Class<V> cls, OutputStream os)
138+
throws Exception {
139+
ProtobufDatumWriter<V> writer = new ProtobufDatumWriter<>(cls);
140+
try (DataFileWriter<V> dataFileWriter = new DataFileWriter<>(writer)) {
141+
Schema schema = ProtobufData.get().getSchema(cls);
142+
dataFileWriter.create(schema, os);
143+
while (iterator.hasNext()) {
144+
dataFileWriter.append(iterator.next());
145+
}
146+
}
147+
}
148+
149+
public static <V> List<V> fromFile(File file, Class<V> cls) throws IOException {
150+
ProtobufDatumReader<V> reader = new ProtobufDatumReader<>(cls);
151+
DataFileReader<V> dataFileReader = new DataFileReader<>(file, reader);
152+
List<V> list = new LinkedList<>();
153+
while (dataFileReader.hasNext()) {
154+
list.add(dataFileReader.next());
155+
}
156+
dataFileReader.close();
157+
return list;
158+
}
127159
}

0 commit comments

Comments
 (0)