Skip to content

Commit 963efed

Browse files
bpstelios10bpstelios10
authored
[BAEL-9315] Introduced module for apache-camel-kserve (#18753)
* [BAEL-9315] Introduced module for apache-camel-kserve * Added module for triton server with pre-loaded model(should be downloaded) * Added module for the sentiment-service in java with apache-camel-kserve * [BAEL-9315] Extracted dependency versions to properties * Fixed the Java version to 21
1 parent fc90039 commit 963efed

File tree

11 files changed

+357
-0
lines changed

11 files changed

+357
-0
lines changed

‎messaging-modules/apache-camel-kserve/.gitignore

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
target/
2+
dependency-reduced-pom.xml
3+
4+
### IntelliJ IDEA ###
5+
.idea
6+
*.iws
7+
*.iml
8+
*.ipr
9+
10+
### Eclipse ###
11+
.apt_generated
12+
.classpath
13+
.factorypath
14+
.project
15+
.settings
16+
.springBeans
17+
.sts4-cache
18+
19+
### NetBeans ###
20+
/nbproject/private/
21+
/nbbuild/
22+
/dist/
23+
/nbdist/
24+
/.nb-gradle/
25+
build/
26+
27+
### VS Code ###
28+
.vscode/
29+
30+
### Mac OS ###
31+
.DS_Store

‎messaging-modules/apache-camel-kserve/README.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
This module contains 2 sub-modules:
2+
3+
1) triton server with pre-loaded model(should be downloaded)
4+
2) the sentiment-service in java with apache-camel-kserve
5+
6+
The modules both contain a Dockerfile and can be easily deployed locally using docker-compose.yml
7+
8+
First, you need to download the model from [huggingface](https://huggingface.co/pjxcharya/onnx-sentiment-model/tree/main) and place it in triton-server/models/sentiment/1.
9+
Then execute:
10+
11+
```bash
12+
docker-compose up --build
13+
```
14+
15+
The endpoint to test everything works is: `http://localhost:8080/sentiments?sentence=i probably like you`

‎messaging-modules/apache-camel-kserve/docker-compose.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
version: '3.8'
2+
3+
services:
4+
triton-server:
5+
build: ./triton-server
6+
environment:
7+
- NVIDIA_VISIBLE_DEVICES=all
8+
ports:
9+
- "8000:8000" # HTTP
10+
- "8001:8001" # gRPC
11+
- "8002:8002" # Metrics
12+
sentiment-service:
13+
build: ./sentiment-service
14+
ports:
15+
- "8080:8080"
16+
restart: unless-stopped

‎messaging-modules/apache-camel-kserve/pom.xml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
<packaging>pom</packaging>
7+
8+
<parent>
9+
<groupId>com.baeldung</groupId>
10+
<artifactId>messaging-modules</artifactId>
11+
<version>0.0.1-SNAPSHOT</version>
12+
</parent>
13+
14+
<artifactId>sentiment-parent-pom</artifactId>
15+
16+
<modules>
17+
<module>sentiment-service</module>
18+
</modules>
19+
</project>

‎messaging-modules/apache-camel-kserve/sentiment-service/Dockerfile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM eclipse-temurin:21-jre
2+
3+
WORKDIR /app
4+
5+
# Copy the fat JAR from the builder stage
6+
COPY target/sentiment-service-1.0-SNAPSHOT.jar app.jar
7+
8+
# Expose HTTP port
9+
EXPOSE 8080
10+
11+
# Run the app
12+
ENTRYPOINT ["java", "-jar", "app.jar"]

‎messaging-modules/apache-camel-kserve/sentiment-service/pom.xml

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<parent>
8+
<groupId>com.baeldung</groupId>
9+
<artifactId>sentiment-parent-pom</artifactId>
10+
<version>0.0.1-SNAPSHOT</version>
11+
</parent>
12+
13+
<name>Sentiment System - Service</name>
14+
<description>This is the main service of the system, that uses Apache Camel to integrate with Triton server and
15+
use an AI model for inference</description>
16+
<artifactId>sentiment-service</artifactId>
17+
18+
<properties>
19+
<java.version>21</java.version>
20+
<maven.compiler.source>${java.version}</maven.compiler.source>
21+
<maven.compiler.release>${java.version}</maven.compiler.release>
22+
<maven.compiler.target>${java.version}</maven.compiler.target>
23+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
24+
25+
<camel.version>4.13.0</camel.version>
26+
<jackson-databind.version>2.19.2</jackson-databind.version>
27+
<tokenizers.version>0.21.0</tokenizers.version>
28+
<maven-shade-plugin.version>3.6.0</maven-shade-plugin.version>
29+
</properties>
30+
31+
<dependencies>
32+
<!-- Core Camel -->
33+
<dependency>
34+
<groupId>org.apache.camel</groupId>
35+
<artifactId>camel-main</artifactId>
36+
<version>${camel.version}</version>
37+
</dependency>
38+
39+
<!-- REST support via Undertow -->
40+
<dependency>
41+
<groupId>org.apache.camel</groupId>
42+
<artifactId>camel-undertow</artifactId>
43+
<version>${camel.version}</version>
44+
</dependency>
45+
<dependency>
46+
<groupId>org.apache.camel</groupId>
47+
<artifactId>camel-rest</artifactId>
48+
<version>${camel.version}</version>
49+
</dependency>
50+
51+
<!-- KServe inference component -->
52+
<dependency>
53+
<groupId>org.apache.camel</groupId>
54+
<artifactId>camel-kserve</artifactId>
55+
<version>${camel.version}</version>
56+
</dependency>
57+
58+
<!-- JSON support -->
59+
<dependency>
60+
<groupId>com.fasterxml.jackson.core</groupId>
61+
<artifactId>jackson-databind</artifactId>
62+
<version>${jackson-databind.version}</version>
63+
</dependency>
64+
65+
<!-- lib to use for tokenizer -->
66+
<dependency>
67+
<groupId>ai.djl.huggingface</groupId>
68+
<artifactId>tokenizers</artifactId>
69+
<version>${tokenizers.version}</version>
70+
</dependency>
71+
</dependencies>
72+
73+
<build>
74+
<plugins>
75+
<plugin>
76+
<groupId>org.apache.maven.plugins</groupId>
77+
<artifactId>maven-jar-plugin</artifactId>
78+
<configuration>
79+
<archive>
80+
<manifest>
81+
<addClasspath>true</addClasspath>
82+
<mainClass>org.learnings.aimodels.sentiments.Application</mainClass>
83+
</manifest>
84+
</archive>
85+
</configuration>
86+
</plugin>
87+
<plugin>
88+
<groupId>org.apache.maven.plugins</groupId>
89+
<artifactId>maven-shade-plugin</artifactId>
90+
<version>${maven-shade-plugin.version}</version>
91+
<executions>
92+
<execution>
93+
<phase>package</phase>
94+
<goals><goal>shade</goal></goals>
95+
<configuration>
96+
<transformers>
97+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
98+
<mainClass>org.learnings.aimodels.sentiments.Application</mainClass>
99+
</transformer>
100+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
101+
</transformers>
102+
</configuration>
103+
</execution>
104+
</executions>
105+
</plugin>
106+
</plugins>
107+
</build>
108+
</project>

‎messaging-modules/apache-camel-kserve/sentiment-service/src/main/java/org/learnings/aimodels/sentiments/Application.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package org.learnings.aimodels.sentiments;
2+
3+
import org.apache.camel.CamelContext;
4+
import org.apache.camel.impl.DefaultCamelContext;
5+
import org.learnings.aimodels.sentiments.web.api.SentimentsRoute;
6+
import org.slf4j.Logger;
7+
import org.slf4j.LoggerFactory;
8+
9+
public class Application {
10+
11+
private static final Logger log = LoggerFactory.getLogger(Application.class);
12+
13+
public static void main(String[] args) throws Exception {
14+
CamelContext context = new DefaultCamelContext();
15+
context.addRoutes(new SentimentsRoute());
16+
17+
context.start();
18+
log.info("🚀 Sentiment service running on http://localhost:8080/sentiments");
19+
Thread.sleep(Long.MAX_VALUE);
20+
context.stop();
21+
}
22+
}

‎messaging-modules/apache-camel-kserve/sentiment-service/src/main/java/org/learnings/aimodels/sentiments/web/api/SentimentsRoute.java

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package org.learnings.aimodels.sentiments.web.api;
2+
3+
import ai.djl.huggingface.tokenizers.Encoding;
4+
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
5+
import com.google.protobuf.ByteString;
6+
import inference.GrpcPredictV2.InferTensorContents;
7+
import inference.GrpcPredictV2.ModelInferRequest;
8+
import inference.GrpcPredictV2.ModelInferResponse;
9+
import org.apache.camel.Exchange;
10+
import org.apache.camel.builder.RouteBuilder;
11+
import org.apache.camel.model.rest.RestBindingMode;
12+
import org.apache.camel.model.rest.RestParamType;
13+
import org.slf4j.Logger;
14+
import org.slf4j.LoggerFactory;
15+
16+
import java.nio.ByteOrder;
17+
import java.util.ArrayList;
18+
import java.util.Arrays;
19+
import java.util.List;
20+
import java.util.stream.Collectors;
21+
22+
public class SentimentsRoute extends RouteBuilder {
23+
24+
private static final Logger log = LoggerFactory.getLogger(SentimentsRoute.class);
25+
private final HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("distilbert-base-uncased");
26+
27+
@Override
28+
public void configure() {
29+
// Configure REST via Undertow
30+
restConfiguration()
31+
.component("undertow")
32+
.host("0.0.0.0")
33+
.port(8080)
34+
.bindingMode(RestBindingMode.off);
35+
36+
// REST GET endpoint
37+
rest("/sentiments")
38+
.get()
39+
.param().name("sentence").required(true).type(RestParamType.query).endParam()
40+
.outType(String[].class)
41+
.responseMessage().code(200).message("the sentence is.. ").endResponseMessage()
42+
.to("direct:classify");
43+
44+
// Main route
45+
from("direct:classify")
46+
.routeId("sentiment-inference")
47+
.setBody(this::createRequest)
48+
.setHeader("Content-Type", constant("application/json"))
49+
.to("kserve:infer?modelName=sentiment&target=host.docker.internal:8001")
50+
// .to("kserve:infer?modelName=sentiment&target=localhost:8001")
51+
.process(this::postProcess);
52+
}
53+
54+
private ModelInferRequest createRequest(Exchange exchange) {
55+
String sentence = exchange.getIn().getHeader("sentence", String.class);
56+
Encoding encoding = tokenizer.encode(sentence);
57+
List<Long> inputIds = Arrays.stream(encoding.getIds()).boxed().collect(Collectors.toList());
58+
List<Long> attentionMask = Arrays.stream(encoding.getAttentionMask()).boxed().collect(Collectors.toList());
59+
60+
var content0 = InferTensorContents.newBuilder().addAllInt64Contents(inputIds);
61+
var input0 = ModelInferRequest.InferInputTensor.newBuilder()
62+
.setName("input_ids").setDatatype("INT64").addShape(1).addShape(inputIds.size())
63+
.setContents(content0);
64+
65+
var content1 = InferTensorContents.newBuilder().addAllInt64Contents(attentionMask);
66+
var input1 = ModelInferRequest.InferInputTensor.newBuilder()
67+
.setName("attention_mask").setDatatype("INT64").addShape(1).addShape(attentionMask.size())
68+
.setContents(content1);
69+
70+
ModelInferRequest requestBody = ModelInferRequest.newBuilder()
71+
.addInputs(0, input0).addInputs(1, input1)
72+
.build();
73+
log.debug("-- payload: [{}]", requestBody);
74+
75+
return requestBody;
76+
}
77+
78+
private void postProcess(Exchange exchange) {
79+
log.debug("-- in response");
80+
ModelInferResponse response = exchange.getMessage().getBody(ModelInferResponse.class);
81+
82+
List<List<Float>> logits = response.getRawOutputContentsList().stream()
83+
.map(ByteString::asReadOnlyByteBuffer)
84+
.map(buf -> buf.order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer())
85+
.map(buf -> {
86+
List<Float> longs = new ArrayList<>(buf.remaining());
87+
while (buf.hasRemaining()) {
88+
longs.add(buf.get());
89+
}
90+
return longs;
91+
})
92+
.toList();
93+
94+
log.debug("-- logits: [{}]", logits);
95+
String result = Math.abs(logits.getFirst().getFirst()) < logits.getFirst().getLast() ? "good" : "bad";
96+
97+
exchange.getMessage().setBody(result);
98+
}
99+
}

‎messaging-modules/apache-camel-kserve/triton-server/Dockerfile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
FROM nvcr.io/nvidia/tritonserver:25.02-py3
2+
3+
# Copy the model repository into the container
4+
COPY models/ /models/
5+
6+
# Expose default Triton ports
7+
EXPOSE 8000 8001 8002
8+
9+
# Set entrypoint to run Triton with your model repo
10+
CMD ["tritonserver", "--model-repository=/models"]

‎messaging-modules/apache-camel-kserve/triton-server/models/sentiment/config.pbtxt

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: "sentiment"
2+
platform: "onnxruntime_onnx"
3+
max_batch_size: 8
4+
5+
input [
6+
{
7+
name: "input_ids"
8+
data_type: TYPE_INT64
9+
dims: [ -1 ]
10+
},
11+
{
12+
name: "attention_mask"
13+
data_type: TYPE_INT64
14+
dims: [ -1 ]
15+
}
16+
]
17+
18+
output [
19+
{
20+
name: "logits"
21+
data_type: TYPE_FP32
22+
dims: [ 2 ]
23+
}
24+
]

0 commit comments

Comments
 (0)