Skip to content

Commit a8f4c38

Browse files
cescoffiercescoffier
committed
Refactor Pull Request Workflow
The pull request workflow has been refactored into four distinct jobs for improved efficiency and parallelism: 1. Quick Build: Triggered on every pull request to provide fast feedback. 2. All Tests (JVM): After the quick build completes, a job runs tests for all individual modules using parallel execution across Java 17, 21, and 23 (covering the base, LTS, and latest GA versions). 3. Native Integration Tests: Following the quick build, another job runs all integration tests in native mode, leveraging parallelism. 4. In-Process Embedding Model Tests: Finally, after the quick build, a job runs all in-process embedding model integration tests in native mode, using parallelism. Jobs 2, 3, and 4 use matrix-based configurations to distribute tasks across multiple parallel jobs.
1 parent afbdb98 commit a8f4c38

File tree

28 files changed

+289
-86
lines changed

28 files changed

+289
-86
lines changed

‎.github/workflows/build-pull-request.yml‎

Lines changed: 98 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,15 @@ defaults:
2020
shell: bash
2121

2222
jobs:
23-
# Build the project, no native tests.
24-
build-and-test-jvm:
25-
name: Main Build
23+
# Quick build (no test, Java 17)
24+
# Artifacts will be reused in the other jobs
25+
quick-build:
26+
name: Quick Build
2627
runs-on: ubuntu-latest
2728
outputs:
28-
matrix: ${{ steps.set-matrix.outputs.matrix }}
29+
all_modules: ${{ steps.matrices.outputs.all_modules }}
30+
it_modules: ${{ steps.matrices.outputs.it_modules }}
31+
in_process_embedding_modules: ${{ steps.matrices.outputs.in_process_embedding_modules }}
2932

3033
steps:
3134
- uses: actions/checkout@v4
@@ -38,44 +41,63 @@ jobs:
3841
cache: 'maven'
3942

4043
- name: Build with Maven
41-
run: ./mvnw -B clean install -Dno-format -ntp
44+
run: ./mvnw -B clean install -DskipTests -Dno-format -ntp
4245

4346
- name: Zip the Maven repo
4447
run: |
4548
tar -czf ${{ runner.temp }}/maven-repo.tgz -C ~ .m2/repository
4649
# Avoid caching our own artifacts
4750
rm -Rf ~/.m2/repository/io/quarkiverse/langchain4j
51+
4852
- name: Persist the Maven repo
4953
uses: actions/upload-artifact@v4
5054
with:
5155
name: maven-repo
5256
path: ${{ runner.temp }}/maven-repo.tgz
5357
retention-days: 5
5458

55-
- name: Output the matrix
56-
id: set-matrix
59+
- name: Compute matrices
60+
id: matrices
5761
run: |
62+
# Compute the JVM tests
63+
ALL_MODULES=$(find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
64+
| sed 's|^\./||' \
65+
| sort -u \
66+
| jq -R -s -c 'split("\n")[:-1]')
67+
68+
# Integration tests (without the in-process embedding models)
69+
# Remove JLama from the list
5870
cd integration-tests
59-
# skip RAG module as it doesn't have any native-compatible tests now
60-
MATRIX='{"testModule":'$( \
71+
IT_MODULES=$( \
6172
find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
6273
| sed 's|^\./||' \
6374
| sort -u \
64-
| grep -v rag \
6575
| grep -v jlama \
66-
| jq -R -s -c 'split("\n")[:-1]' \
67-
)'}'
68-
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
69-
70-
# Test the project with different JDKs.
71-
test-jvm-alt:
76+
| grep -v in-process-embedding-models \
77+
| jq -R -s -c 'split("\n")[:-1]')
78+
79+
# We extract in process embedding models as there are many modules and we want parallelism
80+
cd in-process-embedding-models
81+
IN_PROCESS_EMBEDDING_MODULES=$( \
82+
find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
83+
| sed 's|^\./||' \
84+
| sort -u \
85+
| jq -R -s -c 'split("\n")[:-1]')
86+
87+
88+
echo "all_modules=${ALL_MODULES}" >> $GITHUB_OUTPUT
89+
echo "it_modules=${IT_MODULES}" >> $GITHUB_OUTPUT
90+
echo "in_process_embedding_modules=${IN_PROCESS_EMBEDDING_MODULES}" >> $GITHUB_OUTPUT
91+
92+
test-jvm:
93+
needs: quick-build
7294
strategy:
7395
fail-fast: false
7496
matrix:
75-
os: [ubuntu-latest]
76-
java: [21, 22, 23]
77-
name: Test on ${{ matrix.os }} - ${{ matrix.java }}
78-
runs-on: ${{ matrix.os }}
97+
java: [ 17, 21, 23 ]
98+
module: ${{fromJson(needs.quick-build.outputs.all_modules)}}
99+
name: Build and Test ${{ matrix.module }} on Java ${{ matrix.java }}
100+
runs-on: ubuntu-latest
79101
steps:
80102
- uses: actions/checkout@v4
81103

@@ -84,23 +106,68 @@ jobs:
84106
with:
85107
distribution: temurin
86108
java-version: ${{ matrix.java }}
87-
cache: 'maven'
88109

89-
- name: Build with Maven
90-
run: ./mvnw -B clean install -Dno-format -ntp
110+
- name: Download the Maven repo
111+
uses: actions/download-artifact@v4
112+
with:
113+
name: maven-repo
114+
path: ..
115+
- name: Unzip the Maven Repo
116+
shell: bash
117+
run: |
118+
tar -xzf ../maven-repo.tgz -C ~
91119
92-
- name: Avoid caching our own artifacts
120+
# Build Jlama if JDK >= 21
121+
# It's not build by default as it requires Java 21+
122+
- name: Build JLama extension
123+
if: ${{ matrix.java >= 21 }}
93124
run: |
94-
rm -Rf ~/.m2/repository/io/quarkiverse/langchain4j
125+
./mvnw -B clean install -DskipTests -Dno-format -ntp -f model-providers/jlama/pom.xml
126+
127+
- name: Run tests of ${{ matrix.module }} with JDK ${{ matrix.java }}
128+
run: |
129+
cd ${{ matrix.module }} && ../mvnw -B verify -Dno-format -ntp
130+
95131
96132
native-tests:
97-
needs: build-and-test-jvm
98-
name: ${{matrix.testModule}} native tests
133+
needs: quick-build
134+
name: Native tests ${{matrix.module}}
99135
strategy:
100136
fail-fast: false
101-
matrix: ${{ fromJson(needs.build-and-test-jvm.outputs.matrix) }}
137+
matrix:
138+
module: ${{fromJson(needs.quick-build.outputs.it_modules)}}
102139
runs-on: ubuntu-latest
140+
steps:
141+
- uses: actions/checkout@v4
142+
143+
- name: Set up JDK 17
144+
uses: actions/setup-java@v4
145+
with:
146+
distribution: temurin
147+
java-version: 17
148+
149+
- name: Download the Maven repo
150+
uses: actions/download-artifact@v4
151+
with:
152+
name: maven-repo
153+
path: ..
154+
- name: Unzip the Maven Repo
155+
shell: bash
156+
run: |
157+
tar -xzf ../maven-repo.tgz -C ~
103158
159+
- name: Run integration tests ${{matrix.module}}
160+
run: |
161+
cd integration-tests/${{matrix.module}} && ../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp
162+
163+
in-process-embedding-model-tests:
164+
needs: quick-build
165+
name: Native tests ${{matrix.module}}
166+
strategy:
167+
fail-fast: false
168+
matrix:
169+
module: ${{fromJson(needs.quick-build.outputs.in_process_embedding_modules)}}
170+
runs-on: ubuntu-latest
104171
steps:
105172
- uses: actions/checkout@v4
106173

@@ -120,6 +187,7 @@ jobs:
120187
run: |
121188
tar -xzf ../maven-repo.tgz -C ~
122189
123-
- name: Run integration test ${{matrix.testModule}}
190+
- name: Run integration tests ${{matrix.module}}
124191
run: |
125-
cd integration-tests/${{matrix.testModule}} && ../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp
192+
cd integration-tests/in-process-embedding-models/${{matrix.module}}
193+
../../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp

‎docs/modules/ROOT/pages/includes/attributes.adoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
:project-version: 0.20.3
22
:langchain4j-version: 0.35.0
3-
:examples-dir: ./../examples/
3+
:examples-dir: ./../examples/

‎embedding-stores/pom.xml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>io.quarkiverse.langchain4j</groupId>
6+
<artifactId>quarkus-langchain4j-parent</artifactId>
7+
<version>999-SNAPSHOT</version>
8+
</parent>
9+
<artifactId>quarkus-langchain4j-embedding-stores-parent</artifactId>
10+
<name>Quarkus LangChain4j - Embedding Stores - Parent</name>
11+
<packaging>pom</packaging>
12+
13+
<modules>
14+
<module>chroma</module>
15+
<module>infinispan</module>
16+
<module>milvus</module>
17+
<module>neo4j</module>
18+
<module>pgvector</module>
19+
<module>pinecone</module>
20+
<module>qdrant</module>
21+
<module>redis</module>
22+
</modules>
23+
24+
</project>

‎memory-stores/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>io.quarkiverse.langchain4j</groupId>
6+
<artifactId>quarkus-langchain4j-parent</artifactId>
7+
<version>999-SNAPSHOT</version>
8+
</parent>
9+
<artifactId>quarkus-langchain4j-memory-stores-parent</artifactId>
10+
<name>Quarkus LangChain4j - Memory Stores - Parent</name>
11+
<packaging>pom</packaging>
12+
13+
<modules>
14+
<module>memory-store-redis</module>
15+
</modules>
16+
17+
</project>

‎model-auth-providers/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>io.quarkiverse.langchain4j</groupId>
6+
<artifactId>quarkus-langchain4j-parent</artifactId>
7+
<version>999-SNAPSHOT</version>
8+
</parent>
9+
<artifactId>quarkus-langchain4j-model-auth-providers-parent</artifactId>
10+
<name>Quarkus LangChain4j - Model Auth Providers - Parent</name>
11+
<packaging>pom</packaging>
12+
13+
<modules>
14+
<module>oidc-model-auth-provider</module>
15+
</modules>
16+
17+
</project>

‎model-providers/pom.xml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
<parent>
6+
<groupId>io.quarkiverse.langchain4j</groupId>
7+
<artifactId>quarkus-langchain4j-parent</artifactId>
8+
<version>999-SNAPSHOT</version>
9+
</parent>
10+
<artifactId>quarkus-langchain4j-model-providers-parent</artifactId>
11+
<name>Quarkus LangChain4j - Model Providers - Parent</name>
12+
<packaging>pom</packaging>
13+
14+
<modules>
15+
<module>anthropic</module>
16+
<module>cohere</module>
17+
<module>hugging-face</module>
18+
<module>mistral</module>
19+
<module>ollama</module>
20+
<!-- Add another pom -->
21+
<module>openai/azure-openai</module>
22+
<module>openai/openai-common</module>
23+
<module>openai/openai-vanilla</module>
24+
<module>openai/testing-internal</module>
25+
26+
<module>openshift-ai</module>
27+
<module>vertex-ai</module>
28+
<module>vertex-ai-gemini</module>
29+
<module>watsonx</module>
30+
</modules>
31+
32+
<profiles>
33+
<profile>
34+
<id>jdk21-plus</id>
35+
<activation>
36+
<jdk>[21,)</jdk>
37+
</activation>
38+
<modules>
39+
<module>jlama</module>
40+
</modules>
41+
</profile>
42+
</profiles>
43+
44+
45+
</project>

‎pom.xml‎

Lines changed: 9 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -13,42 +13,13 @@
1313
<name>Quarkus LangChain4j - Parent</name>
1414
<modules>
1515
<module>core</module>
16-
17-
<module>embedding-stores/chroma</module>
18-
19-
<module>embedding-stores/infinispan</module>
20-
<module>embedding-stores/milvus</module>
21-
<module>embedding-stores/neo4j</module>
22-
<module>embedding-stores/pgvector</module>
23-
<module>embedding-stores/pinecone</module>
24-
<module>embedding-stores/qdrant</module>
25-
<module>embedding-stores/redis</module>
26-
<module>memory-stores/memory-store-redis</module>
27-
28-
<module>model-providers/anthropic</module>
29-
30-
<module>model-providers/cohere</module>
31-
<module>model-providers/hugging-face</module>
32-
<module>model-providers/mistral</module>
33-
<module>model-providers/ollama</module>
34-
<module>model-providers/openai/testing-internal</module>
35-
<module>model-providers/openai/azure-openai</module>
36-
<module>model-providers/openai/openai-common</module>
37-
<module>model-providers/openai/openai-vanilla</module>
38-
<module>model-providers/openshift-ai</module>
39-
<module>model-providers/vertex-ai</module>
40-
<module>model-providers/vertex-ai-gemini</module>
41-
<module>model-providers/watsonx</module>
42-
43-
<module>model-auth-providers/oidc-model-auth-provider</module>
44-
45-
<module>quarkus-integrations/websockets-next</module>
46-
47-
<module>web-search-engines/tavily</module>
48-
49-
<module>rag/easy-rag</module>
50-
51-
<module>rag/parsers-base</module>
16+
<module>embedding-stores</module>
17+
<module>memory-stores</module>
18+
<module>model-auth-providers</module>
19+
<module>model-providers</module>
20+
<module>quarkus-integrations</module>
21+
<module>rag</module>
22+
<module>tools</module>
5223
<module>testing-internal</module>
5324
</modules>
5425
<scm>
@@ -208,26 +179,9 @@
208179
</property>
209180
</activation>
210181
<modules>
211-
<module>samples/email-a-poem</module>
212-
<module>samples/cli-translator</module>
213-
<module>samples/review-triage</module>
214-
<module>samples/fraud-detection</module>
215-
<module>samples/secure-fraud-detection</module>
216-
<module>samples/secure-vertex-ai-gemini-poem</module>
217-
<module>samples/chatbot</module>
218-
<module>samples/chatbot-easy-rag</module>
219-
<module>samples/sql-chatbot</module>
220-
<module>samples/chatbot-web-search</module>
221-
</modules>
222-
</profile>
223-
<profile>
224-
<id>jdk21-plus</id>
225-
<activation>
226-
<jdk>[21,)</jdk>
227-
</activation>
228-
<modules>
229-
<module>model-providers/jlama</module>
182+
<module>samples</module>
230183
</modules>
231184
</profile>
185+
232186
</profiles>
233187
</project>

‎quarkus-integrations/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>io.quarkiverse.langchain4j</groupId>
6+
<artifactId>quarkus-langchain4j-parent</artifactId>
7+
<version>999-SNAPSHOT</version>
8+
</parent>
9+
<artifactId>quarkus-langchain4j-integrations-parent</artifactId>
10+
<name>Quarkus LangChain4j - Quarkus Integrations - Parent</name>
11+
<packaging>pom</packaging>
12+
13+
<modules>
14+
<module>websockets-next</module>
15+
</modules>
16+
17+
</project>

0 commit comments

Comments
 (0)