Skip to content

Commit 4eada1d

Browse files
cescoffiercescoffier
authored
Merge pull request #972 from cescoffier/ci-rework
Refactor CI workflows (only pull requests)
2 parents 38f1f44 + a57251d commit 4eada1d

File tree

29 files changed

+290
-86
lines changed

29 files changed

+290
-86
lines changed

‎.github/workflows/build-pull-request.yml‎

Lines changed: 97 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,15 @@ defaults:
2020
shell: bash
2121

2222
jobs:
23-
# Build the project, no native tests.
24-
build-and-test-jvm:
25-
name: Main Build
23+
# Quick build (no test, Java 17)
24+
# Artifacts will be reused in the other jobs
25+
quick-build:
26+
name: Quick Build
2627
runs-on: ubuntu-latest
2728
outputs:
28-
matrix: ${{ steps.set-matrix.outputs.matrix }}
29+
all_modules: ${{ steps.matrices.outputs.all_modules }}
30+
it_modules: ${{ steps.matrices.outputs.it_modules }}
31+
in_process_embedding_modules: ${{ steps.matrices.outputs.in_process_embedding_modules }}
2932

3033
steps:
3134
- uses: actions/checkout@v4
@@ -38,44 +41,62 @@ jobs:
3841
cache: 'maven'
3942

4043
- name: Build with Maven
41-
run: ./mvnw -B clean install -Dno-format -ntp
44+
run: ./mvnw -B clean install -DskipTests -Dno-format -ntp
4245

4346
- name: Zip the Maven repo
4447
run: |
4548
tar -czf ${{ runner.temp }}/maven-repo.tgz -C ~ .m2/repository
4649
# Avoid caching our own artifacts
4750
rm -Rf ~/.m2/repository/io/quarkiverse/langchain4j
51+
4852
- name: Persist the Maven repo
4953
uses: actions/upload-artifact@v4
5054
with:
5155
name: maven-repo
5256
path: ${{ runner.temp }}/maven-repo.tgz
5357
retention-days: 5
5458

55-
- name: Output the matrix
56-
id: set-matrix
59+
- name: Compute matrices
60+
id: matrices
5761
run: |
62+
# Compute the JVM tests
63+
ALL_MODULES=$(find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
64+
| sed 's|^\./||' \
65+
| sort -u \
66+
| jq -R -s -c 'split("\n")[:-1]')
67+
68+
# Integration tests (without the in-process embedding models)
69+
# Remove JLama from the list
5870
cd integration-tests
59-
# skip RAG module as it doesn't have any native-compatible tests now
60-
MATRIX='{"testModule":'$( \
71+
IT_MODULES=$( \
6172
find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
6273
| sed 's|^\./||' \
6374
| sort -u \
64-
| grep -v rag \
6575
| grep -v jlama \
66-
| jq -R -s -c 'split("\n")[:-1]' \
67-
)'}'
68-
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
69-
70-
# Test the project with different JDKs.
71-
test-jvm-alt:
76+
| grep -v in-process-embedding-models \
77+
| jq -R -s -c 'split("\n")[:-1]')
78+
79+
# We extract in process embedding models as there are many modules and we want parallelism
80+
cd in-process-embedding-models
81+
IN_PROCESS_EMBEDDING_MODULES=$( \
82+
find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
83+
| sed 's|^\./||' \
84+
| sort -u \
85+
| jq -R -s -c 'split("\n")[:-1]')
86+
87+
echo "all_modules=${ALL_MODULES}" >> $GITHUB_OUTPUT
88+
echo "it_modules=${IT_MODULES}" >> $GITHUB_OUTPUT
89+
echo "in_process_embedding_modules=${IN_PROCESS_EMBEDDING_MODULES}" >> $GITHUB_OUTPUT
90+
91+
test-jvm:
92+
needs: quick-build
7293
strategy:
7394
fail-fast: false
7495
matrix:
75-
os: [ubuntu-latest]
76-
java: [21, 22, 23]
77-
name: Test on ${{ matrix.os }} - ${{ matrix.java }}
78-
runs-on: ${{ matrix.os }}
96+
java: [ 17, 21, 23 ]
97+
module: ${{fromJson(needs.quick-build.outputs.all_modules)}}
98+
name: Build and Test ${{ matrix.module }} on Java ${{ matrix.java }}
99+
runs-on: ubuntu-latest
79100
steps:
80101
- uses: actions/checkout@v4
81102

@@ -84,23 +105,68 @@ jobs:
84105
with:
85106
distribution: temurin
86107
java-version: ${{ matrix.java }}
87-
cache: 'maven'
88108

89-
- name: Build with Maven
90-
run: ./mvnw -B clean install -Dno-format -ntp
109+
- name: Download the Maven repo
110+
uses: actions/download-artifact@v4
111+
with:
112+
name: maven-repo
113+
path: ..
114+
- name: Unzip the Maven Repo
115+
shell: bash
116+
run: |
117+
tar -xzf ../maven-repo.tgz -C ~
91118
92-
- name: Avoid caching our own artifacts
119+
# Build Jlama if JDK >= 21
120+
# It's not build by default as it requires Java 21+
121+
- name: Build JLama extension
122+
if: ${{ matrix.java >= 21 }}
93123
run: |
94-
rm -Rf ~/.m2/repository/io/quarkiverse/langchain4j
124+
./mvnw -B clean install -DskipTests -Dno-format -ntp -f model-providers/jlama/pom.xml
125+
126+
- name: Run tests of ${{ matrix.module }} with JDK ${{ matrix.java }}
127+
run: |
128+
cd ${{ matrix.module }} && ../mvnw -B verify -Dci=true -Dno-format -ntp
129+
95130
96131
native-tests:
97-
needs: build-and-test-jvm
98-
name: ${{matrix.testModule}} native tests
132+
needs: quick-build
133+
name: Native tests ${{matrix.module}}
99134
strategy:
100135
fail-fast: false
101-
matrix: ${{ fromJson(needs.build-and-test-jvm.outputs.matrix) }}
136+
matrix:
137+
module: ${{fromJson(needs.quick-build.outputs.it_modules)}}
102138
runs-on: ubuntu-latest
139+
steps:
140+
- uses: actions/checkout@v4
141+
142+
- name: Set up JDK 17
143+
uses: actions/setup-java@v4
144+
with:
145+
distribution: temurin
146+
java-version: 17
147+
148+
- name: Download the Maven repo
149+
uses: actions/download-artifact@v4
150+
with:
151+
name: maven-repo
152+
path: ..
153+
- name: Unzip the Maven Repo
154+
shell: bash
155+
run: |
156+
tar -xzf ../maven-repo.tgz -C ~
103157
158+
- name: Run integration tests ${{matrix.module}}
159+
run: |
160+
cd integration-tests/${{matrix.module}} && ../../mvnw -B verify -Dnative -Dci=true -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp
161+
162+
in-process-embedding-model-tests:
163+
needs: quick-build
164+
name: Native tests ${{matrix.module}}
165+
strategy:
166+
fail-fast: false
167+
matrix:
168+
module: ${{fromJson(needs.quick-build.outputs.in_process_embedding_modules)}}
169+
runs-on: ubuntu-latest
104170
steps:
105171
- uses: actions/checkout@v4
106172

@@ -120,6 +186,7 @@ jobs:
120186
run: |
121187
tar -xzf ../maven-repo.tgz -C ~
122188
123-
- name: Run integration test ${{matrix.testModule}}
189+
- name: Run integration tests ${{matrix.module}}
124190
run: |
125-
cd integration-tests/${{matrix.testModule}} && ../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dnative.surefire.skip -Dno-format -ntp
191+
cd integration-tests/in-process-embedding-models/${{matrix.module}}
192+
../../../mvnw -B verify -Dnative -Dquarkus.native.container-build -Dci=true -Dnative.surefire.skip -Dno-format -ntp

‎docs/modules/ROOT/pages/includes/attributes.adoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
:project-version: 0.20.3
22
:langchain4j-version: 0.35.0
3-
:examples-dir: ./../examples/
3+
:examples-dir: ./../examples/

‎embedding-stores/pom.xml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>io.quarkiverse.langchain4j</groupId>
6+
<artifactId>quarkus-langchain4j-parent</artifactId>
7+
<version>999-SNAPSHOT</version>
8+
</parent>
9+
<artifactId>quarkus-langchain4j-embedding-stores-parent</artifactId>
10+
<name>Quarkus LangChain4j - Embedding Stores - Parent</name>
11+
<packaging>pom</packaging>
12+
13+
<modules>
14+
<module>chroma</module>
15+
<module>infinispan</module>
16+
<module>milvus</module>
17+
<module>neo4j</module>
18+
<module>pgvector</module>
19+
<module>pinecone</module>
20+
<module>qdrant</module>
21+
<module>redis</module>
22+
</modules>
23+
24+
</project>

‎memory-stores/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>io.quarkiverse.langchain4j</groupId>
6+
<artifactId>quarkus-langchain4j-parent</artifactId>
7+
<version>999-SNAPSHOT</version>
8+
</parent>
9+
<artifactId>quarkus-langchain4j-memory-stores-parent</artifactId>
10+
<name>Quarkus LangChain4j - Memory Stores - Parent</name>
11+
<packaging>pom</packaging>
12+
13+
<modules>
14+
<module>memory-store-redis</module>
15+
</modules>
16+
17+
</project>

‎model-auth-providers/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>io.quarkiverse.langchain4j</groupId>
6+
<artifactId>quarkus-langchain4j-parent</artifactId>
7+
<version>999-SNAPSHOT</version>
8+
</parent>
9+
<artifactId>quarkus-langchain4j-model-auth-providers-parent</artifactId>
10+
<name>Quarkus LangChain4j - Model Auth Providers - Parent</name>
11+
<packaging>pom</packaging>
12+
13+
<modules>
14+
<module>oidc-model-auth-provider</module>
15+
</modules>
16+
17+
</project>

‎model-providers/pom.xml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
<parent>
6+
<groupId>io.quarkiverse.langchain4j</groupId>
7+
<artifactId>quarkus-langchain4j-parent</artifactId>
8+
<version>999-SNAPSHOT</version>
9+
</parent>
10+
<artifactId>quarkus-langchain4j-model-providers-parent</artifactId>
11+
<name>Quarkus LangChain4j - Model Providers - Parent</name>
12+
<packaging>pom</packaging>
13+
14+
<modules>
15+
<module>anthropic</module>
16+
<module>cohere</module>
17+
<module>hugging-face</module>
18+
<module>mistral</module>
19+
<module>ollama</module>
20+
<!-- Add another pom -->
21+
<module>openai/azure-openai</module>
22+
<module>openai/openai-common</module>
23+
<module>openai/openai-vanilla</module>
24+
<module>openai/testing-internal</module>
25+
26+
<module>openshift-ai</module>
27+
<module>vertex-ai</module>
28+
<module>vertex-ai-gemini</module>
29+
<module>watsonx</module>
30+
</modules>
31+
32+
<profiles>
33+
<profile>
34+
<id>jdk21-plus</id>
35+
<activation>
36+
<jdk>[21,)</jdk>
37+
</activation>
38+
<modules>
39+
<module>jlama</module>
40+
</modules>
41+
</profile>
42+
</profiles>
43+
44+
45+
</project>

‎pom.xml‎

Lines changed: 9 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -13,42 +13,13 @@
1313
<name>Quarkus LangChain4j - Parent</name>
1414
<modules>
1515
<module>core</module>
16-
17-
<module>embedding-stores/chroma</module>
18-
19-
<module>embedding-stores/infinispan</module>
20-
<module>embedding-stores/milvus</module>
21-
<module>embedding-stores/neo4j</module>
22-
<module>embedding-stores/pgvector</module>
23-
<module>embedding-stores/pinecone</module>
24-
<module>embedding-stores/qdrant</module>
25-
<module>embedding-stores/redis</module>
26-
<module>memory-stores/memory-store-redis</module>
27-
28-
<module>model-providers/anthropic</module>
29-
30-
<module>model-providers/cohere</module>
31-
<module>model-providers/hugging-face</module>
32-
<module>model-providers/mistral</module>
33-
<module>model-providers/ollama</module>
34-
<module>model-providers/openai/testing-internal</module>
35-
<module>model-providers/openai/azure-openai</module>
36-
<module>model-providers/openai/openai-common</module>
37-
<module>model-providers/openai/openai-vanilla</module>
38-
<module>model-providers/openshift-ai</module>
39-
<module>model-providers/vertex-ai</module>
40-
<module>model-providers/vertex-ai-gemini</module>
41-
<module>model-providers/watsonx</module>
42-
43-
<module>model-auth-providers/oidc-model-auth-provider</module>
44-
45-
<module>quarkus-integrations/websockets-next</module>
46-
47-
<module>web-search-engines/tavily</module>
48-
49-
<module>rag/easy-rag</module>
50-
51-
<module>rag/parsers-base</module>
16+
<module>embedding-stores</module>
17+
<module>memory-stores</module>
18+
<module>model-auth-providers</module>
19+
<module>model-providers</module>
20+
<module>quarkus-integrations</module>
21+
<module>rag</module>
22+
<module>tools</module>
5223
<module>testing-internal</module>
5324
</modules>
5425
<scm>
@@ -208,26 +179,9 @@
208179
</property>
209180
</activation>
210181
<modules>
211-
<module>samples/email-a-poem</module>
212-
<module>samples/cli-translator</module>
213-
<module>samples/review-triage</module>
214-
<module>samples/fraud-detection</module>
215-
<module>samples/secure-fraud-detection</module>
216-
<module>samples/secure-vertex-ai-gemini-poem</module>
217-
<module>samples/chatbot</module>
218-
<module>samples/chatbot-easy-rag</module>
219-
<module>samples/sql-chatbot</module>
220-
<module>samples/chatbot-web-search</module>
221-
</modules>
222-
</profile>
223-
<profile>
224-
<id>jdk21-plus</id>
225-
<activation>
226-
<jdk>[21,)</jdk>
227-
</activation>
228-
<modules>
229-
<module>model-providers/jlama</module>
182+
<module>samples</module>
230183
</modules>
231184
</profile>
185+
232186
</profiles>
233187
</project>

‎quarkus-integrations/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>io.quarkiverse.langchain4j</groupId>
6+
<artifactId>quarkus-langchain4j-parent</artifactId>
7+
<version>999-SNAPSHOT</version>
8+
</parent>
9+
<artifactId>quarkus-langchain4j-integrations-parent</artifactId>
10+
<name>Quarkus LangChain4j - Quarkus Integrations - Parent</name>
11+
<packaging>pom</packaging>
12+
13+
<modules>
14+
<module>websockets-next</module>
15+
</modules>
16+
17+
</project>

0 commit comments

Comments
 (0)