Merge pull request #1148 from edeandrea/update-ollama-dev-service

Re-write the Ollama dev service

Author: geoand

core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/devservice/DevServicesOllamaProcessor.java

@@ -4,6 +4,7 @@
 import java.math.RoundingMode;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -23,6 +24,7 @@
 import io.quarkiverse.langchain4j.deployment.items.DevServicesChatModelRequiredBuildItem;
 import io.quarkiverse.langchain4j.deployment.items.DevServicesEmbeddingModelRequiredBuildItem;
 import io.quarkiverse.langchain4j.deployment.items.DevServicesModelRequired;
+import io.quarkiverse.langchain4j.deployment.items.DevServicesOllamaConfigBuildItem;
 import io.quarkus.deployment.IsNormal;
 import io.quarkus.deployment.annotations.BuildProducer;
 import io.quarkus.deployment.annotations.BuildStep;
@@ -46,9 +48,11 @@ private void handleModels(List<DevServicesChatModelRequiredBuildItem> devService
             List<DevServicesEmbeddingModelRequiredBuildItem> devServicesEmbeddingModels,
             LoggingSetupBuildItem loggingSetupBuildItem,
             Optional<ConsoleInstalledBuildItem> consoleInstalledBuildItem,
+            Optional<DevServicesOllamaConfigBuildItem> ollamaDevServicesConfig,
             LaunchModeBuildItem launchMode,
             LangChain4jBuildConfig config,
             BuildProducer<DevServicesResultBuildItem> producer) {
+
         if (devServicesChatModels.isEmpty() && devServicesEmbeddingModels.isEmpty()) {
             return;
         }
@@ -57,18 +61,27 @@ private void handleModels(List<DevServicesChatModelRequiredBuildItem> devService
         var ollamaEmbeddingModels = devServicesEmbeddingModels.stream().filter(bi -> OLLAMA_PROVIDER.equals(bi.getProvider()))
                 .toList();
 
-        List<DevServicesModelRequired> allOllamaModels = new ArrayList<>();
+        Set<DevServicesModelRequired> allOllamaModels = new LinkedHashSet<>();
         allOllamaModels.addAll(ollamaChatModels);
         allOllamaModels.addAll(ollamaEmbeddingModels);
         if (allOllamaModels.isEmpty()) {
             return;
         }
 
-        OllamaClient client = OllamaClient.create(new OllamaClient.Options("localhost", config.devservices().port()));
+        var devServiceHost = ollamaDevServicesConfig
+                .map(c -> c.getConfig().get("langchain4j-ollama-dev-service.ollama.host"))
+                .orElse("localhost");
+
+        var devServicePort = ollamaDevServicesConfig
+                .map(c -> c.getConfig().get("langchain4j-ollama-dev-service.ollama.port"))
+                .map(Integer::parseInt)
+                .orElseGet(() -> config.devservices().port());
+
+        OllamaClient client = OllamaClient.create(new OllamaClient.Options(devServiceHost, devServicePort));
         try {
             Set<ModelName> localModels = client.localModels().stream().map(mi -> ModelName.of(mi.name()))
                     .collect(Collectors.toSet());
-            List<String> modelsToPull = new ArrayList<>(ollamaChatModels.size() + ollamaEmbeddingModels.size());
+            List<String> modelsToPull = new ArrayList<>(allOllamaModels.size());
             for (var requiredModel : allOllamaModels) {
                 if (localModels.contains(ModelName.of(requiredModel.getModelName()))) {
                     LOGGER.debug("Ollama already has model " + requiredModel.getModelName() + " pulled locally");
@@ -92,7 +105,7 @@ private void handleModels(List<DevServicesChatModelRequiredBuildItem> devService
                     });
             for (String model : modelsToPull) {
                 // we pull one model at a time and provide progress updates to the user via logging
-                LOGGER.info("Pulling model " + model);
+                LOGGER.infof("Pulling model %s", model);
                 AtomicReference<Long> LAST_UPDATE_REF = new AtomicReference<>();
 
                 CompletableFuture<Void> cf = new CompletableFuture<>();
@@ -120,9 +133,9 @@ public void onNext(OllamaClient.PullAsyncLine line) {
                             BigDecimal progress = percentage.setScale(2, RoundingMode.HALF_DOWN);
                             if (progress.compareTo(ONE_HUNDRED) >= 0) {
                                 // avoid showing 100% for too long
-                                LOGGER.infof("Verifying and cleaning up\n", progress);
+                                LOGGER.info("Verifying and cleaning up\n");
                             } else {
-                                LOGGER.infof("%s - Progress: %s%%\n", model, progress);
+                                LOGGER.infof("Downloading %s - Progress: %s%%\n", model, progress);
                             }
                         }
                     }
@@ -171,7 +184,7 @@ public void onComplete() {
 
             compressor.close();
 
-            String ollamaBaseUrl = String.format("http://localhost:%d", config.devservices().port());
+            String ollamaBaseUrl = String.format("http://%s:%d", devServiceHost, devServicePort);
 
             Map<String, String> modelBaseUrls = new HashMap<>();
             for (var bi : allOllamaModels) {

core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/devservice/JdkOllamaClient.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.ConnectException;
+import java.net.Socket;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.http.HttpClient;
@@ -37,6 +38,15 @@ public JdkOllamaClient(Options options) {
                 .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
     }
 
+    @Override
+    public boolean isRunning() {
+        try (var s = new Socket(this.options.host(), this.options.port())) {
+            return true;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
     @Override
     public List<ModelInfo> localModels() {
         String serverUrl = String.format("http://%s:%d/api/tags", options.host(), options.port());

core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/devservice/OllamaClient.java

@@ -11,6 +11,11 @@ static OllamaClient create(Options options) {
         return new JdkOllamaClient(options);
     }
 
+    /**
+     * Whether or not the server is running
+     */
+    boolean isRunning();
+
     /**
      * Returns a list of all models the server has
      */

model-providers/ollama/devservices/src/main/java/io/quarkiverse/langchain4j/ollama/devservices/OllamaDevServicesConfigBuildItem.java renamed to core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/items/DevServicesOllamaConfigBuildItem.java

@@ -1,4 +1,4 @@
-package io.quarkiverse.langchain4j.ollama.devservices;
+package io.quarkiverse.langchain4j.deployment.items;
 
 import java.util.Map;
 
@@ -7,11 +7,11 @@
 /**
  * Build item used to carry running values to Dev UI.
  */
-public final class OllamaDevServicesConfigBuildItem extends SimpleBuildItem {
+public final class DevServicesOllamaConfigBuildItem extends SimpleBuildItem {
 
     private final Map<String, String> config;
 
-    public OllamaDevServicesConfigBuildItem(Map<String, String> config) {
+    public DevServicesOllamaConfigBuildItem(Map<String, String> config) {
         this.config = config;
     }
 

docs/modules/ROOT/pages/includes/quarkus-langchain4j-ollama.adoc

@@ -42,6 +42,40 @@ endif::add-copy-button-to-env-var[]
 |boolean
 |`true`
 
+a|icon:lock[title=Fixed at build time] [[quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-devservices-enabled]] [.property-path]##link:#quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-devservices-enabled[`quarkus.langchain4j.ollama.devservices.enabled`]##
+
+[.description]
+--
+If Dev Services for Ollama has been explicitly enabled or disabled. Dev Services are generally enabled by default, unless there is an existing configuration present.
+
+
+ifdef::add-copy-button-to-env-var[]
+Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OLLAMA_DEVSERVICES_ENABLED+++[]
+endif::add-copy-button-to-env-var[]
+ifndef::add-copy-button-to-env-var[]
+Environment variable: `+++QUARKUS_LANGCHAIN4J_OLLAMA_DEVSERVICES_ENABLED+++`
+endif::add-copy-button-to-env-var[]
+--
+|boolean
+|`true`
+
+a|icon:lock[title=Fixed at build time] [[quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-devservices-image-name]] [.property-path]##link:#quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-devservices-image-name[`quarkus.langchain4j.ollama.devservices.image-name`]##
+
+[.description]
+--
+The Ollama container image to use.
+
+
+ifdef::add-copy-button-to-env-var[]
+Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OLLAMA_DEVSERVICES_IMAGE_NAME+++[]
+endif::add-copy-button-to-env-var[]
+ifndef::add-copy-button-to-env-var[]
+Environment variable: `+++QUARKUS_LANGCHAIN4J_OLLAMA_DEVSERVICES_IMAGE_NAME+++`
+endif::add-copy-button-to-env-var[]
+--
+|string
+|`ollama/ollama:latest`
+
 a|icon:lock[title=Fixed at build time] [[quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-chat-model-model-id]] [.property-path]##link:#quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-chat-model-model-id[`quarkus.langchain4j.ollama.chat-model.model-id`]##
 
 [.description]

docs/modules/ROOT/pages/includes/quarkus-langchain4j-ollama_quarkus.langchain4j.adoc

@@ -42,6 +42,40 @@ endif::add-copy-button-to-env-var[]
 |boolean
 |`true`
 
+a|icon:lock[title=Fixed at build time] [[quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-devservices-enabled]] [.property-path]##link:#quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-devservices-enabled[`quarkus.langchain4j.ollama.devservices.enabled`]##
+
+[.description]
+--
+If Dev Services for Ollama has been explicitly enabled or disabled. Dev Services are generally enabled by default, unless there is an existing configuration present.
+
+
+ifdef::add-copy-button-to-env-var[]
+Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OLLAMA_DEVSERVICES_ENABLED+++[]
+endif::add-copy-button-to-env-var[]
+ifndef::add-copy-button-to-env-var[]
+Environment variable: `+++QUARKUS_LANGCHAIN4J_OLLAMA_DEVSERVICES_ENABLED+++`
+endif::add-copy-button-to-env-var[]
+--
+|boolean
+|`true`
+
+a|icon:lock[title=Fixed at build time] [[quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-devservices-image-name]] [.property-path]##link:#quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-devservices-image-name[`quarkus.langchain4j.ollama.devservices.image-name`]##
+
+[.description]
+--
+The Ollama container image to use.
+
+
+ifdef::add-copy-button-to-env-var[]
+Environment variable: env_var_with_copy_button:+++QUARKUS_LANGCHAIN4J_OLLAMA_DEVSERVICES_IMAGE_NAME+++[]
+endif::add-copy-button-to-env-var[]
+ifndef::add-copy-button-to-env-var[]
+Environment variable: `+++QUARKUS_LANGCHAIN4J_OLLAMA_DEVSERVICES_IMAGE_NAME+++`
+endif::add-copy-button-to-env-var[]
+--
+|string
+|`ollama/ollama:latest`
+
 a|icon:lock[title=Fixed at build time] [[quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-chat-model-model-id]] [.property-path]##link:#quarkus-langchain4j-ollama_quarkus-langchain4j-ollama-chat-model-model-id[`quarkus.langchain4j.ollama.chat-model.model-id`]##
 
 [.description]

docs/modules/ROOT/pages/ollama.adoc

@@ -20,7 +20,26 @@ Once installed, check that Ollama is running using:
 
 === Dev Service
 
-Quarkus LangChain4j automatically handles the pulling of the models configured by the application, so there is no need for users to do so manually.
+The Dev Service included with the Ollama extension can do lots of things.
+
+The Dev Service automatically handles the pulling of the models configured by the application, so there is no need for users to do so manually.
+
+Additionally, if you aren't already running a local Ollama instance (either via the https://ollama.com/download[desktop client] or a local container image) then it will first start the https://java.testcontainers.org/modules/ollama[Ollama container] on a random port and bind it to your application by setting `quarkus.langchain4j.ollama.*.base-url` to the URL where Ollama is running.
+
+The container will also share downloaded models with any local client, so a model only needs to be downloaded the first time, regardless of whether you use the local Ollama client or the container provided by the Dev Service.
+
+If the Dev Service starts an Ollama container, it will expose the following configuration properties that you can use within your own configuration should you need to:
+
+[source,properties,subs=attributes+]
+----
+langchain4j-ollama-dev-service.ollama.host=host <1>
+langchain4j-ollama-dev-service.ollama.port=port <2>
+langchain4j-ollama-dev-service.ollama.endpoint=http://${langchain4j-ollama-dev-service.ollama.host}:${langchain4j-ollama-dev-service.ollama.port} <3>
+----
+
+. The host that the container is running on. Typically `localhost`, but it could be the name of the container network.
+. The port that the Ollama container is running on.
+. The fully-qualified url (host + port) to the running Ollama container.
 
 WARNING: Models are huge. For example Llama3 is 4.7Gb, so make sure you have enough disk space.
 
@@ -41,7 +60,7 @@ To integrate with models running on Ollama, add the following dependency into yo
 
 If no other LLM extension is installed, link:../ai-services.adoc[AI Services] will automatically utilize the configured Ollama model.
 
-By default, the extension uses `llama3.1`, the model we pulled in the previous section.
+By default, the extension uses `llama3.2`, the model we pulled in the previous section.
 You can change it by setting the `quarkus.langchain4j.ollama.chat-model.model-id` property in the `application.properties` file:
 
 [source,properties,subs=attributes+]

integration-tests/simple-ollama/pom.xml

@@ -22,11 +22,6 @@
       <artifactId>quarkus-langchain4j-ollama</artifactId>
       <version>${project.version}</version>
     </dependency>
-    <dependency>
-      <groupId>io.quarkiverse.langchain4j</groupId>
-      <artifactId>quarkus-langchain4j-ollama-devservices</artifactId>
-      <version>${project.version}</version>
-    </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-junit5</artifactId>

model-providers/ollama/deployment/pom.xml

@@ -31,6 +31,16 @@
             <artifactId>quarkus-langchain4j-ollama</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>ollama</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>junit</groupId>
+                    <artifactId>junit</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-junit5-internal</artifactId>

model-providers/ollama/deployment/src/main/java/io/quarkiverse/langchain4j/ollama/deployment/LangChain4jOllamaOpenAiBuildConfig.java

@@ -2,6 +2,7 @@
 
 import static io.quarkus.runtime.annotations.ConfigPhase.BUILD_TIME;
 
+import io.quarkiverse.langchain4j.ollama.deployment.devservices.OllamaDevServicesBuildConfig;
 import io.quarkus.runtime.annotations.ConfigRoot;
 import io.smallrye.config.ConfigMapping;
 
@@ -18,4 +19,9 @@ public interface LangChain4jOllamaOpenAiBuildConfig {
      * Embedding model related settings
      */
     EmbeddingModelBuildConfig embeddingModel();
+
+    /**
+     * Dev services related settings
+     */
+    OllamaDevServicesBuildConfig devservices();
 }