Query tools vector store

Author: slimslenderslacks

dev/insert_tool.clj

@@ -0,0 +1,41 @@
+(ns insert-tool 
+  (:require
+   [babashka.process :as process]
+   [cheshire.core :as json]
+   [clojure.string :as string]))
+
+(comment
+  ;; query
+  ;; score is distance so lower is closer
+  (->
+    (deref
+      (process/process
+        {:cwd "/Users/slim/docker/labs-ai-tools-for-devs/tools_vector_store/"
+         :out :string
+         :err :string}
+        "docker"
+        "run"
+        "--rm"
+        "-e" (format "OPENAI_API_KEY=%s" (string/trim (slurp "/Users/slim/.openai-api-key")))
+        "-v" (format "%s:%s" "/Users/slim/docker/labs-ai-tools-for-devs/tools_vector_store/chroma_db" "/app/chroma_db")
+        "vonwig/tools-vector-store:local" "hola"
+        ))
+    :out
+    (json/parse-string true)
+    ))
+
+(defn insert []
+  ;; insert
+  (deref
+    (process/process
+      {:cwd "/Users/slim/docker/labs-ai-tools-for-devs/tools_vector_store/"
+       :out :string
+       :err :string}
+      "docker"
+      "run"
+      "--rm"
+      "--entrypoint" "python"
+      "-e" (format "OPENAI_API_KEY=%s" (string/trim (slurp "/Users/slim/.openai-api-key")))
+      "-v" (format "%s:%s" "/Users/slim/docker/labs-ai-tools-for-devs/tools_vector_store/chroma_db" "/app/chroma_db")
+      "vonwig/tools-vector-store:local" "insert.py"
+      )))

flake.nix

@@ -70,7 +70,7 @@
             # our application makes calls to the curl binary
             #  therefore, wrap the custom-jdk in a script with curl in the PATH
             entrypoint = pkgs.writeShellScriptBin "entrypoint" ''
-              export PATH=${pkgs.lib.makeBinPath [pkgs.curl]}
+              export PATH=${pkgs.lib.makeBinPath [pkgs.curl pkgs.git]}
               export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
               ${custom-jdk}/bin/agent-graph "$@"
             '';

graphs/prompts/journals/2024_09_03.md

@@ -46,11 +46,8 @@
 		- if the project being analyzed is located somewhere other than $PWD then the `--host-dir` will need to be updated.
 - [[Running the Prompt Engine]]
 - [[Authoring Prompts]]
-- Here is a prompt file with lots of non-default metadata (it uses  [extractors]([[Prompt Extractors]]), a [[tool]], and uses a local llm in [[ollama]]).  It uses one system prompt, and one user prompt.  Note that the user prompt contains a moustache template to pull data in from an extractor.
+- ---
   id:: 66d7f3ff-8769-40b3-b6b5-fc4fceea879e
-  
-  ```md
-  ---
   extractors:
     - name: linguist
       image: vonwig/go-linguist:latest
@@ -87,6 +84,4 @@
   This project contains {{language}} code.
   {{/linguist}}
 
-  Can you find any language specific project files and list them?
-  
-  ```
+  Can you find any language specific project files and list them?

graphs/prompts/journals/2024_10_23.md

@@ -0,0 +1 @@
+-

graphs/prompts/pages/tools_vector_store.md

@@ -0,0 +1,3 @@
+- [langchain adapter for sqlitevec](https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/vectorstores/sqlitevec.py) #link
+	- this imports the `sqlite3` and `sqlite_vec` python packages.  At connection time, we use `sqlite_vec` to load the vector extensions into sqlite.
+-

src/graph.clj

@@ -106,6 +106,10 @@
                     calls)
                    (async/reduce conj []))))})))
 
+(defn tools-query
+  [_]
+  (async/go {}))
+
 (declare stream chat-with-tools)
 
 ; tool_calls are maps with an id and a function with arguments an name
@@ -192,9 +196,11 @@
       (add-node "tool" tool)
       (add-node "end" end)
       (add-node "sub-graph" sub-graph)
-      (add-edge "start" "completion")
-      (add-edge "tool" "completion")
-      (add-edge "sub-graph" "completion")
+      (add-node "tools-query" tools-query)
+      (add-edge "start" "tools-query")
+      (add-edge "tool" "tools-query")
+      (add-edge "sub-graph" "tools-query")
+      (add-edge "tools-query" "completion")
       (add-conditional-edges "completion" tool-or-end)))
 
 (comment

tools_vector_store/.dockerignore

@@ -0,0 +1,68 @@
+# Include any files or directories that you don't want to be copied to your
+# container here (e.g., local build artifacts, temporary files, etc.).
+#
+# For more help, visit the .dockerignore file reference guide at
+# https://docs.docker.com/engine/reference/builder/#dockerignore-file
+
+**/.DS_Store
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
+# Include any files or directories that you don't want to be copied to your
+# container here (e.g., local build artifacts, temporary files, etc.).
+#
+# For more help, visit the .dockerignore file reference guide at
+# https://docs.docker.com/go/build-context-dockerignore/
+
+**/.DS_Store
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose.y*ml
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

tools_vector_store/.envrc

@@ -0,0 +1 @@
+use flake

tools_vector_store/Dockerfile

@@ -0,0 +1,42 @@
+# syntax=docker/dockerfile:1
+
+FROM python:3.12.6-slim AS base
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+ENV OPENAI_API_KEY=""
+
+WORKDIR /app
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/go/dockerfile-user-best-practices/
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
+# Leverage a bind mount to requirements.txt to avoid having to copy them into
+# into this layer.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements.txt,target=requirements.txt \
+    python -m pip install -r requirements.txt
+
+# Switch to the non-privileged user to run the application.
+USER appuser
+
+# Copy the source code into the container.
+COPY ./*.py .
+
+ENTRYPOINT ["python","query.py"]

tools_vector_store/derivation.nix

@@ -0,0 +1,18 @@
+{ lib, python3Packages, sqlite-vec }:
+with python3Packages;
+buildPythonApplication rec {
+  pname = "vectorstore";
+  version = "0.1.0";
+  src = ./.;
+  dependencies = [
+    setuptools
+  ];
+  propagatedBuildInputs = [langchain langchain-community langchain-openai sqlite-vec];
+  nativeCheckInputs = [pytest];
+  checkPhase = ''
+  '';
+  build-system = [
+    setuptools
+  ];
+}
+