Optimize git pulls

Author: slimslenderslacks

docs/content/tools/docs/gordon.md

@@ -20,7 +20,7 @@ services:
 
 volumes:
   docker-prompts:
-    external: true
+    external: false
 ```
 
 ## debugging

src/git.clj

@@ -38,15 +38,11 @@
           parse-ref
           add-attributes))
 
-(defn- hashch
+(defn hashch
   "returns #uuid"
   [m]
   (str (hasch/uuid5 (hasch/edn-hash (select-keys m [:owner :repo :ref])))))
 
-(comment
-  (hashch {:owner "docker" :repo "labs-make-runbook"})
-  (hashch {:owner "docker" :repo "labs-make-runbook" :ref "main"}))
-
 ;(def prompts-cache (fs/file "/Users/slim/docker/labs-make-runbook/prompts-cache"))
 (defn prompts-cache []
   (let [default-dir (fs/file (System/getenv "HOME") ".prompts-cache")]
@@ -96,13 +92,40 @@
 (comment
   (clone {:dir "/Users/slim/crap" :owner "docker" :repo "labs-make-runbook" :ref "main" :ref-hash "crap"}))
 
+(defn cache-dir [{:keys [ref-hash]}]
+  (fs/file (prompts-cache) ref-hash))
+
+(defn collect-unique-cache-dirs [coll]
+  (->> coll
+       (sort-by (comp :ref-hash :ref))
+       (partition-by (comp :ref-hash :ref))
+       (map first)))
+
+(defn refresh-cache 
+  " pure side-effect - clone or pull the cache"
+  [coll]
+  (doseq [m coll]
+    (if (fs/exists? (-> m :ref :dir))
+      (pull (:ref m))
+      (clone (:ref m)))
+    (when (not (= 0 (:exit-code m)))
+      (jsonrpc/notify :error {:content (str m)}))))
+
+(defn cached-prompt-file [{{:keys [dir path]} :ref :as m}]
+  (if path
+    (let [cached-path (fs/file dir path)]
+      (if (fs/exists? cached-path)
+        (assoc m :cached-path cached-path)
+        m))
+    m))
+
 (defn prompt-file
   "returns the path or nil if the github ref does not resolve
    throws if the path in the repo does not exist or if the clone fails"
   [ref]
   (when-let [{:keys [ref path] :as git-ref-map} (parse-github-ref ref)]
     (let [ref-hash (hashch (select-keys git-ref-map [:owner :repo :ref]))
-          dir (fs/file (prompts-cache) ref-hash)
+          dir (cache-dir {:ref-hash ref-hash})
           m (if (fs/exists? dir)
               (pull {:dir dir :ref ref})
               (clone (merge git-ref-map {:dir (fs/parent dir) :ref-hash ref-hash})))]

src/jsonrpc/db.clj

@@ -8,18 +8,9 @@
 (def db* (atom {}))
 
 (defn get-prompt-data [{:keys [register] :as opts}]
-  (logger/info "get-prompt-data " register)
   (->> register
-       (map (fn [ref] 
-              [ref 
-               (try 
-                 (git/prompt-file ref) 
-                 (catch Throwable t 
-                   (logger/error t (format "missing ref %s" ref)) 
-                   :missing))]))
-       (filter (complement #(= :missing (second %))))
-       (map (fn [[ref f]]
-              (let [m (prompts/get-prompts (assoc opts :prompts f))]
+       (map (fn [{:keys [cached-path]}]
+              (let [m (prompts/get-prompts (assoc opts :prompts cached-path))]
                 [(or (-> m :metadata :name) ref) m])))
        (into {})))
 
@@ -34,22 +25,52 @@
   (-> db
       (assoc :mcp.prompts/registry (merge m (:mcp.prompts/static db)))))
 
-(defn add
-  "add any static prompts to db"
-  [opts]
-  (logger/info "adding static prompts" (:register opts))
-  (let [prompt-registry (get-prompt-data opts)]
-    (swap! db* add-static-prompts prompt-registry)))
-
-(comment
-  (add {:register ["github:docker/labs-ai-tools-for-devs?path=prompts/examples/explain_dockerfile.md"
-                   "github:docker/labs-ai-tools-for-devs?path=prompts/examples/hello_world.md"]}))
-
-(defn merge-dynamic-prompts [{:keys [registry-content] :as opts}]
-  (logger/info "adding dynamic prompts" registry-content)
-  (try
-    (let [{:keys [registry]} (yaml/parse-string registry-content)
-          prompt-registry (get-prompt-data (assoc opts :register (map :ref (vals registry))))]
-      (swap! db* add-dynamic-prompts prompt-registry))
-    (catch Throwable e
-      (logger/error e "could not merge dynamic prompts"))))
+(defn update-dynamic [coll]
+  (swap! db* add-dynamic-prompts (get-prompt-data {:register coll})))
+(defn update-static [coll]
+  (swap! db* add-static-prompts (get-prompt-data {:register coll})))
+
+(defn missing-cached-prompt-file? [m]
+  (when (not (contains? m :cached-path))
+    (logger/warn "missing cached path: %s" (:ref-string m))
+    m))
+
+(defn add-refs
+  "update the db with new refs
+     params
+       refs - coll of [type ref] type is static or dynamic"
+  [refs]
+  (if (seq refs)
+    (let [git-map-coll (->> refs
+                            (map (fn [[t ref]] {:type t :ref-string ref :ref (git/parse-github-ref ref)}))
+                            (map (fn [m] (assoc-in m [:ref :ref-hash] (git/hashch (:ref m)))))
+                            (map (fn [m] (assoc-in m [:ref :dir] (git/cache-dir (:ref m))))))]
+      (-> git-map-coll
+          git/collect-unique-cache-dirs
+          git/refresh-cache)
+      (let [typed-colls
+            (->> git-map-coll
+                 (map git/cached-prompt-file)
+                 (filter (complement missing-cached-prompt-file?))
+                 (sort-by :type)
+                 (partition-by :type))]
+        (doseq [coll typed-colls]
+          (case (-> coll first :type)
+            :dynamic (update-dynamic coll)
+            :static (update-static coll)
+            (logger/info "unknown type " (first coll))))))
+    ;; if registry.yaml is empty
+    (swap! db* (fn [db] (-> db
+                            (update 
+                              :mcp.prompts/registry 
+                              (constantly (or (:mcp.prompts/static db) {}))))))))
+
+(defn registry-refs 
+  "parse refs from the registry.yaml file - these are dynamic"
+  [f]
+  (->>
+   (yaml/parse-string (slurp f))
+   :registry
+   (vals)
+   (map (fn [{:keys [ref]}] [:dynamic ref]))))
+

src/jsonrpc/server.clj

@@ -1,31 +1,32 @@
 (ns jsonrpc.server
   (:refer-clojure :exclude [run!])
   (:require
-    [babashka.fs :as fs]
-    [cheshire.core :as json]
-    [clojure.core :as c]
-    [clojure.core.async :as async]
-    [clojure.pprint :as pprint]
-    [clojure.string :as string]
-    docker
-    git
-    graph
-    jsonrpc
-    [jsonrpc.db :as db]
-    [jsonrpc.logger :as logger]
-    [jsonrpc.producer :as producer]
-    [lsp4clj.coercer :as coercer]
-    [lsp4clj.io-chan :as io-chan]
-    [lsp4clj.io-server :refer [stdio-server]]
-    [lsp4clj.server :as lsp.server]
-    [promesa.core :as p]
-    shutdown
-    state
-    [taoensso.timbre :as timbre]
-    [taoensso.timbre.appenders.core :as appenders]
-    tools
-    user-loop
-    volumes)
+   [babashka.fs :as fs]
+   [cheshire.core :as json]
+   [clj-yaml.core :as yaml]
+   [clojure.core :as c]
+   [clojure.core.async :as async]
+   [clojure.pprint :as pprint]
+   [clojure.string :as string]
+   docker
+   git
+   graph
+   jsonrpc
+   [jsonrpc.db :as db]
+   [jsonrpc.logger :as logger]
+   [jsonrpc.producer :as producer]
+   [lsp4clj.coercer :as coercer]
+   [lsp4clj.io-chan :as io-chan]
+   [lsp4clj.io-server :refer [stdio-server]]
+   [lsp4clj.server :as lsp.server]
+   [promesa.core :as p]
+   shutdown
+   state
+   [taoensso.timbre :as timbre]
+   [taoensso.timbre.appenders.core :as appenders]
+   tools
+   user-loop
+   volumes)
   (:gen-class))
 
 (set! *warn-on-reflection* true)
@@ -183,9 +184,7 @@
       :is-error false})))
 
 (defmethod lsp.server/receive-request "docker/prompts/register" [_ {:keys [db* id]} params]
-  ;; supports only git refs
-  (lsp.server/discarding-stdout
-   (db/add (merge @db* params))))
+  (logger/info "docker/prompts/register"))
 
 (defmethod lsp.server/receive-request "docker/prompts/run"
   [_ {:keys [db* id] :as components} {:keys [thread-id] {:keys [file content uri]} :prompts :as params}]
@@ -289,6 +288,8 @@
   (publish-docker-notify [_ method params]
     (lsp.server/send-notification server method params)))
 
+(def registry "/prompts/registry.yaml")
+
 (defn run-server! [{:keys [trace-level] :or {trace-level "off"} :as opts}]
   (lsp.server/discarding-stdout
    (let [timbre-logger (->TimbreLogger)
@@ -314,14 +315,13 @@
                      :server server}]
      (swap! db* merge {:log-path log-path} (dissoc opts :in))
      ;; register static prompts
-     (when (:register opts)
-       (try
-         (db/add opts)
-         (catch Throwable t
-           (logger/error t))))
-     ;; register dynamic prompts
-     (when (fs/exists? (fs/file "/prompts/registry.yaml"))
-       (db/merge-dynamic-prompts (assoc opts :registry-content (slurp "/prompts/registry.yaml"))))
+     (db/add-refs
+       (concat
+         (->> (:register opts)
+              (map (fn [ref] [:static ref])))
+         ;; register dynamic prompts
+         (when (fs/exists? (fs/file registry))
+           (db/registry-refs registry))))
      ;; watch dynamic prompts in background
      (async/thread
        (let [{x :container}
@@ -334,7 +334,7 @@
                 (let [[_dir _event f] (string/split line #"\s+")]
                   (when (= f "registry.yaml")
                     (try
-                      (db/merge-dynamic-prompts (assoc opts :registry-content (slurp "/prompts/registry.yaml")))
+                      (db/add-refs (logger/trace (into [] (db/registry-refs registry))))
                       (producer/publish-tool-list-changed producer {})
                       (producer/publish-prompt-list-changed producer {})
                       (catch Throwable t

test/jsonrpc/db_t.clj

@@ -0,0 +1,37 @@
+(ns jsonrpc.db-t
+  (:require
+   [clojure.pprint :refer [pprint]]
+   [clojure.test :as t]
+   [jsonrpc.db :as db]))
+
+(def correct-refs
+  [[:dynamic
+    "github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/hello_world.md"]
+   [:dynamic
+    "github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/mcp-sqlite.md"]
+   [:dynamic
+    "github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/curl.md"]
+   [:dynamic
+    "github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/ffmpeg.md"]
+   [:dynamic
+    "github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/explain_dockerfile.md"]
+   [:dynamic
+    "github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/qrencode.md"]])
+
+(t/deftest registry-ref-tests
+  (t/testing "parse-registry"
+    (t/is (= correct-refs
+             (db/registry-refs "test/registry.yaml")))))
+
+(comment
+  (db/add-refs 
+    (db/registry-refs "test/registry.yaml"))
+
+  (db/add-refs
+    [[:dynamic "github:docker/labs-ai-tools-for-devs?path=prompts/examples/explain_dockerfile.md"]
+     [:dynamic "github:docker/labs-ai-tools-for-devs?path=prompts/examples/explain_dockerfile1.md"]
+     [:dynamic "github:docker/labs-ai-tools-for-devs?ref=branch&path=prompts/examples/explain_dockerfile1.md"]
+     [:dynamic "github:docker/labs-ai-tools-for-devs?ref=branch&path=prompts/examples/explain_dockerfile1.md"]]))
+
+(comment
+  (pprint @db/db*))

test/registry.yaml

@@ -0,0 +1,13 @@
+registry:
+  hello world:
+    ref: github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/hello_world.md
+  mcp-sqlite:
+    ref: github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/mcp-sqlite.md
+  curl:
+    ref: github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/curl.md
+  ffmpeg:
+    ref: github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/ffmpeg.md
+  explain dockerfiles:
+    ref: github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/explain_dockerfile.md
+  qr code:
+    ref: github:docker/labs-ai-tools-for-devs?ref=main&path=prompts/examples/qrencode.md