Resource Streaming (#74)

* Switch mcp/gdrive over to vonwig while we different auth
* Add function call that does not expect exit but still waits for one
message
* Infra for resource streams is in place

Author: slimslenderslacks

dev/catalog.clj

@@ -6,14 +6,15 @@
    [flatland.ordered.map :refer [ordered-map]]
    git
    [markdown :as markdown-parser]
+   [mcp.client :as client]
    [medley.core :as medley]
    prompts
    repl))
 
 (defn mcp-metadata-cache [f]
   (edn/read-string
-    {:readers {'ordered/map (fn [pairs] (into (ordered-map pairs)))}}
-    (slurp f)))
+   {:readers {'ordered/map (fn [pairs] (into (ordered-map pairs)))}}
+   (slurp f)))
 
 (comment
   (mcp-metadata-cache "test/resources/mcp-metadata-cache.edn"))
@@ -66,10 +67,28 @@
          :registry
          vals
          (map :ref)
-         (map git/prompt-file)))
+         (map #(conj [%] (git/prompt-file %)))
+         (into {})))
 
   ;; parse all of the current git prompts
-  (map f->prompt local-prompt-files)
+  (with-redefs [#'client/get-mcp-tools-from-prompt (constantly [])]
+    (def all-prompt-files (map (fn [[k v]] [k (f->prompt v)]) local-prompt-files)))
 
-  ;;
+  (def all-images
+    (->>
+      (concat
+        (->> (into {} all-prompt-files)
+             (vals)
+             (mapcat (comp :mcp :metadata))
+             (map (comp :image :container))
+             (into #{}))
+        (->> (into {} all-prompt-files)
+             (vals)
+             (mapcat :functions)
+             (map (comp :image :container :function))
+             (into #{})))
+      (into #{})
+      (sort)))
+
+;;
   (markdown-parser/parse-markdown (slurp "prompts/examples/sequentialthinking.md")))

dev/stdio_client.clj

@@ -0,0 +1,76 @@
+(ns stdio-client
+  (:require
+   [babashka.process :as process]
+   [cheshire.core :as json]
+   [clojure.core.async :as async]
+   [clojure.java.io :as io])
+  (:import
+   [java.io BufferedOutputStream]))
+
+(defn write [writer m]
+  (.write writer (json/generate-string m))
+  (.write writer "\n")
+  (.flush writer))
+
+(do
+  (def server (process/process
+                {:out :stream :in :stream}
+                "docker" "run" "-i" "--rm" "--workdir=/app"
+                "-v" "mcp-gdrive:/gdrive-server"
+                "-e" "GDRIVE_CREDENTIALS_PATH=/gdrive-server/credentials.json"
+                "-e" "GDRIVE_OAUTH_PATH=/secret/google.gcp-oauth.keys.json"
+                "--label" "x-secret:google.gcp-oauth.keys.json=/secret/google.gcp-oauth.keys.json"
+                "vonwig/gdrive:latest"))
+
+  (async/thread
+    (loop []
+      (let [line (.readLine (io/reader (:out server)))]
+        (when line
+          (println "stdout: " line)
+          (recur)))))
+
+  (async/thread
+    (loop []
+      (let [line (.readLine (io/reader (:err server)))]
+        (when line
+          (println "stderr: " line)
+          (recur)))))
+
+  (def writer (io/writer (:in server)))
+
+  (write writer
+         {:jsonrpc "2.0"
+          :method "initialize"
+          :id 0
+          :params {:protocolVersion "2024-11-05"
+                   :capabilities {}
+                   :clientInfo {:name "Stdio Client" :version "0.1"}}})
+  (write writer
+         {:jsonrpc "2.0" :method "notifications/initialized" :params {}}))
+
+
+(write writer
+       {:jsonrpc "2.0" :method "resources/list" :params {} :id 1})
+
+(write writer
+       {:jsonrpc "2.0" 
+        :method "tools/call" 
+        :params 
+        {:name "search"
+         :arguments {:query "mcp"}} 
+        :id "4"})
+
+(async/thread
+  (loop []
+    (let [line (.readLine (io/reader (:out server)))]
+      (when line
+        (println "stdout: " line)
+        (recur)))))
+
+(async/thread
+  (loop []
+    (let [line (.readLine (io/reader (:err server)))]
+      (when line
+        (println "stderr: " line)
+        (recur)))))
+

prompts/catalog.yaml

@@ -264,7 +264,7 @@ registry:
     description: |
       UNDER CONSTRUCTION (support long running browser cache)
       Browser automation and web scraping using Puppeteer.
-    ref: https://img.icons8.com/officel/80/under-construction.png
+    ref: github:docker/labs-ai-tools-for-devs?path=prompts/mcp/puppeteer.md
     icon: https://img.icons8.com/officel/80/under-construction.png
     tools: []
     prompts: 0

prompts/mcp/atlassian.md

@@ -9,7 +9,7 @@ mcp:
         JIRA_URL: "{{atlassian.jira.url}}"
         JIRA_USERNAME: "{{atlassian.jira.username}}"
       secrets:
-        JIRA_PERSONAL_TOKEN: atlassian.jira.token
+        atlassian.jira.token: JIRA_PERSONAL_TOKEN
 ---
 
 # Configuration

prompts/mcp/gdrive.md

@@ -3,8 +3,8 @@ tools:
   - name: gdrive_auth
     description: Authorize this server to use your Google Drive.
     container:
-      image: mcp/gdrive:latest
-      background: true
+      image: vonwig/gdrive:latest
+      background-callback: true
       workdir: /app
       volumes:
         - "mcp-gdrive:/gdrive-server"
@@ -25,5 +25,70 @@ mcp:
         - "mcp-gdrive:/gdrive-server"
       environment:
         GDRIVE_CREDENTIALS_PATH: /gdrive-server/credentials.json
+        GDRIVE_OAUTH_PATH: /secret/google.gcp-oauth.keys.json
+      secrets:
+        google.gcp-oauth.keys.json: GDRIVE
 ---
 
+# Configuration
+
+Before you can use this server, users will need to add one secret named `google.gcp-oauth.keys.json`.  The value
+to copy into this secret is the _content_ of the file that you download from Google when you create a new Google
+OAuth application.  The steps to create a new OAuth client are below.
+
+## Detailed Google Cloud Setup
+
+### Create a Google Cloud Project
+* Visit the Google Cloud Console
+* Click "New Project"
+* Enter a project name (e.g., "MCP GDrive Server")
+* Click "Create"
+* Wait for the project to be created and select it
+
+### Enable the Google Drive API
+* Go to the API Library
+* Search for "Google Drive API"
+* Click on "Google Drive API"
+* Click "Enable"
+* Wait for the API to be enabled
+
+### Configure OAuth Consent Screen
+
+* Navigate to OAuth consent screen
+* Select User Type:
+    * "Internal" if you're using Google Workspace
+    * "External" for personal Google accounts
+* Click "Create"
+* Fill in the required fields:
+    * App name: "MCP GDrive Server"
+    * User support email: your email
+    * Developer contact email: your email
+* Click "Save and Continue"
+* On the "Scopes" page:
+    * Click "Add or Remove Scopes"
+    * Add https://www.googleapis.com/auth/drive.readonly
+    * Click "Update"
+* Click "Save and Continue"
+* Review the summary and click "Back to Dashboard"
+
+### Create OAuth Client ID
+* Go to Credentials
+* Click "Create Credentials" at the top
+* Select "OAuth client ID"
+* Choose Application type: "Desktop app"
+    * Name: "MCP GDrive Server Desktop Client"
+    * Click "Create"
+* Add one Authorized redirect URI which should be `http://localhost:3000/oauth2callback`
+* In the popup:
+    * Click "Download JSON"
+    * Save the file
+    * Click "OK"
+
+# Usage
+
+After enabling this server, refresh your MCP client and ask the agent to "Authorize your server for Google Drive", or
+something to that effect. This will give you with a URL that you can use to authorize the MCP server to access
+Google Drive readonly.
+
+Once you've completed the authorization flow, try searching for some Google drive files.
+

runbook.md

@@ -20,7 +20,7 @@ docker pull mcp/docker:prerelease
 
 ```sh
 # docker:command=build-release
-VERSION="0.0.8"
+VERSION="0.0.9"
 docker buildx build \
     --builder hydrobuild \
     --platform linux/amd64,linux/arm64 \

src/docker.clj

@@ -9,6 +9,7 @@
    jsonrpc
    [jsonrpc.logger :as logger]
    logging
+   repl
    schema
    shutdown)
   (:import
@@ -339,8 +340,7 @@
   (injected-entrypoint {:a "A"} ["BLAH=whatever"] "my command")
   (injected-entrypoint nil nil "my command")
   (injected-entrypoint {:a "A"} nil "my command")
-  (injected-entrypoint nil nil nil)
-  )
+  (injected-entrypoint nil nil nil))
 
 (defn inject-secret-transform [container-definition]
   (check-then-pull container-definition)
@@ -350,18 +350,18 @@
           (-> (images {"reference" [(:image container-definition)]})
               first))
          :Config)
-        real-entrypoint (string/join " " (concat 
-                                           (or (:entrypoint container-definition) Entrypoint) 
-                                           (or (:command container-definition) Cmd)))]
+        real-entrypoint (string/join " " (concat
+                                          (or (:entrypoint container-definition) Entrypoint)
+                                          (or (:command container-definition) Cmd)))]
     (-> container-definition
-        (assoc :entrypoint ["/bin/sh" "-c" (injected-entrypoint 
-                                             (:secrets container-definition) 
-                                             (concat
-                                               Env
-                                               (->> (:environment container-definition)
-                                                    (map (fn [[k v]] (format "%s=%s" k v)))
-                                                    (into []))) 
-                                             real-entrypoint)])
+        (assoc :entrypoint ["/bin/sh" "-c" (injected-entrypoint
+                                            (:secrets container-definition)
+                                            (concat
+                                             Env
+                                             (->> (:environment container-definition)
+                                                  (map (fn [[k v]] (format "%s=%s" (if (keyword? k) (name k) k) v)))
+                                                  (into [])))
+                                            real-entrypoint)])
         (dissoc :command))))
 
 (defn run-streaming-function-with-no-stdin
@@ -604,6 +604,46 @@
     (start x)
     (assoc x :socket (write-stdin (:Id x) (:content m)))))
 
+(defn function-call-with-attached-socket
+  "create and start a container with an attached socket
+   this does not wait for the container to finish
+    returns a map with an 
+      :output-channel for stdout/stderr/:stopped/:timeout/:closed messages
+      :write function to write to the stdin of the container"
+  [container]
+  (check-then-pull container)
+  (let [x (docker/create (assoc container
+                                :opts {:StdinOnce true
+                                       :OpenStdin true
+                                       :AttachStding true}))
+        socket-channel (attach-socket (:Id x))
+        c (async/chan)
+        output-channel (async/chan)]
+    (start x)
+    (async/thread (read-loop socket-channel c))
+    (async/go
+      (docker/wait x)
+      (async/>! c :stopped)
+      #_(delete x))
+    (async/go-loop
+     [block (async/<! c)]
+      (logger/info "background socket read loop " block)
+      (cond
+        (#{:stopped :timeout} block)
+        (do
+          (logger/info "socket read loop " block)
+          (async/put! output-channel block))
+        (nil? block)
+        (async/put! output-channel :closed)
+        :else
+        (do
+          (async/put! output-channel block)
+          (recur (async/<! c)))))
+    (assoc
+     x
+     :output-channel output-channel
+     :write (fn [s] (write-to-stdin socket-channel s)))))
+
 (defn finish-call
   "This is a blocking call that waits for the container to finish and then returns the output and exit code."
   [{:keys [timeout] :or {timeout 10000} :as x}]
@@ -638,6 +678,24 @@
         (delete x)
         {}))))
 
+(defn run-in-background-with-one-message [m]
+  (let [{:keys [output-channel]} (function-call-with-attached-socket m)]
+    (async/go-loop
+     []
+      (let [m (async/<! output-channel)]
+        (cond
+          (#{:timeout} m)
+          {:done :timeout :timeout "timeout waiting for message"}
+          (#{:stopped :closed} m)
+          {:done :exited :pty-output "tool stopped before responding on stdout" :exit-code 1}
+          (and (map? m) (contains? m :stdout))
+          {:pty-output (:stdout m)
+           :done :running}
+          :else
+          (do
+            (logger/info "background stderr " m)
+            (recur)))))))
+
 (defn run-with-stdin-content
   "run container with stdin read from file or from string
    this is several engine calls
@@ -665,9 +723,24 @@
     (run-with-stdin-content m)
     (true? (:background m))
     (run-background-function m)
+    (true? (:background-callback m))
+    (async/<!! (run-in-background-with-one-message m))
     :else
     (run-function m)))
 
+(comment
+  (repl/setup-stdout-logger)
+  (run-container
+   {:image "vonwig/gdrive:latest"
+    :background-callback true
+    :workdir "/app"
+    :ports ["3000:3000"]
+    :volumes ["mcp-gdrive:/gdrive-server"]
+    :environment {"GDRIVE_CREDENTIALS_PATH" "/gdrive-server/credentials.json"
+                  "GDRIVE_OAUTH_PATH" "/secret/google.gcp-oauth.keys.json"}
+    :secrets {:google.gcp-oauth.keys.json "GDRIVE"}
+    :command ["auth"]}))
+
 (defn get-login-info-from-desktop-backend
   "returns token or nil if not logged in or backend.sock is not available"
   []

src/extension/Makefile

@@ -1,5 +1,5 @@
 IMAGE?=docker/labs-ai-tools-for-devs
-TAG?=0.2.27
+TAG?=0.2.29
 
 BUILDER=buildx-multi-arch
 

src/extension/docker-compose.yaml

@@ -1,6 +1,6 @@
 services:
   mcp_docker:
-    image: mcp/docker:0.0.8
+    image: mcp/docker:0.0.9
     ports:
       - 8811:8811
     volumes: