Merge branch 'karthink:master' into gh-change-token-management

Author: marcus-lundgren

NEWS

@@ -1,5 +1,18 @@
 # -*- mode: org; -*-
 
+* 0.9.9.3-pre
+
+** New features and UI changes
+
+- ~gptel-mcp-connect~ can now start MCP servers synchronously.  This
+  is useful for scripting purposes, when MCP tools need to be
+  available before performing other actions.  One common use is
+  starting MCP servers when applying a gptel preset.
+
+- "gitignored" files are omitted by default when adding files to gptel's
+  context.  This setting can be controlled via the user option
+  ~gptel-context-restrict-to-project-files~.
+
 * 0.9.9 2025-08-02
 
 ** Breaking changes

README.org

@@ -1380,6 +1380,8 @@ You can examine the active context from the menu:
 And then browse through or remove context from the context buffer:
 #+html: <img src="https://github.com/karthink/gptel/assets/8607532/79a5ffe8-3d63-4bf7-9bf6-0457ab61bf2a" align="center" alt="Image showing gptel's context buffer.">
 
+By default, files in a version control system that are not project files ("gitignored" files) will not be added to the context.  To be able to add these files, set =gptel-context-restrict-to-project-files= to =nil=.  Note that remote files are always included, regardless of the value of =gptel-context-restrict-to-project-files=.
+
 *** Handle "reasoning" content
 
 Some LLMs include in their response a "thinking" or "reasoning" block.  This text improves the quality of the LLM’s final output, but may not be interesting to you by itself.  You can decide how you would like this "reasoning" content to be handled by gptel by setting the user option =gptel-include-reasoning=.  You can include it in the LLM response (the default), omit it entirely, include it in the buffer but ignore it on subsequent conversation turns, or redirect it to another buffer.  As with most options, you can specify this behvaior from gptel's transient menu globally, buffer-locally or for the next request only.

gptel-context.el

@@ -27,9 +27,9 @@
 
 ;;; Code:
 
-;;; -*- lexical-binding: t -*-
 (require 'gptel)
 (require 'cl-lib)
+(require 'project)
 
 (declare-function gptel-menu "gptel-transient")
 (declare-function dired-get-marked-files "dired")
@@ -91,6 +91,23 @@ context chunk.  This is accessible as, for example:
   :group 'gptel
   :type 'function)
 
+(defcustom gptel-context-restrict-to-project-files t
+  "Restrict files eligible to be added to the context to project files.
+
+When set to t, files in a VCS that are not project files (such as files
+listed in `.gitignore' in a Git repository) will not be added to the
+context."
+  :group 'gptel
+  :type 'boolean)
+
+(defvar gptel-context--project-files nil
+  "Cached alist of project files per project.")
+
+(defvar gptel-context--reset-cache nil
+  "Whether a project files cache-buster has been scheduled.")
+
+;;; Commands
+
 (defun gptel-context-add-current-kill (&optional arg)
   "Add current-kill to gptel, accumulating if arg is non-nil"
   (interactive "P")
@@ -149,8 +166,9 @@ context chunk.  This is accessible as, for example:
 	(mapc action-fn files))))
    ;; If in an image buffer
    ((and (derived-mode-p 'image-mode)
-         (gptel--model-capable-p 'media)
-         (buffer-file-name))
+	 (gptel--model-capable-p 'media)
+	 (buffer-file-name)
+	 (not (gptel-context--skip-p (buffer-file-name))))
     (funcall (if (and arg (< (prefix-numeric-value arg) 0))
               #'gptel-context-remove
               #'gptel-context-add-file)
@@ -212,35 +230,66 @@ Return PATH if added, nil if ignored."
 (defun gptel-context--add-directory (path action)
   "Process all files in directory at PATH according to ACTION.
 ACTION should be either `add' or `remove'."
-  (let ((files (directory-files-recursively path "." t)))
-    (mapc (lambda (file)
-            (unless (file-directory-p file)
-              (pcase-exhaustive action
-                ('add
-                 (if (gptel--file-binary-p file)
-                     (gptel-context--add-binary-file file)
-                   (gptel-context--add-text-file file)))
-                ('remove
-                 (setf (alist-get file gptel-context--alist nil 'remove #'equal) nil)))))
-          files)
-    (when (eq action 'remove)
-      (message "Directory \"%s\" removed from context." path))))
+  (dolist (file (directory-files-recursively path "."))
+    (pcase-exhaustive action
+      ('add (gptel-context-add-file file))
+      ('remove
+       (setf (alist-get file gptel-context--alist nil 'remove #'equal) nil)))))
 
 (defun gptel-context-add-file (path)
   "Add the file at PATH to the gptel context.
 
-If PATH is a directory, recursively add all files in it.
-PATH should be readable as text."
+If PATH is a directory, recursively add all files in it.  PATH should be
+readable as text."
   (interactive "fChoose file to add to context: ")
+  (unless gptel-context--reset-cache
+    (setq gptel-context--reset-cache t)
+    (run-at-time
+     0 nil
+     (lambda () (setq gptel-context--reset-cache nil
+                 gptel-context--project-files nil))))
   (cond ((file-directory-p path)
-	 (gptel-context--add-directory path 'add))
+         (gptel-context--add-directory path 'add))
+        ((gptel-context--skip-p path)
+         ;; Don't message about .git, as this creates thousands of messages
+         (unless (string-match-p "\\.git/" path)
+           (gptel-context--message-skipped path)))
 	((gptel--file-binary-p path)
          (gptel-context--add-binary-file path))
-	((gptel-context--add-text-file path))))
+	(t (gptel-context--add-text-file path))))
 
 ;;;###autoload (autoload 'gptel-add-file "gptel-context" "Add files to gptel's context." t)
 (defalias 'gptel-add-file #'gptel-context-add-file)
 
+;;; project-related functions
+(defun gptel-context--get-project-files (dir)
+  "Return a list of files in the project DIR, or nil if no project is found."
+  (when-let* ((project (project-current nil dir)))
+    (with-memoization (alist-get dir gptel-context--project-files
+                                 nil nil #'equal)
+      (project-files project))))
+
+(defun gptel-context--skip-p (file)
+  "Return non-nil if FILE should not be added to the context."
+  (when (and gptel-context-restrict-to-project-files
+	     (not (file-remote-p file)))
+    (and-let* ((project (project-current nil file)))
+      (not (member (expand-file-name file)
+                   (gptel-context--get-project-files (project-root project)))))))
+
+(defun gptel-context--message-skipped (file)
+  "Message that FILE is skipped because it is not a project file."
+  (let* ((type (if (file-directory-p file) "directory" "file"))
+	 (reminder (format "To include it, unset `%S'."
+			   'gptel-context-restrict-to-project-files)))
+    (if-let* ((root (cl-some (lambda (dir) (and (file-in-directory-p file dir) dir))
+                             (map-keys gptel-context--project-files)))
+	      (rel-file (file-relative-name file root)))
+	(message "Skipping %s \"%s\" in project \"%s\".  %s"
+		 type rel-file root reminder)
+      (message "Skipping %s \"%s\". %s" type file reminder))))
+
+;;; Remove context
 (defun gptel-context-remove (&optional context)
   "Remove the CONTEXT overlay from the contexts list.
 
@@ -287,6 +336,7 @@ afterwards."
        finally do (setq gptel-context--alist nil)))
     (when verbose (message "Removed all gptel context sources."))))
 
+;;; Context wrap
 (defun gptel-context--make-overlay (start end &optional advance)
   "Highlight the region from START to END.
 

gptel-gh.el

@@ -159,7 +159,13 @@
      :context-window 1000
      :input-cost 0.10
      :output-cost 0.40
-     :cutoff-date "2024-08")))
+     :cutoff-date "2024-08")
+    (grok-code-fast-1
+     :description "Fast reasoning model for agentic coding"
+     :capabilities '(tool-use json reasoning)
+     :context-window 128
+     :input-cost 0.2
+     :output-cost 1.5)))
 
 (cl-defstruct (gptel--gh (:include gptel-openai)
                          (:copier nil)

gptel-org.el

@@ -424,11 +424,13 @@ parameters.
 ARGS are the original function call arguments."
   (if (derived-mode-p 'org-mode)
       (pcase-let ((`(,gptel--system-message ,gptel-backend ,gptel-model
-                     ,gptel-temperature ,gptel-max-tokens)
+                     ,gptel-temperature ,gptel-max-tokens
+                     ,gptel--num-messages-to-send ,gptel-tools)
                    (seq-mapn (lambda (a b) (or a b))
                              (gptel-org--entry-properties)
                              (list gptel--system-message gptel-backend gptel-model
-                                   gptel-temperature gptel-max-tokens))))
+                                   gptel-temperature gptel-max-tokens
+                                   gptel--num-messages-to-send gptel-tools))))
         (apply send-fun args))
     (apply send-fun args)))
 
@@ -445,12 +447,12 @@ ARGS are the original function call arguments."
 (defun gptel-org--entry-properties (&optional pt)
   "Find gptel configuration properties stored at PT."
   (pcase-let
-      ((`(,system ,backend ,model ,temperature ,tokens ,num)
+      ((`(,system ,backend ,model ,temperature ,tokens ,num ,tools)
          (mapcar
           (lambda (prop) (org-entry-get (or pt (point)) prop 'selective))
           '("GPTEL_SYSTEM" "GPTEL_BACKEND" "GPTEL_MODEL"
             "GPTEL_TEMPERATURE" "GPTEL_MAX_TOKENS"
-            "GPTEL_NUM_MESSAGES_TO_SEND"))))
+            "GPTEL_NUM_MESSAGES_TO_SEND" "GPTEL_TOOLS"))))
     (when system
       (setq system (string-replace "\\n" "\n" system)))
     (when backend
@@ -461,7 +463,16 @@ ARGS are the original function call arguments."
       (setq temperature (gptel--to-number temperature)))
     (when tokens (setq tokens (gptel--to-number tokens)))
     (when num (setq num (gptel--to-number num)))
-    (list system backend model temperature tokens num)))
+    (when tools
+      (setq tools (cl-loop
+                   for tname in (split-string tools)
+                   for tool = (with-demoted-errors "gptel: %S"
+                                (gptel-get-tool tname))
+                   if tool collect tool else do
+                   (display-warning
+                    '(gptel org tools)
+                    (format "Tool %s not found, ignoring" tname)))))
+    (list system backend model temperature tokens num tools)))
 
 (defun gptel-org--restore-state ()
   "Restore gptel state for Org buffers when turning on `gptel-mode'."
@@ -472,7 +483,7 @@ ARGS are the original function call arguments."
           (progn
             (when-let* ((bounds (org-entry-get (point-min) "GPTEL_BOUNDS")))
               (gptel--restore-props (read bounds)))
-            (pcase-let ((`(,system ,backend ,model ,temperature ,tokens ,num)
+            (pcase-let ((`(,system ,backend ,model ,temperature ,tokens ,num ,tools)
                          (gptel-org--entry-properties (point-min))))
               (when system (setq-local gptel--system-message system))
               (if backend (setq-local gptel-backend backend)
@@ -486,7 +497,8 @@ ARGS are the original function call arguments."
               (when model (setq-local gptel-model model))
               (when temperature (setq-local gptel-temperature temperature))
               (when tokens (setq-local gptel-max-tokens tokens))
-              (when num (setq-local gptel--num-messages-to-send num))))
+              (when num (setq-local gptel--num-messages-to-send num))
+              (when tools (setq-local gptel-tools tools))))
         (:success (message "gptel chat restored."))
         (error (message "Could not restore gptel state, sorry! Error: %s" status)))
       (set-buffer-modified-p modified))))
@@ -503,20 +515,27 @@ non-nil (default), display a message afterwards."
   (interactive (list (point) t))
   (org-entry-put pt "GPTEL_MODEL" (gptel--model-name gptel-model))
   (org-entry-put pt "GPTEL_BACKEND" (gptel-backend-name gptel-backend))
-  (unless (equal (default-value 'gptel-temperature) gptel-temperature)
-    (org-entry-put pt "GPTEL_TEMPERATURE"
-                   (number-to-string gptel-temperature)))
-  (when (natnump gptel--num-messages-to-send)
-    (org-entry-put pt "GPTEL_NUM_MESSAGES_TO_SEND"
-                   (number-to-string gptel--num-messages-to-send)))
-  (org-entry-put pt "GPTEL_SYSTEM"
+  (org-entry-put pt "GPTEL_SYSTEM"      ;TODO: Handle nil case correctly
                  (and-let* ((msg (car-safe
                                   (gptel--parse-directive
                                    gptel--system-message))))
                    (string-replace "\n" "\\n" msg)))
-  (when gptel-max-tokens
-    (org-entry-put
-     pt "GPTEL_MAX_TOKENS" (number-to-string gptel-max-tokens)))
+  (if gptel-tools
+      (org-entry-put
+       pt "GPTEL_TOOLS" (mapconcat #'gptel-tool-name gptel-tools " "))
+    (org-entry-delete pt "GPTEL_TOOLS"))
+  (if (equal (default-value 'gptel-temperature) gptel-temperature)
+      (org-entry-delete pt "GPTEL_TEMPERATURE")
+    (org-entry-put pt "GPTEL_TEMPERATURE"
+                   (number-to-string gptel-temperature)))
+  (if (natnump gptel--num-messages-to-send)
+      (org-entry-put pt "GPTEL_NUM_MESSAGES_TO_SEND"
+                     (number-to-string gptel--num-messages-to-send))
+    (org-entry-delete pt "GPTEL_NUM_MESSAGES_TO_SEND"))
+  (if gptel-max-tokens
+      (org-entry-put
+       pt "GPTEL_MAX_TOKENS" (number-to-string gptel-max-tokens))
+    (org-entry-delete pt "GPTEL_MAX_TOKENS"))
   (when msg
     (message "Added gptel configuration to current headline.")))
 

gptel.el

@@ -802,6 +802,13 @@ This opens up advanced options in `gptel-menu'.")
 (defvar gptel--num-messages-to-send nil)
 (put 'gptel--num-messages-to-send 'safe-local-variable #'always)
 
+(defvar-local gptel--tool-names nil
+  "Store to persist tool names to file across Emacs sessions.
+
+Note: Changing this variable does not affect gptel\\='s behavior
+in any way.")
+(put 'gptel--backend-name 'safe-local-variable #'always)
+
 (defcustom gptel-log-level nil
   "Logging level for gptel.
 
@@ -1429,7 +1436,17 @@ applied before being re-persisted in the new structure."
              "Could not activate gptel backend \"%s\"!  "
              "Switch backends with \\[universal-argument] \\[gptel-send]"
              " before using gptel."))
-           gptel--backend-name))))))
+           gptel--backend-name)))
+      (when gptel--tool-names
+        (if-let* ((tools (cl-loop
+                          for tname in gptel--tool-names
+                          for tool = (with-demoted-errors "gptel: %S"
+                                       (gptel-get-tool tname))
+                          if tool collect tool else do
+                          (display-warning
+                           '(gptel org tools)
+                           (format "Tool %s not found, ignoring" tname)))))
+            (setq-local gptel-tools tools))))))
 
 (defun gptel--save-state ()
   "Write the gptel state to the buffer.
@@ -1448,18 +1465,25 @@ file."
           (add-file-local-variable 'gptel-model gptel-model)
           (add-file-local-variable 'gptel--backend-name
                                    (gptel-backend-name gptel-backend))
-          (unless (equal (default-value 'gptel-temperature) gptel-temperature)
-            (add-file-local-variable 'gptel-temperature gptel-temperature))
           (unless (equal (default-value 'gptel--system-message)
                            gptel--system-message)
             (add-file-local-variable
-             'gptel--system-message
+             'gptel--system-message     ;TODO: Handle nil case correctly
              (car-safe (gptel--parse-directive gptel--system-message))))
-          (when gptel-max-tokens
-            (add-file-local-variable 'gptel-max-tokens gptel-max-tokens))
-          (when (natnump gptel--num-messages-to-send)
-            (add-file-local-variable 'gptel--num-messages-to-send
-                                     gptel--num-messages-to-send))
+          (if gptel-tools
+              (add-file-local-variable
+               'gptel--tool-names (mapcar #'gptel-tool-name gptel-tools))
+            (delete-file-local-variable 'gptel--tool-names))
+          (if (equal (default-value 'gptel-temperature) gptel-temperature)
+              (delete-file-local-variable 'gptel-temperature)
+            (add-file-local-variable 'gptel-temperature gptel-temperature))
+          (if gptel-max-tokens
+              (add-file-local-variable 'gptel-max-tokens gptel-max-tokens)
+            (delete-file-local-variable 'gptel-max-tokens))
+          (if (natnump gptel--num-messages-to-send)
+              (add-file-local-variable 'gptel--num-messages-to-send
+                                       gptel--num-messages-to-send)
+            (delete-file-local-variable 'gptel--num-messages-to-send))
           (add-file-local-variable 'gptel--bounds (gptel--get-buffer-bounds)))))))
 
 
@@ -1940,9 +1964,11 @@ The following keys are optional
 CATEGORY: A string indicating a category for the tool.  This is
 used only for grouping in gptel's UI.  Defaults to \"misc\".
 
-CONFIRM: Whether the tool call should wait for the user to run
-it.  If true, the user will be prompted with the proposed tool
-call, which can be examined, accepted, deferred or canceled.
+CONFIRM: Whether the tool call should wait for the user to run it.  If
+true, the user will be prompted with the proposed tool call, which can
+be examined, accepted, deferred or canceled.  It can also be a function
+that receives the same arguments as FUNCTION and returns true if the
+user should be prompted.
 
 INCLUDE: Whether the tool results should be included as part of
 the LLM output.  This is useful for logging and as context for
@@ -2420,8 +2446,10 @@ Run post-response hooks."
                           (plist-get args key)))
                       (gptel-tool-args tool-spec)))
                ;; Check if tool requires confirmation
-               (if (and gptel-confirm-tool-calls (or (eq gptel-confirm-tool-calls t)
-                                                     (gptel-tool-confirm tool-spec)))
+               (if (and gptel-confirm-tool-calls
+                        (or (eq gptel-confirm-tool-calls t) ;always confirm, or
+                            (and-let* ((confirm (gptel-tool-confirm tool-spec)))
+                              (or (not (functionp confirm)) (apply confirm arg-values)))))
                    (push (list tool-spec arg-values process-tool-result)
                          pending-calls)
                  ;; If not, run the tool