FEATURE: implement thinking token support (#1155)

adds support for "thinking tokens" - a feature that exposes the model's reasoning process before providing the final response. Key improvements include:

- Add a new Thinking class to handle thinking content from LLMs
- Modify endpoints (Claude, AWS Bedrock) to handle thinking output
- Update AI bot to display thinking in collapsible details section
- Fix SEARCH/REPLACE blocks to support empty replacement strings and general improvements to artifact editing
- Allow configurable temperature in triage and report automations
- Various bug fixes and improvements to diff parsing

Author: SamSaffron

assets/javascripts/discourse/connectors/composer-fields/persona-llm-selector.gjs

@@ -168,12 +168,14 @@ export default class BotSelector extends Component {
       .filter((bot) => !bot.is_persona)
       .filter(Boolean);
 
-    return availableBots.map((bot) => {
-      return {
-        id: bot.id,
-        name: bot.display_name,
-      };
-    });
+    return availableBots
+      .map((bot) => {
+        return {
+          id: bot.id,
+          name: bot.display_name,
+        };
+      })
+      .sort((a, b) => a.name.localeCompare(b.name));
   }
 
   <template>

config/locales/client.en.yml

@@ -85,10 +85,10 @@ en:
               description: "Prioritize content from this group in the report"
             temperature:
               label: "Temperature"
-              description: "Temperature to use for the LLM. Increase to increase randomness (0 to use model default)"
+              description: "Temperature to use for the LLM. Increase to increase randomness (leave empty to use model default)"
             top_p:
               label: "Top P"
-              description: "Top P to use for the LLM, increase to increase randomness (0 to use model default)"
+              description: "Top P to use for the LLM, increase to increase randomness (leave empty to use model default)"
 
         llm_triage:
           fields:
@@ -131,6 +131,9 @@ en:
             model:
               label: "Model"
               description: "Language model used for triage"
+            temperature:
+              label: "Temperature"
+              description: "Temperature to use for the LLM. Increase to increase randomness (leave empty to use model default)"
 
     discourse_ai:
       title: "AI"
@@ -403,7 +406,7 @@ en:
           open_ai-o1: "Open AI's most capable reasoning model"
           open_ai-o3-mini: "Advanced Cost-efficient reasoning model"
           samba_nova-Meta-Llama-3-1-8B-Instruct: "Efficient lightweight multilingual model"
-          samba_nova-Meta-Llama-3-1-70B-Instruct": "Powerful multipurpose model"
+          samba_nova-Meta-Llama-3-3-70B-Instruct": "Powerful multipurpose model"
           mistral-mistral-large-latest: "Mistral's most powerful model"
           mistral-pixtral-large-latest: "Mistral's most powerful vision capable model"
 

config/locales/server.en.yml

@@ -261,6 +261,7 @@ en:
     ai_bot:
       reply_error: "Sorry, it looks like our system encountered an unexpected issue while trying to reply.\n\n[details='Error details']\n%{details}\n[/details]"
       default_pm_prefix: "[Untitled AI bot PM]"
+      thinking: "Thinking..."
       personas:
         default_llm_required: "Default LLM model is required prior to enabling Chat"
         cannot_delete_system_persona: "System personas cannot be deleted, please disable it instead"

discourse_automation/llm_report.rb

@@ -37,8 +37,8 @@ module DiscourseAutomation::LlmReport
 
     field :allow_secure_categories, component: :boolean
 
-    field :top_p, component: :text, required: true, default_value: 0.1
-    field :temperature, component: :text, required: true, default_value: 0.2
+    field :top_p, component: :text
+    field :temperature, component: :text
 
     field :suppress_notifications, component: :boolean
     field :debug_mode, component: :boolean
@@ -64,12 +64,19 @@ module DiscourseAutomation::LlmReport
         exclude_category_ids = fields.dig("exclude_categories", "value")
         exclude_tags = fields.dig("exclude_tags", "value")
 
-        # set defaults in code to support easy migration for old rules
-        top_p = 0.1
-        top_p = fields.dig("top_p", "value").to_f if fields.dig("top_p", "value")
+        top_p = fields.dig("top_p", "value")
+        if top_p == "" || top_p.nil?
+          top_p = nil
+        else
+          top_p = top_p.to_f
+        end
 
-        temperature = 0.2
-        temperature = fields.dig("temperature", "value").to_f if fields.dig("temperature", "value")
+        temperature = fields.dig("temperature", "value")
+        if temperature == "" || temperature.nil?
+          temperature = nil
+        else
+          temperature = temperature.to_f
+        end
 
         suppress_notifications = !!fields.dig("suppress_notifications", "value")
         DiscourseAi::Automation::ReportRunner.run!(

discourse_automation/llm_triage.rb

@@ -24,6 +24,7 @@
     field :hide_topic, component: :boolean
     field :flag_post, component: :boolean
     field :include_personal_messages, component: :boolean
+    field :temperature, component: :text
     field :flag_type,
           component: :choices,
           required: false,
@@ -53,6 +54,12 @@
       flag_post = fields.dig("flag_post", "value")
       flag_type = fields.dig("flag_type", "value")
       max_post_tokens = fields.dig("max_post_tokens", "value").to_i
+      temperature = fields.dig("temperature", "value")
+      if temperature == "" || temperature.nil?
+        temperature = nil
+      else
+        temperature = temperature.to_f
+      end
 
       max_post_tokens = nil if max_post_tokens <= 0
 
@@ -93,6 +100,7 @@
           max_post_tokens: max_post_tokens,
           stop_sequences: stop_sequences,
           automation: self.automation,
+          temperature: temperature,
         )
       rescue => e
         Discourse.warn_exception(e, message: "llm_triage: skipped triage on post #{post.id}")

lib/ai_bot/artifact_update_strategies/diff.rb

@@ -90,15 +90,45 @@ def apply_changes(changes)
 
         def extract_search_replace_blocks(content)
           return nil if content.blank? || content.to_s.strip.downcase.match?(/^\(?no changes?\)?$/m)
-          return [{ replace: content }] if !content.match?(/<<+\s*SEARCH/)
+          return [{ replace: content }] if !content.include?("<<< SEARCH")
 
           blocks = []
-          remaining = content
+          current_block = {}
+          state = :initial
+          search_lines = []
+          replace_lines = []
+
+          content.each_line do |line|
+            line = line.chomp
+
+            case state
+            when :initial
+              state = :collecting_search if line.match?(/^<<<* SEARCH/)
+            when :collecting_search
+              if line.start_with?("===")
+                current_block[:search] = search_lines.join("\n").strip
+                search_lines = []
+                state = :collecting_replace
+              else
+                search_lines << line
+              end
+            when :collecting_replace
+              if line.match?(/>>>* REPLACE/)
+                current_block[:replace] = replace_lines.join("\n").strip
+                replace_lines = []
+                blocks << current_block
+                current_block = {}
+                state = :initial
+              else
+                replace_lines << line
+              end
+            end
+          end
 
-          pattern = /<<+\s*SEARCH\s*\n(.*?)\n=+\s*\n(.*?)\n>>+\s*REPLACE/m
-          while remaining =~ pattern
-            blocks << { search: $1.strip, replace: $2.strip }
-            remaining = $'
+          # Handle any remaining block
+          if state == :collecting_replace && !replace_lines.empty?
+            current_block[:replace] = replace_lines.join("\n").strip
+            blocks << current_block
           end
 
           blocks.empty? ? nil : blocks
@@ -108,26 +138,50 @@ def system_prompt
           <<~PROMPT
             You are a web development expert generating precise search/replace changes for updating HTML, CSS, and JavaScript code.
 
-            Important rules:
+            CRITICAL RULES:
 
             1. Use EXACTLY this format for changes:
                <<<<<<< SEARCH
-               (first line of code to replace)
-               (other lines of code to avoid ambiguity)
-               (last line of code to replace)
+               (code to replace)
                =======
                (replacement code)
                >>>>>>> REPLACE
-            2. DO NOT modify the markers or add spaces around them
-            3. DO NOT add explanations or comments within sections
-            4. ONLY include [HTML], [CSS], and [JavaScript] sections if they have changes
-            5. HTML should not include <html>, <head>, or <body> tags, it is injected into a template
-            6. When specifying a SEARCH block, ALWAYS keep it 8 lines or less, you will be interrupted and a retry will be required if you exceed this limit
-            7. NEVER EVER ask followup questions, ALL changes must be performed in a single response, you are consumed via an API, there is no opportunity for humans in the loop
-            8. When performing a non-contiguous search, ALWAYS use ... to denote the skipped lines
-            9. Be mindful that ... non-contiguous search is not greedy, the following line will only match the first occurrence of the search block
-            10. Never mix a full section replacement with a search/replace block in the same section
-            11. ALWAYS skip sections you to not want to change, do not include them in the response
+
+            2. SEARCH blocks MUST be 8 lines or less. Break larger changes into multiple smaller search/replace blocks.
+
+            3. DO NOT modify the markers or add spaces around them.
+
+            4. DO NOT add explanations or comments within sections.
+
+            5. ONLY include [HTML], [CSS], and [JavaScript] sections if they have changes.
+
+            6. HTML should not include <html>, <head>, or <body> tags, it is injected into a template.
+
+            7. NEVER EVER ask followup questions, ALL changes must be performed in a single response.
+
+            8. When performing a non-contiguous search, ALWAYS use ... to denote the skipped lines.
+
+            9. Be mindful that ... non-contiguous search is not greedy, it will only match the first occurrence.
+
+            10. Never mix a full section replacement with a search/replace block in the same section.
+
+            11. ALWAYS skip sections you do not want to change, do not include them in the response.
+
+            HANDLING LARGE CHANGES:
+
+            - Break large HTML structures into multiple smaller search/replace blocks.
+            - Use strategic anchor points like unique IDs or class names to target specific elements.
+            - Consider replacing entire components rather than modifying complex internals.
+            - When elements contain dynamic content, use precise context markers or replace entire containers.
+
+            VALIDATION CHECKLIST:
+            - Each SEARCH block is 8 lines or less
+            - Every SEARCH has exactly one matching REPLACE
+            - All blocks are properly closed
+            - No SEARCH/REPLACE blocks are nested
+            - Each change is a complete, separate block with its own SEARCH/REPLACE markers
+
+            WARNING: Never nest search/replace blocks. Each change must be a complete sequence.
 
             JavaScript libraries must be sourced from the following CDNs, otherwise CSP will reject it:
             #{AiArtifact::ALLOWED_CDN_SOURCES.join("\n")}
@@ -143,7 +197,7 @@ def system_prompt
             (changes or empty if no changes or entire JavaScript)
             [/JavaScript]
 
-            Example - Multiple changes in one file:
+            EXAMPLE 1 - Multiple small changes in one file:
 
             [JavaScript]
             <<<<<<< SEARCH
@@ -158,39 +212,35 @@ def system_prompt
             >>>>>>> REPLACE
             [/JavaScript]
 
-            Example - CSS with multiple blocks:
+            EXAMPLE 2 - Breaking up large HTML changes:
 
-            [CSS]
+            [HTML]
             <<<<<<< SEARCH
-            .button { color: blue; }
+            <div class="header">
+              <div class="logo">
+                <img src="old-logo.png">
+              </div>
             =======
-            .button { color: red; }
+            <div class="header">
+              <div class="logo">
+                <img src="new-logo.png">
+              </div>
             >>>>>>> REPLACE
+
             <<<<<<< SEARCH
-            .text { font-size: 12px; }
+              <div class="navigation">
+                <ul>
+                  <li>Home</li>
+                  <li>Products</li>
             =======
-            .text { font-size: 16px; }
+              <div class="navigation">
+                <ul>
+                  <li>Home</li>
+                  <li>Services</li>
             >>>>>>> REPLACE
-            [/CSS]
-
-            Example - Non contiguous search in CSS (replace most CSS with new CSS)
+            [/HTML]
 
-            Original CSS:
-
-            [CSS]
-            body {
-              color: red;
-            }
-            .button {
-              color: blue;
-            }
-            .alert {
-              background-color: green;
-            }
-            .alert2 {
-              background-color: green;
-            }
-            [/CSS]
+            EXAMPLE 3 - Non-contiguous search in CSS:
 
             [CSS]
             <<<<<<< SEARCH
@@ -203,37 +253,20 @@ def system_prompt
               color: red;
             }
             >>>>>>> REPLACE
-
-            RESULT:
-
-            [CSS]
-            body {
-              color: red;
-            }
-            .alert2 {
-              background-color: green;
-            }
             [/CSS]
 
-            Example - full HTML replacement:
+            EXAMPLE 4 - Full HTML replacement:
 
             [HTML]
             <div>something old</div>
-            <div>another somethin old</div>
+            <div>another something old</div>
             [/HTML]
 
             output:
 
             [HTML]
             <div>something new</div>
             [/HTML]
-
-            result:
-            [HTML]
-            <div>something new</div>
-            [/HTML]
-
-
           PROMPT
         end