Improve Anthropic prompt caching with static/dynamic split

Split system prompt into static (agent prompt, rules, skills, stable
contexts) and dynamic (cursor, MCP resources, MCP instructions) blocks
with separate cache_control markers. Static instructions are memoized
per chat in db, so only the small dynamic block is recomputed each turn.

- prompt.clj: split build-chat-instructions into build-static/dynamic,
  return {:static :dynamic} map, volatile contexts (cursor, mcpResource)
  go to dynamic block
- anthropic.clj: 3-block system prompt (identity, static cached, dynamic
  cached), add cache_control to last tool via add-cache-to-last-tool
- llm_api.clj: flatten instructions map for non-Anthropic providers
- chat.clj: cache static instructions in [:chats chat-id :prompt-cache]

🤖 Generated with [eca](https://eca.dev)

Co-Authored-By: eca <[email protected]>

Author: ericdallo

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - Fix exceptions on openai responses models when creating tasks.
 - Fix potential infinite auto-compact loop when context overflow persists after compaction. #391
+- Improve Anthropic prompt caching: split system prompt into static/dynamic blocks, add cache markers to the tools array, and memoize static instructions per chat.
 
 ## 0.123.2
 

src/eca/features/chat.clj

@@ -923,15 +923,15 @@
                           (f.context/agents-file-contexts db)
                           (f.context/raw-contexts->refined contexts db))
         repo-map* (delay (f.index/repo-map db config {:as-string? true}))
-        instructions (f.prompt/build-chat-instructions refined-contexts
-                                                       rules
-                                                       skills
-                                                       repo-map*
-                                                       agent
-                                                       config
-                                                       chat-id
-                                                       all-tools
-                                                       db)
+        cached-static (get-in db [:chats chat-id :prompt-cache :static])
+        instructions (if cached-static
+                       {:static cached-static
+                        :dynamic (f.prompt/build-dynamic-instructions refined-contexts db)}
+                       (let [result (f.prompt/build-chat-instructions
+                                     refined-contexts rules skills repo-map*
+                                     agent config chat-id all-tools db)]
+                         (swap! db* assoc-in [:chats chat-id :prompt-cache :static] (:static result))
+                         result))
         image-contents (->> refined-contexts
                             (filter #(= :image (:type %))))
         expanded-prompt-contexts (when-let [contexts-str (some-> (f.context/contexts-str-from-prompt message db)

src/eca/features/prompt.clj

@@ -126,8 +126,17 @@
        "</mcp-server-instructions>"
        ""))))
 
-(defn build-chat-instructions [refined-contexts rules skills repo-map* agent-name config chat-id all-tools db]
-  (let [selmer-ctx (->base-selmer-ctx all-tools chat-id db)]
+(def ^:private volatile-context-types
+  "Context types that change between turns and belong in the dynamic prompt block."
+  #{:cursor :mcpResource})
+
+(defn build-static-instructions
+  "Builds the stable portion of the system prompt: agent prompt, rules, skills,
+   stable contexts, and additional system info. Stable within a session — callers
+   should cache the result in [:chats chat-id :prompt-cache :static]."
+  [refined-contexts rules skills repo-map* agent-name config chat-id all-tools db]
+  (let [selmer-ctx (->base-selmer-ctx all-tools chat-id db)
+        stable-contexts (remove #(volatile-context-types (:type %)) refined-contexts)]
     (multi-str
      (selmer/render (eca-chat-prompt agent-name config) selmer-ctx)
      (when (seq rules)
@@ -152,14 +161,43 @@
          skills)
         "</skills>"
         ""])
-     (when (seq refined-contexts)
+     (when (seq stable-contexts)
        ["## Contexts"
         ""
-        (contexts-str refined-contexts repo-map* (get-in db [:chats chat-id :startup-context]))])
-     (mcp-instructions-section db)
+        (contexts-str stable-contexts repo-map* (get-in db [:chats chat-id :startup-context]))])
      ""
      (selmer/render (load-builtin-prompt "additional_system_info.md") selmer-ctx))))
 
+(defn build-dynamic-instructions
+  "Builds the volatile portion of the system prompt: cursor/MCP resource contexts
+   and MCP server instructions. Recomputed every turn. Returns nil when empty."
+  [refined-contexts db]
+  (let [volatile-contexts (filter #(volatile-context-types (:type %)) refined-contexts)
+        result (multi-str
+                (when (seq volatile-contexts)
+                  (contexts-str volatile-contexts nil nil))
+                (mcp-instructions-section db))]
+    (when-not (string/blank? result) result)))
+
+(defn build-chat-instructions
+  "Returns {:static \"...\" :dynamic \"...\"}.
+   Static content (agent prompt, rules, skills, stable contexts) is stable within a session.
+   Dynamic content (cursor, MCP resources, MCP instructions) is recomputed every turn.
+   Callers should cache :static in [:chats chat-id :prompt-cache :static] for
+   Anthropic API cache prefix stability across turns."
+  [refined-contexts rules skills repo-map* agent-name config chat-id all-tools db]
+  {:static (build-static-instructions refined-contexts rules skills repo-map*
+                                      agent-name config chat-id all-tools db)
+   :dynamic (build-dynamic-instructions refined-contexts db)})
+
+(defn instructions->str
+  "Flattens a {:static :dynamic} instructions map into a single string.
+   Used by non-Anthropic providers that don't support split system prompt blocks."
+  [{:keys [static dynamic]}]
+  (if dynamic
+    (multi-str static dynamic)
+    static))
+
 (defn build-rewrite-instructions [text path full-text range all-tools config db]
   (let [legacy-prompt-file (-> config :rewrite :systemPromptFile)
         legacy-config-prompt (-> config :rewrite :systemPrompt)

src/eca/llm_api.clj

@@ -3,6 +3,7 @@
    [babashka.fs :as fs]
    [clojure.string :as string]
    [eca.config :as config]
+   [eca.features.prompt :as f.prompt]
    [eca.llm-providers.anthropic :as llm-providers.anthropic]
    [eca.llm-providers.azure]
    [eca.llm-providers.copilot]
@@ -185,6 +186,8 @@
         reasoning-history (or (:reasoningHistory model-config) :all)
         [auth-type api-key] (llm-util/provider-api-key provider provider-auth config)
         api-url (llm-util/provider-api-url provider config)
+        ;; Flatten {:static :dynamic} instructions map into a single string for non-Anthropic providers
+        flat-instructions (if (map? instructions) (f.prompt/instructions->str instructions) instructions)
         callbacks (when-not sync?
                     {:on-message-received on-message-received
                      :on-error on-error
@@ -199,7 +202,7 @@
         (= "openai" provider)
         (handler
          {:model real-model
-          :instructions instructions
+          :instructions flat-instructions
           :user-messages user-messages
           :max-output-tokens max-output-tokens
           :reason? reason?
@@ -248,7 +251,7 @@
                                         "copilot-integration-id" "vscode-chat"}
                                        extra-headers))
               base-opts {:model real-model
-                         :instructions instructions
+                         :instructions flat-instructions
                          :user-messages user-messages
                          :max-output-tokens max-output-tokens
                          :reason? reason?
@@ -278,7 +281,7 @@
         (= "google" provider)
         (handler
          {:model real-model
-          :instructions instructions
+          :instructions flat-instructions
           :user-messages user-messages
           :max-output-tokens max-output-tokens
           :reason? reason?
@@ -303,7 +306,7 @@
           :reason? (:reason? model-capabilities)
           :supports-image? supports-image?
           :model real-model
-          :instructions instructions
+          :instructions flat-instructions
           :user-messages user-messages
           :past-messages past-messages
           :tools tools
@@ -320,7 +323,7 @@
               http-client (:httpClient provider-config)]
           (handler
            {:model real-model
-            :instructions instructions
+            :instructions flat-instructions
             :user-messages user-messages
             :max-output-tokens max-output-tokens
             :web-search web-search

src/eca/llm_providers/anthropic.clj

@@ -66,8 +66,17 @@
             :name (:full-name tool)}) tools)
     web-search (conj {:type "web_search_20250305"
                       :name "web_search"
-                      :max_uses 10
-                      :cache_control {:type "ephemeral"}})))
+                      :max_uses 10})))
+
+(defn ^:private add-cache-to-last-tool
+  "Adds cache_control to the last tool in the tools array, ensuring
+   the full tools list is part of the cached prefix."
+  [tools]
+  (if (seq tools)
+    (shared/update-last
+     (vec tools)
+     (fn [tool] (assoc tool :cache_control {:type "ephemeral"})))
+    tools))
 
 (defn ^:private base-request! [{:keys [rid body api-url api-key auth-type url-relative-path content-block* on-error on-stream http-client extra-headers cancelled? stream-idle-timeout-seconds]}]
   (let [url (join-api-url api-url (or url-relative-path messages-path))
@@ -303,15 +312,21 @@
                      merge-adjacent-assistants
                      merge-adjacent-tool-results)
         stream? (boolean callbacks)
+        {:keys [static dynamic]} (if (map? instructions)
+                                    instructions
+                                    {:static instructions :dynamic nil})
+        system-blocks (cond-> [{:type "text" :text "You are Claude Code, Anthropic's official CLI for Claude."}
+                               {:type "text" :text static :cache_control {:type "ephemeral"}}]
+                        (not (string/blank? dynamic))
+                        (conj {:type "text" :text dynamic :cache_control {:type "ephemeral"}}))
         body (merge
               (assoc-some
                {:model model
                 :messages (add-cache-to-last-message messages)
                 :max_tokens (or max-output-tokens 32000)
                 :stream stream?
-                :tools (->tools tools web-search)
-                :system [{:type "text" :text "You are Claude Code, Anthropic's official CLI for Claude."}
-                         {:type "text" :text instructions :cache_control {:type "ephemeral"}}]}
+                :tools (add-cache-to-last-tool (->tools tools web-search))
+                :system system-blocks}
                :thinking (when reason?
                            {:type "enabled" :budget_tokens 2048}))
               extra-payload)