Optimize cache miss path with deferred cache write

configs/records.yaml.default.in

@@ -86,6 +86,12 @@ records:
 # https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-http-cache-when-to-revalidate
       when_to_revalidate: 0
 
+# Defer cache open write until response headers are received.
+# Improves performance for non-cacheable responses but may affect
+# read-while-write and request coalescing for popular uncached URLs.
+# 0 - disabled (default, safe), 1 - enabled
+      defer_write_on_miss: 0
+
 ##############################################################################
 # Origin server connect attempts. Docs:
 #    https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#origin-server-connect-attempts

include/proxy/http/HttpConfig.h

@@ -578,6 +578,11 @@ struct OverridableHttpConfigParams {
 
   MgmtByte cache_open_write_fail_action = 0;
 
+  ////////////////////////////////////////////////
+  // Defer cache open write until response hdrs //
+  ////////////////////////////////////////////////
+  MgmtByte cache_defer_write_on_miss = 0;
+
   ////////////////////////
   // Check Post request //
   ////////////////////////

include/proxy/http/HttpTransact.h

@@ -693,6 +693,10 @@ class HttpTransact
     bool is_method_stats_incremented  = false;
     bool skip_ip_allow_yaml           = false;
 
+    /// True if we deferred opening the cache for write until after response headers.
+    /// This is an optimization to avoid cache overhead for non-cacheable responses.
+    bool cache_open_write_deferred = false;
+
     /// True if the response is cacheable because of negative caching configuration.
     ///
     /// This being true implies the following:
@@ -997,6 +1001,7 @@ class HttpTransact
   static void set_cache_prepare_write_action_for_new_request(State *s);
   static void build_response_from_cache(State *s, HTTPWarningCode warning_code);
   static void handle_cache_write_lock(State *s);
+  static void handle_deferred_cache_write_complete(State *s);
   static void HandleResponse(State *s);
   static void HandleUpdateCachedObject(State *s);
   static void HandleUpdateCachedObjectContinue(State *s);

include/proxy/http/OverridableConfigDefs.h

@@ -249,6 +249,7 @@
   X(HTTP_NEGATIVE_CACHING_LIST,                     negative_caching_list,                      "proxy.config.http.negative_caching_list",                        STRING, HttpStatusCodeList_Conv) \
   X(HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE,       connect_attempts_retry_backoff_base,        "proxy.config.http.connect_attempts_retry_backoff_base",          INT,    GENERIC) \
   X(HTTP_NEGATIVE_REVALIDATING_LIST,                negative_revalidating_list,                 "proxy.config.http.negative_revalidating_list",                   STRING, HttpStatusCodeList_Conv) \
-  X(HTTP_CACHE_POST_METHOD,                         cache_post_method,                          "proxy.config.http.cache.post_method",                            INT,    GENERIC)
+  X(HTTP_CACHE_POST_METHOD,                         cache_post_method,                          "proxy.config.http.cache.post_method",                            INT,    GENERIC) \
+  X(HTTP_CACHE_DEFER_WRITE_ON_MISS,                 cache_defer_write_on_miss,                  "proxy.config.http.cache.defer_write_on_miss",                    INT,    GENERIC)
 
 // clang-format on

include/ts/apidefs.h.in

@@ -908,6 +908,7 @@ enum TSOverridableConfigKey {
   TS_CONFIG_HTTP_CONNECT_ATTEMPTS_RETRY_BACKOFF_BASE,
   TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIST,
   TS_CONFIG_HTTP_CACHE_POST_METHOD,
+  TS_CONFIG_HTTP_CACHE_DEFER_WRITE_ON_MISS,
   TS_CONFIG_LAST_ENTRY,
 };
 

src/proxy/http/HttpConfig.cc

@@ -1087,6 +1087,7 @@ HttpConfig::startup()
   HttpEstablishStaticConfigByte(c.disallow_post_100_continue, "proxy.config.http.disallow_post_100_continue");
 
   HttpEstablishStaticConfigByte(c.oride.cache_open_write_fail_action, "proxy.config.http.cache.open_write_fail_action");
+  HttpEstablishStaticConfigByte(c.oride.cache_defer_write_on_miss, "proxy.config.http.cache.defer_write_on_miss");
 
   HttpEstablishStaticConfigByte(c.oride.cache_when_to_revalidate, "proxy.config.http.cache.when_to_revalidate");
   HttpEstablishStaticConfigByte(c.oride.cache_required_headers, "proxy.config.http.cache.required_headers");
@@ -1391,6 +1392,7 @@ HttpConfig::reconfigure()
               params->oride.max_cache_open_write_retries);
     }
   }
+  params->oride.cache_defer_write_on_miss = m_master.oride.cache_defer_write_on_miss;
 
   params->oride.cache_when_to_revalidate = m_master.oride.cache_when_to_revalidate;
   params->max_post_size                  = m_master.max_post_size;

src/proxy/http/HttpTransact.cc

@@ -3231,6 +3231,62 @@ HttpTransact::handle_cache_write_lock(State *s)
   }
 }
 
+///////////////////////////////////////////////////////////////////////////////
+// Name       : handle_deferred_cache_write_complete
+// Description: Called after cache write opens for deferred cache write.
+//              We already have the server response headers, so we continue
+//              with cache write handling.
+//
+// Possible Next States From Here:
+// - handle_cache_operation_on_forward_server_response (if cache write succeeded)
+// - handle_no_cache_operation_on_forward_server_response (if cache write failed)
+//
+///////////////////////////////////////////////////////////////////////////////
+void
+HttpTransact::handle_deferred_cache_write_complete(State *s)
+{
+  TxnDbg(dbg_ctl_http_trans, "handle_deferred_cache_write_complete");
+  TxnDbg(dbg_ctl_http_seq, "Entering handle_deferred_cache_write_complete");
+
+  ink_assert(s->cache_info.action == CacheAction_t::PREPARE_TO_WRITE);
+
+  switch (s->cache_info.write_lock_state) {
+  case CacheWriteLock_t::SUCCESS:
+    // Cache write lock acquired successfully
+    TxnDbg(dbg_ctl_http_trans, "[hdcwc] cache write lock acquired");
+    SET_UNPREPARE_CACHE_ACTION(s->cache_info);
+    // Now we have WRITE action, handle the response with caching
+    handle_cache_operation_on_forward_server_response(s);
+    return;
+
+  case CacheWriteLock_t::FAIL:
+    // Could not get cache write lock, continue without caching
+    TxnDbg(dbg_ctl_http_trans, "[hdcwc] cache write lock failed, continuing without cache");
+    Metrics::Counter::increment(http_rsb.cache_open_write_fail_count);
+    s->cache_info.action       = CacheAction_t::NO_ACTION;
+    s->cache_info.write_status = CacheWriteStatus_t::LOCK_MISS;
+    break;
+
+  case CacheWriteLock_t::READ_RETRY:
+    // This shouldn't happen for deferred write since we don't have an object to read
+    TxnDbg(dbg_ctl_http_trans, "[hdcwc] unexpected READ_RETRY for deferred write");
+    s->cache_info.action       = CacheAction_t::NO_ACTION;
+    s->cache_info.write_status = CacheWriteStatus_t::LOCK_MISS;
+    break;
+
+  case CacheWriteLock_t::INIT:
+  default:
+    ink_release_assert(0);
+    break;
+  }
+
+  // If we get here, cache write failed - continue without caching
+  // The original handle_no_cache_operation_on_forward_server_response will be called
+  // but we need to skip the deferred cache write check since we just tried it
+  // s->cache_open_write_deferred is already false from the first call
+  handle_no_cache_operation_on_forward_server_response(s);
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Name       : HandleCacheOpenReadMiss
 // Description: cache looked up, miss or hit, but needs authorization
@@ -3276,6 +3332,13 @@ HttpTransact::HandleCacheOpenReadMiss(State *s)
     s->cache_info.action = CacheAction_t::NO_ACTION;
   } else if (s->api_server_response_no_store) { // plugin may have decided not to cache the response
     s->cache_info.action = CacheAction_t::NO_ACTION;
+  } else if (s->txn_conf->cache_defer_write_on_miss) {
+    // Defer cache open write until after response headers are received.
+    // This avoids cache overhead for non-cacheable responses but may
+    // affect read-while-write and request coalescing for popular URLs.
+    s->cache_open_write_deferred = true;
+    s->cache_info.action         = CacheAction_t::NO_ACTION;
+    TxnDbg(dbg_ctl_http_trans, "[HandleCacheOpenReadMiss] deferring cache open write until response");
   } else {
     HttpTransact::set_cache_prepare_write_action_for_new_request(s);
   }
@@ -4677,6 +4740,24 @@ HttpTransact::handle_no_cache_operation_on_forward_server_response(State *s)
   TxnDbg(dbg_ctl_http_trans, "(hncoofsr)");
   TxnDbg(dbg_ctl_http_seq, "Entering handle_no_cache_operation_on_forward_server_response");
 
+  // Check if we deferred the cache open write and the response is cacheable.
+  // If so, we need to open the cache for write now.
+  if (s->cache_open_write_deferred && s->txn_conf->cache_http) {
+    bool cacheable = is_response_cacheable(s, &s->hdr_info.client_request, &s->hdr_info.server_response);
+    TxnDbg(dbg_ctl_http_trans, "[hncoofsr] deferred cache write, response %s cacheable", cacheable ? "is" : "is not");
+
+    if (cacheable) {
+      // Response is cacheable - open the cache for write now
+      s->cache_open_write_deferred   = false;
+      s->cache_info.action           = CacheAction_t::PREPARE_TO_WRITE;
+      s->cache_info.write_lock_state = CacheWriteLock_t::INIT;
+      // After cache write opens, continue with handle_cache_operation_on_forward_server_response
+      TRANSACT_RETURN(StateMachineAction_t::CACHE_ISSUE_WRITE, handle_deferred_cache_write_complete);
+    }
+    // Not cacheable - continue with no-cache operation
+    s->cache_open_write_deferred = false;
+  }
+
   bool        keep_alive = s->current.server->keep_alive == HTTPKeepAlive::KEEPALIVE;
   const char *warn_text  = nullptr;
 

src/records/RecordsConfig.cc

@@ -616,6 +616,13 @@ static constexpr RecordElement RecordsConfig[] =
   //       #  4 - return error if cache miss or if revalidate
   {RECT_CONFIG, "proxy.config.http.cache.open_write_fail_action", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
+  //       #  defer_write_on_miss:
+  //       #  0 - disabled (default). Open cache for write immediately on miss.
+  //       #  1 - enabled. Defer cache open write until response headers received.
+  //       #      Improves performance for non-cacheable responses but may affect
+  //       #      read-while-write and request coalescing for popular uncached URLs.
+  {RECT_CONFIG, "proxy.config.http.cache.defer_write_on_miss", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
   //       #  when_to_revalidate has 4 options:
   //       #
   //       #  0 - default. use cache directives or heuristic