Messy WIP.

Author: GUI

config/schema.cue

@@ -57,6 +57,8 @@ import "path"
     proxy_buffer_size: string | *"8k"
     proxy_buffers: string | *"8 8k"
     keepalive_timeout: uint | *75
+    upstream_keepalive_connections_per_worker: uint | *10
+    upstream_keepalive_idle_timeout: uint | *60
     ssl_protocols: string | *"TLSv1 TLSv1.1 TLSv1.2"
     ssl_ciphers: string | *"ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS"
     ssl_session_cache: string | *"shared:ssl_sessions:50m"

src/api-umbrella/utils/active_config_store/set_envoy_config.lua

@@ -99,6 +99,17 @@ local function build_cluster_resource(cluster_name, options)
         },
       },
     },
+    typed_extension_protocol_options = {
+      ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] = {
+        ["@type"] = "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
+        upstream_http_protocol_options = {
+          auto_sni = true,
+        },
+        common_http_protocol_options = {
+          idle_timeout = "1s",
+        },
+      },
+    },
     connect_timeout = file_config["envoy"]["_connect_timeout"],
     upstream_connection_options = {
       tcp_keepalive = {
@@ -289,7 +300,7 @@ local function build_listener()
               stat_prefix = "router",
               common_http_protocol_options = {
                 max_headers_count = 200,
-                idle_timeout = "120s",
+                idle_timeout = "15s",
               },
               generate_request_id = false,
               server_header_transformation = "PASS_THROUGH",

templates/etc/nginx/router.conf.etlua

@@ -194,7 +194,8 @@ http {
 
   upstream api_umbrella_trafficserver_backend {
     server <%- config["trafficserver"]["host"] %>:<%- config["trafficserver"]["port"] %>;
-    keepalive 10;
+    keepalive <%- config["nginx"]["upstream_keepalive_connections_per_worker"] %>;
+    keepalive_timeout <%- config["nginx"]["upstream_keepalive_idle_timeout"] %>s;
   }
 
   server {

templates/etc/trafficserver/plugin.config.etlua

@@ -11,6 +11,8 @@ header_rewrite.so "<%- config["etc_dir"] %>/trafficserver/header_rewrite.conf"
 # the same URL:  https://issues.apache.org/jira/browse/TS-3431
 cachekey.so --include-headers=Host,X-Api-Umbrella-Backend-Host,X-Api-Umbrella-Cache-Request-Method
 
+stats_over_http.so
+
 <% if config["_strip_response_cookies_regex"] then %>
   tslua.so "<%- config["etc_dir"] %>/trafficserver/strip_response_cookies.lua" "<%- config["etc_dir"] %>/trafficserver/strip_response_cookies_regex.txt"
 <% end %>

templates/etc/trafficserver/records.yaml.etlua

@@ -31,9 +31,9 @@ records:
       warning: E
 
     # Enable for debug logging.
-    # debug:
-    #   enabled: 1
-    #   tags: ".*"
+    debug:
+      enabled: 1
+      tags: ".*"
 
   error:
     logfile:
@@ -122,17 +122,20 @@ records:
 
     # Keepalive connections to backend Envoy server. Retain some idle
     # connections open to improve performance.
-    keep_alive_no_activity_timeout_out: <%= json_encode(config["router"]["api_backends"]["keepalive_idle_timeout"]) %>
+    keep_alive_no_activity_timeout_out: 99 # <%= json_encode(config["router"]["api_backends"]["keepalive_idle_timeout"]) %>
+    # transaction_no_activity_timeout_out: 1
     per_server:
       connection:
-        min: <%= json_encode(config["router"]["api_backends"]["keepalive_connections"]) %>
+        min: 1 # <%= json_encode(config["router"]["api_backends"]["keepalive_connections"]) %>
+        match: ip
 
     # Since we're only connecting to our backend Envoy server, we can significantly
     # reduce connections TrafficServer tries to open by ignoring the Host for
     # session reuse (since Envoy will be responsible for actually connecting to the
     # backend servers).
     server_session_sharing:
       match: ip
+      pool: global_locked
 
     # For read_while_writer configuration in the `cache` section.
     background_fill_active_timeout: 0
@@ -182,12 +185,12 @@ records:
     #   Requires enabling redirect following, which we don't want for other
     #   responses:
     #   https://lists.apache.org/thread.html/0eff5d9a53ef8fdf28be341f648c708bd651ad1208cddf71d532d78d@%3Cusers.trafficserver.apache.org%3E
-    max_doc_size: 0
-    enable_read_while_writer: 1
-    read_while_writer:
-      max_retries: 21
-    read_while_writer_retry:
-      delay: 50
+    # max_doc_size: 0
+    # enable_read_while_writer: 1
+    # read_while_writer:
+    #   max_retries: 21
+    # read_while_writer_retry:
+    #   delay: 50
 
   dns:
     # Set the DNS nameservers used to potentially resolve a remote envoy layer.

test/proxy/keep_alive/test_server_side.rb

@@ -9,24 +9,35 @@ def setup
     setup_server
     reset_api_backend_idle_connections
     @keepalive_idle_timeout = 2
+    @nginx_workers = 2
+    @nginx_upstream_keepalive_connections_per_worker = 15
+    @router_keepalive_connections = 25
     once_per_class_setup do
       override_config_set({
+        :nginx => {
+          :workers => @nginx_workers,
+          :upstream_keepalive_connections_per_worker => @nginx_upstream_keepalive_connections_per_worker,
+          :upstream_keepalive_idle_timeout => @keepalive_idle_timeout,
+        },
         :router => {
           :api_backends => {
             :keepalive_idle_timeout => @keepalive_idle_timeout,
+            :keepalive_connections => @router_keepalive_connections,
           },
         },
       })
 
       prepend_api_backends([
         {
+          :name => unique_test_class_id,
           :frontend_host => "127.0.0.1",
           :backend_host => "127.0.0.1",
           :servers => [{ :host => "127.0.0.1", :port => 9444 }],
           :url_matches => [{ :frontend_prefix => "/#{unique_test_class_id}/keepalive-default/", :backend_prefix => "/" }],
         },
       ])
     end
+    @api_backend = ApiBackend.find_by!(name: unique_test_class_id)
   end
 
   def after_all
@@ -39,7 +50,7 @@ def test_keeps_idle_connections_open
     # might fix things: https://github.com/apache/trafficserver/pull/8083 In
     # the meantime, the current behavior means idle connections perhaps stay
     # around too long, but I think this should be okay for now.
-    skip("Keepalive idle handling doesn't work as expected in Traffic Server 9.1, but the behavior should still be acceptable. Revisit in Traffic Server 9.2+.")
+    # skip("Keepalive idle handling doesn't work as expected in Traffic Server 9.1, but the behavior should still be acceptable. Revisit in Traffic Server 9.2+.")
 
     assert_idle_connections("/#{unique_test_class_id}/keepalive-default/connection-stats/", $config["router"]["api_backends"]["keepalive_connections"])
   end
@@ -103,17 +114,24 @@ def reset_api_backend_idle_connections
 
   def assert_idle_connections(path, idle_connections)
     # After just making one connection, sanity check the keepalive connections
-    # to ensure it's just 1-2 (for the current connection). Keepalive
+    # to ensure it's just few (for the current connection). Keepalive
     # connections are lazily established, so this just verifies the current
     # behavior of the connections only being kept once they're actually used.
     response = Typhoeus.get("http://127.0.0.1:9080#{path}", http_options)
     assert_response_code(200, response)
-    data = MultiJson.load(response.body)
-    assert_operator(data["connections_waiting"], :<=, 2)
+    stats = connection_stats
+    ap stats
+    assert_includes(1..4, stats.fetch(:nginx_router_to_trafficserver_active_connections_per_trafficserver))
+    assert_includes(1..4, stats.fetch(:trafficserver_to_envoy_active_connections_per_trafficserver))
+    assert_includes(1..4, stats.fetch(:trafficserver_to_envoy_active_connections_per_envoy))
+    assert_includes(1..4, stats.fetch(:envoy_to_api_backend_active_connections_per_envoy))
+    assert_includes(1..4, stats.fetch(:envoy_to_api_backend_active_connections_per_api_backend))
+    assert_includes(1..4, stats.fetch(:envoy_to_api_backend_idle_connections_per_api_backend))
 
     # Open a bunch of concurrent connections first, and then inspect the number
     # of number of connections still active afterwards.
-    hydra = Typhoeus::Hydra.new(:max_concurrency => 200)
+    max_concurrency = 200
+    hydra = Typhoeus::Hydra.new(max_concurrency: max_concurrency)
     500.times do
       request = Typhoeus::Request.new("http://127.0.0.1:9080#{path}", http_options)
       request.on_complete do |resp|
@@ -127,10 +145,32 @@ def assert_idle_connections(path, idle_connections)
     # bunch of idle connections open, since Traffic Server keeps these
     # connections around until the keepalive_idle_timeout is reached (which
     # we've lowered for testing purposes).
-    response = Typhoeus.get("http://127.0.0.1:9444/connection-stats/", http_options)
-    assert_response_code(200, response)
-    data = MultiJson.load(response.body)
-    assert_operator(data["connections_waiting"], :>, idle_connections + 2)
+    stats = connection_stats
+    ap stats
+    assert_in_delta(@nginx_upstream_keepalive_connections_per_worker * @nginx_workers, stats.fetch(:nginx_router_to_trafficserver_active_connections_per_trafficserver), 5)
+    assert_operator(stats.fetch(:nginx_router_to_trafficserver_active_connections_per_trafficserver), :<=, max_concurrency)
+    assert_operator(stats.fetch(:trafficserver_to_envoy_active_connections_per_trafficserver), :>, idle_connections + 2)
+    assert_operator(stats.fetch(:trafficserver_to_envoy_active_connections_per_trafficserver), :<=, max_concurrency)
+    assert_operator(stats.fetch(:trafficserver_to_envoy_active_connections_per_envoy), :>, idle_connections + 2)
+    assert_operator(stats.fetch(:trafficserver_to_envoy_active_connections_per_envoy), :<=, max_concurrency)
+    assert_operator(stats.fetch(:envoy_to_api_backend_active_connections_per_envoy), :>, idle_connections + 2)
+    assert_operator(stats.fetch(:envoy_to_api_backend_active_connections_per_envoy), :<=, max_concurrency)
+    assert_operator(stats.fetch(:envoy_to_api_backend_active_connections_per_api_backend), :>, idle_connections + 2)
+    assert_operator(stats.fetch(:envoy_to_api_backend_active_connections_per_api_backend), :<=, max_concurrency)
+    assert_operator(stats.fetch(:envoy_to_api_backend_idle_connections_per_api_backend), :>, idle_connections + 2)
+    assert_operator(stats.fetch(:envoy_to_api_backend_idle_connections_per_api_backend), :<=, max_concurrency)
+
+    300.times do
+      request = Typhoeus::Request.new("http://127.0.0.1:9080#{path}", http_options)
+      request.on_complete do |resp|
+        assert_response_code(200, resp)
+      end
+      hydra.queue(request)
+    end
+    hydra.run
+
+    stats = connection_stats
+    ap stats
 
     # Wait for the keepalive timeout to expire, after which the number of idle
     # connections should be lowered to just the persistent ones that are kept
@@ -141,15 +181,30 @@ def assert_idle_connections(path, idle_connections)
       # but add a considerable buffer to this, since we see some some sporadic
       # issues where this sometimes takes longer in the test suite (but the
       # exact timing of this behavior isn't really that important).
-      Timeout.timeout(@keepalive_idle_timeout + 10) do
+      Timeout.timeout(@keepalive_idle_timeout + 300) do
         loop do
-          response = Typhoeus.get("http://127.0.0.1:9444/connection-stats/", http_options)
-          if(response.code == 200)
-            data = MultiJson.load(response.body)
-            if(data["connections_waiting"] <= idle_connections + 2)
-              break
-            end
-          end
+          stats = connection_stats
+          ap stats
+
+          # response = Typhoeus.get("http://127.0.0.1:13001/stats?filter=downstream_cx", http_options)
+          # if(response.code == 200)
+          #   puts response.body
+          # end
+
+          # response = Typhoeus.get("http://127.0.0.1:13009/_stats/csv", http_options)
+          # if(response.code == 200)
+          #   puts response.body
+          # end
+
+
+          # response = Typhoeus.get("http://127.0.0.1:9444/connection-stats/", http_options)
+          # if(response.code == 200)
+          #   data = MultiJson.load(response.body)
+          #   ap data
+          #   if(data["connections_waiting"] <= idle_connections + 2)
+          #     break
+          #   end
+          # end
 
           sleep 0.1
         end
@@ -181,4 +236,40 @@ def assert_idle_connections(path, idle_connections)
     assert_operator(data["connections_waiting"], :>=, idle_connections - 1)
     assert_operator(data["connections_waiting"], :<=, idle_connections + 2)
   end
+
+  def connection_stats
+    stats = {}
+
+    response = Typhoeus.get("http://127.0.0.1:9444/connection-stats/", http_options)
+    assert_response_code(200, response)
+    stats[:api_backend] = MultiJson.load(response.body)
+
+    response = Typhoeus.get("http://127.0.0.1:13001/stats", http_options.deep_merge({
+      params: {
+        format: "json",
+        filter: "(downstream_cx|upstream_cx)",
+      },
+    }))
+    assert_response_code(200, response)
+    stats[:envoy] = MultiJson.load(response.body).fetch("stats").each_with_object({}) { |stat, data| data[stat["name"]] = stat["value"] if stat["name"] }
+
+    response = Typhoeus.get("http://127.0.0.1:13009/_stats", http_options)
+    assert_response_code(200, response)
+    stats[:trafficserver] = MultiJson.load(response.body).fetch("global").each_with_object({}) { |(key, value), data| data[key] = Integer(value, exception: false) || Float(value, exception: false) || value }
+
+    stats[:nginx_router_to_trafficserver_active_connections_per_trafficserver] = stats.fetch(:trafficserver).fetch("proxy.process.http.current_client_connections")
+    stats[:trafficserver_to_envoy_active_connections_per_trafficserver] = stats.fetch(:trafficserver).fetch("proxy.process.http.current_server_connections")
+    stats[:trafficserver_to_envoy_active_connections_per_envoy] = stats.fetch(:envoy).fetch("http.router.downstream_cx_active")
+    stats[:envoy_to_api_backend_active_connections_per_envoy] = stats.fetch(:envoy).fetch("cluster.api-backend-cluster-#{@api_backend.id}.upstream_cx_active")
+    stats[:envoy_to_api_backend_active_connections_per_api_backend] = stats.fetch(:api_backend).fetch("connections_active")
+    stats[:envoy_to_api_backend_idle_connections_per_api_backend] = stats.fetch(:api_backend).fetch("connections_waiting")
+    # assert_operator(.to_i, :>, idle_connections + 2)
+    # assert_operator(stats.fetch(:trafficserver).fetch("proxy.process.http.current_server_connections").to_i, :>, idle_connections + 2)
+    # assert_operator(stats.fetch(:trafficserver).fetch("proxy.process.http.pooled_server_connections").to_i, :>, idle_connections + 2)
+    # assert_operator(stats.fetch(:envoy).fetch("http.router.downstream_cx_active"), :>, idle_connections + 2)
+    # assert_equal(0, stats.fetch(:envoy).fetch("http.router.downstream_cx_destroy_remote"))
+    # assert_operator(stats.fetch(:api_backend).fetch("connections_waiting"), :>, idle_connections + 2)
+
+    stats
+  end
 end