Skip to content

Commit 2500db7

Browse files
feat: allow fetching stream healthcheck data through control api (#12996)
1 parent 430e92e commit 2500db7

File tree

10 files changed

+174
-16
lines changed

10 files changed

+174
-16
lines changed

apisix/cli/config.lua

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ local _M = {
105105
["prometheus-cache"] = "10m",
106106
["standalone-config"] = "10m",
107107
["status-report"] = "1m",
108+
["upstream-healthcheck"] = "10m",
108109
}
109110
},
110111
stream = {
@@ -120,7 +121,6 @@ local _M = {
120121
["plugin-limit-conn-stream"] = "10m",
121122
["worker-events-stream"] = "10m",
122123
["tars-stream"] = "1m",
123-
["upstream-healthcheck-stream"] = "10m",
124124
}
125125
},
126126
main_configuration_snippet = "",
@@ -162,7 +162,6 @@ local _M = {
162162
["plugin-limit-count"] = "10m",
163163
["prometheus-metrics"] = "10m",
164164
["plugin-limit-conn"] = "10m",
165-
["upstream-healthcheck"] = "10m",
166165
["worker-events"] = "10m",
167166
["lrucache-lock"] = "10m",
168167
["balancer-ewma"] = "10m",

apisix/cli/ngx_tpl.lua

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ lua {
7878
lua_shared_dict status-report {* meta.lua_shared_dict["status-report"] *};
7979
{% end %}
8080
lua_shared_dict nacos 10m;
81+
lua_shared_dict upstream-healthcheck {* meta.lua_shared_dict["upstream-healthcheck"] *};
8182
}
8283
8384
{% if enabled_stream_plugins["prometheus"] and not enable_http then %}
@@ -149,10 +150,6 @@ stream {
149150
lua_shared_dict etcd-cluster-health-check-stream {* stream.lua_shared_dict["etcd-cluster-health-check-stream"] *};
150151
lua_shared_dict worker-events-stream {* stream.lua_shared_dict["worker-events-stream"] *};
151152
152-
{% if stream.lua_shared_dict["upstream-healthcheck-stream"] then %}
153-
lua_shared_dict upstream-healthcheck-stream {* stream.lua_shared_dict["upstream-healthcheck-stream"] *};
154-
{% end %}
155-
156153
{% if enabled_discoveries["tars"] then %}
157154
lua_shared_dict tars-stream {* stream.lua_shared_dict["tars-stream"] *};
158155
{% end %}
@@ -283,7 +280,6 @@ http {
283280
{% end %}
284281
285282
lua_shared_dict internal-status {* http.lua_shared_dict["internal-status"] *};
286-
lua_shared_dict upstream-healthcheck {* http.lua_shared_dict["upstream-healthcheck"] *};
287283
lua_shared_dict worker-events {* http.lua_shared_dict["worker-events"] *};
288284
lua_shared_dict lrucache-lock {* http.lua_shared_dict["lrucache-lock"] *};
289285
lua_shared_dict balancer-ewma {* http.lua_shared_dict["balancer-ewma"] *};

apisix/control/v1.lua

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ local require = require
1818
local core = require("apisix.core")
1919
local plugin = require("apisix.plugin")
2020
local get_routes = require("apisix.router").http_routes
21+
local get_stream_routes = require("apisix.router").stream_routes
2122
local get_services = require("apisix.http.service").services
2223
local upstream_mod = require("apisix.upstream")
2324
local healthcheck_manager = require("apisix.healthcheck_manager")
@@ -179,6 +180,8 @@ local function _get_health_checkers()
179180
local infos = {}
180181
local routes = get_routes()
181182
iter_and_add_healthcheck_info(infos, routes)
183+
local stream_routes = get_stream_routes()
184+
iter_and_add_healthcheck_info(infos, stream_routes)
182185
local services = get_services()
183186
iter_and_add_healthcheck_info(infos, services)
184187
local upstreams = get_upstreams()
@@ -240,6 +243,8 @@ function _M.get_health_checker()
240243
values = get_services()
241244
elseif src_type == "upstreams" then
242245
values = get_upstreams()
246+
elseif src_type == "stream_routes" then
247+
values = get_stream_routes()
243248
else
244249
return 400, {error_msg = str_format("invalid src type %s", src_type)}
245250
end

apisix/healthcheck_manager.lua

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,6 @@ local waiting_pool = {} -- resource_path -> resource_ver
3636

3737
local DELAYED_CLEAR_TIMEOUT = 10
3838
local healthcheck_shdict_name = "upstream-healthcheck"
39-
local is_http = ngx.config.subsystem == "http"
40-
if not is_http then
41-
healthcheck_shdict_name = healthcheck_shdict_name .. "-" .. ngx.config.subsystem
42-
end
4339

4440

4541
local function get_healthchecker_name(value)

apisix/router.lua

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,14 @@ function _M.http_init_worker()
8787
router_ssl.init_worker()
8888
_M.router_ssl = router_ssl
8989

90+
-- Initialize stream router in HTTP workers only if stream mode is enabled
91+
-- This allows the Control API (which runs in HTTP workers) to access stream routes
92+
if conf and conf.apisix and conf.apisix.stream_proxy then
93+
local router_stream = require("apisix.stream.router.ip_port")
94+
router_stream.stream_init_worker(filter)
95+
_M.router_stream = router_stream
96+
end
97+
9098
_M.api = require("apisix.api_router")
9199
end
92100

conf/config.yaml.example

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ nginx_config: # Config for render the template to generate n
189189
# Please resize when the `error.log` prompts that the data is full.
190190
# NOTE: Restart APISIX to take effect.
191191
standalone-config: 10m
192+
upstream-healthcheck: 10m
192193

193194
stream:
194195
enable_access_log: false # Enable stream proxy access logging.
@@ -202,7 +203,6 @@ nginx_config: # Config for render the template to generate n
202203
plugin-limit-conn-stream: 10m
203204
worker-events-stream: 10m
204205
tars-stream: 1m
205-
upstream-healthcheck-stream: 10m
206206

207207
# Add other custom Nginx configurations.
208208
# Users are responsible for validating the custom configurations
@@ -283,7 +283,6 @@ nginx_config: # Config for render the template to generate n
283283
plugin-limit-count: 10m
284284
prometheus-metrics: 10m # In production, less than 50m is recommended
285285
plugin-limit-conn: 10m
286-
upstream-healthcheck: 10m
287286
worker-events: 10m
288287
lrucache-lock: 10m
289288
balancer-ewma: 10m

t/APISIX.pm

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ lua {
294294
lua_shared_dict standalone-config 10m;
295295
lua_shared_dict status-report 1m;
296296
lua_shared_dict nacos 10m;
297+
lua_shared_dict upstream-healthcheck 10m;
297298
}
298299
_EOC_
299300
}
@@ -416,7 +417,6 @@ _EOC_
416417
lua_shared_dict plugin-limit-conn-stream 10m;
417418
lua_shared_dict etcd-cluster-health-check-stream 10m;
418419
lua_shared_dict worker-events-stream 10m;
419-
lua_shared_dict upstream-healthcheck-stream 10m;
420420
421421
lua_shared_dict kubernetes-stream 1m;
422422
lua_shared_dict kubernetes-first-stream 1m;
@@ -592,7 +592,6 @@ _EOC_
592592
lua_shared_dict plugin-ai-rate-limiting 10m;
593593
lua_shared_dict plugin-ai-rate-limiting-reset-header 10m;
594594
lua_shared_dict internal-status 10m;
595-
lua_shared_dict upstream-healthcheck 32m;
596595
lua_shared_dict worker-events 10m;
597596
lua_shared_dict lrucache-lock 10m;
598597
lua_shared_dict balancer-ewma 1m;

t/admin/stream-routes.t

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,6 +599,9 @@ xrpc:
599599
ngx.say(body)
600600
end
601601
end
602+
603+
-- Clean up the stream route to avoid interfering with subsequent tests
604+
local code, body = t('/apisix/admin/stream_routes/1', ngx.HTTP_DELETE)
602605
}
603606
}
604607
--- request

t/cli/test_main.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -884,14 +884,16 @@ git checkout conf/config.yaml
884884

885885
echo '
886886
nginx_config:
887+
meta:
888+
lua_shared_dict:
889+
upstream-healthcheck: 20m
887890
http:
888891
lua_shared_dict:
889892
internal-status: 20m
890893
plugin-limit-req: 20m
891894
plugin-limit-count: 20m
892895
prometheus-metrics: 20m
893896
plugin-limit-conn: 20m
894-
upstream-healthcheck: 20m
895897
worker-events: 20m
896898
lrucache-lock: 20m
897899
balancer-ewma: 20m
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
use t::APISIX 'no_plan';
18+
19+
log_level('info');
20+
no_root_location();
21+
22+
add_block_preprocessor(sub {
23+
my ($block) = @_;
24+
25+
if (!$block->error_log && !$block->no_error_log) {
26+
$block->set_value("no_error_log", "[error]\n[alert]");
27+
}
28+
29+
my $config = ($block->config // "") . <<_EOC_;
30+
location /hit {
31+
content_by_lua_block {
32+
33+
local sock = ngx.socket.tcp()
34+
local ok, err = sock:connect("127.0.0.1", 1985)
35+
if not ok then
36+
ngx.log(ngx.ERR, "failed to connect: ", err)
37+
return ngx.exit(503)
38+
end
39+
40+
local bytes, err = sock:send("mmm")
41+
if not bytes then
42+
ngx.log(ngx.ERR, "send stream request error: ", err)
43+
return ngx.exit(503)
44+
end
45+
46+
local data, err = sock:receive("*a")
47+
if not data then
48+
sock:close()
49+
return ngx.exit(503)
50+
end
51+
ngx.print(data)
52+
}
53+
}
54+
55+
_EOC_
56+
57+
$block->set_value("config", $config);
58+
});
59+
60+
run_tests();
61+
62+
__DATA__
63+
64+
=== TEST 1: set stream route(id: 1)
65+
--- stream_enable
66+
--- config
67+
location /test {
68+
content_by_lua_block {
69+
local core = require("apisix.core")
70+
local dkjson = require("dkjson")
71+
local t = require("lib.test_admin").test
72+
local code, body = t('/apisix/admin/stream_routes/1',
73+
ngx.HTTP_PUT,
74+
[[{
75+
"remote_addr": "127.0.0.1",
76+
"upstream": {
77+
"nodes": {
78+
"127.0.0.1:1995": 1
79+
},
80+
"type": "roundrobin",
81+
"checks": {
82+
"active": {
83+
"timeout": 40,
84+
"type": "tcp",
85+
"unhealthy": {
86+
"interval": 60,
87+
"failures": 2
88+
},
89+
"healthy": {
90+
"interval": 60,
91+
"successes": 2
92+
},
93+
"concurrency": 2
94+
}
95+
},
96+
"retries": 3,
97+
"timeout": {
98+
"read": 40,
99+
"send": 40,
100+
"connect": 40
101+
}
102+
}
103+
}]]
104+
)
105+
106+
if code >= 300 then
107+
ngx.status = code
108+
return
109+
end
110+
111+
local stream = t("/hit", ngx.HTTP_GET)
112+
if stream >= 300 then
113+
ngx.status = stream
114+
return
115+
end
116+
117+
ngx.sleep(3)
118+
local healthcheck, _, body = t("/v1/healthcheck", ngx.HTTP_GET)
119+
if healthcheck >= 300 then
120+
ngx.status = healthcheck
121+
return
122+
end
123+
124+
local healthcheck_data, err = core.json.decode(body)
125+
if not healthcheck_data then
126+
ngx.log(ngx.ERR, "failed to decode healthcheck data: ", err)
127+
return ngx.exit(503)
128+
end
129+
ngx.say(dkjson.encode(healthcheck_data))
130+
131+
-- healthcheck of stream route
132+
local healthcheck, _, body = t("/v1/healthcheck/stream_routes/1", ngx.HTTP_GET)
133+
if healthcheck >= 300 then
134+
ngx.status = healthcheck
135+
return
136+
end
137+
138+
local healthcheck_data, err = core.json.decode(body)
139+
if not healthcheck_data then
140+
ngx.log(ngx.ERR, "failed to decode healthcheck data: ", err)
141+
return ngx.exit(503)
142+
end
143+
ngx.say(dkjson.encode(healthcheck_data))
144+
}
145+
}
146+
--- timeout: 5
147+
--- request
148+
GET /test
149+
--- response_body
150+
[{"name":"/apisix/stream_routes/1","nodes":[{"counter":{"http_failure":0,"success":0,"tcp_failure":0,"timeout_failure":0},"hostname":"127.0.0.1","ip":"127.0.0.1","port":1995,"status":"healthy"}],"type":"tcp"}]
151+
{"name":"/apisix/stream_routes/1","nodes":[{"counter":{"http_failure":0,"success":0,"tcp_failure":0,"timeout_failure":0},"hostname":"127.0.0.1","ip":"127.0.0.1","port":1995,"status":"healthy"}],"type":"tcp"}

0 commit comments

Comments
 (0)