diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index bfca1ccba5a2..d3149c3d3e71 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -712,11 +712,21 @@ http { {% end %} {% end %} {% end %} - {% if proxy_protocol and proxy_protocol.listen_http_port then %} - listen {* proxy_protocol.listen_http_port *} default_server proxy_protocol; + {% if proxy_protocol and proxy_protocol.listen_http then %} + {% for _, item in ipairs(proxy_protocol.listen_http) do %} + listen {* item.ip *}:{* item.port *} default_server proxy_protocol {% if enable_reuseport then %} reuseport {% end %}; + {% if item.enable_ipv6 then %} + listen [::]:{* item.port *} default_server proxy_protocol {% if enable_reuseport then %} reuseport {% end %}; + {% end %} + {% end %} + {% end %} + {% if proxy_protocol and proxy_protocol.listen_https then %} + {% for _, item in ipairs(proxy_protocol.listen_https) do %} + listen {* item.ip *}:{* item.port *} ssl default_server proxy_protocol {% if enable_reuseport then %} reuseport {% end %}; + {% if item.enable_ipv6 then %} + listen [::]:{* item.port *} ssl default_server proxy_protocol {% if enable_reuseport then %} reuseport {% end %}; + {% end %} {% end %} - {% if proxy_protocol and proxy_protocol.listen_https_port then %} - listen {* proxy_protocol.listen_https_port *} ssl default_server proxy_protocol; {% end %} server_name _; diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index 0bb64b68977a..781e53abe511 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -376,139 +376,176 @@ Please modify "admin_key" in conf/config.yaml . util.die("L4 prometheus metric should be exposed via export server\n") end - local ip_port_to_check = {} - local function listen_table_insert(listen_table, scheme, ip, port, - enable_http3, enable_ipv6) - if type(ip) ~= "string" then - util.die(scheme, " listen ip format error, must be string", "\n") - end + enable_http3, enable_ipv6, ip_port_to_check) + if type(ip) ~= "string" then + util.die(scheme, " listen ip format error, must be string", "\n") + end - if type(port) ~= "number" then - util.die(scheme, " listen port format error, must be number", "\n") - end + if type(port) ~= "number" then + util.die(scheme, " listen port format error, must be number", "\n") + end - if ports_to_check[port] ~= nil then - util.die(scheme, " listen port ", port, " conflicts with ", + if ports_to_check[port] ~= nil then + util.die(scheme, " listen port ", port, " conflicts with ", ports_to_check[port], "\n") + end + + local addr = ip .. ":" .. port + + if ip_port_to_check[addr] == nil then + table_insert(listen_table, + { + ip = ip, + port = port, + enable_http3 = enable_http3 + }) + ip_port_to_check[addr] = scheme + end + + if enable_ipv6 then + ip = "[::]" + addr = ip .. ":" .. port + + if ip_port_to_check[addr] == nil then + table_insert(listen_table, + { + ip = ip, + port = port, + enable_http3 = enable_http3 + }) + ip_port_to_check[addr] = scheme + end + end + end + + local function iterate_listen_values(conf, handler) + if type(conf) == "table" and conf[1] ~= nil then + for _, v in ipairs(conf) do + handler(v) + end + else + handler(conf) end + end - local addr = ip .. ":" .. port - - if ip_port_to_check[addr] == nil then - table_insert(listen_table, - { - ip = ip, - port = port, - enable_http3 = enable_http3 - }) - ip_port_to_check[addr] = scheme - end - - if enable_ipv6 then - ip = "[::]" - addr = ip .. ":" .. port + local function normalize_listen_conf(conf, opts) + local list = {} + local ip_port_to_check = {} + local any_http3 = false + local default_ip = opts.default_ip or "0.0.0.0" + local config_path = opts.config_path - if ip_port_to_check[addr] == nil then - table_insert(listen_table, - { - ip = ip, - port = port, - enable_http3 = enable_http3 - }) - ip_port_to_check[addr] = scheme + local function handle(value) + if value == nil then + return end - end - end - local node_listen = {} - -- listen in http, support multiple ports and specific IP, compatible with the original style - if type(yaml_conf.apisix.node_listen) == "number" then - listen_table_insert(node_listen, "http", "0.0.0.0", yaml_conf.apisix.node_listen, - false, yaml_conf.apisix.enable_ipv6) - elseif type(yaml_conf.apisix.node_listen) == "table" then - for _, value in ipairs(yaml_conf.apisix.node_listen) do if type(value) == "number" then - listen_table_insert(node_listen, "http", "0.0.0.0", value, - false, yaml_conf.apisix.enable_ipv6) - elseif type(value) == "table" then - local ip = value.ip - local port = value.port - local enable_ipv6 = false - local enable_http2 = value.enable_http2 - - if ip == nil then - ip = "0.0.0.0" - if yaml_conf.apisix.enable_ipv6 then - enable_ipv6 = true - end - end + listen_table_insert(list, opts.scheme, default_ip, value, + false, opts.enable_ipv6, ip_port_to_check) + return + end - if port == nil then - port = 9080 - end + if type(value) ~= "table" then + util.die(config_path, " listen format error, must be number or object", "\n") + end - if enable_http2 ~= nil then - util.die("ERROR: port level enable_http2 in node_listen is deprecated" - .. "from 3.9 version, and you should use enable_http2 in " - .. "apisix level.", "\n") - end + local ip = value.ip or default_ip + -- ensure IPv6 literals are wrapped in brackets for Nginx listen + if ip and ip:find(":", 1, true) and ip:sub(1, 1) ~= "[" then + ip = "[" .. ip .. "]" + end + local port = value.port or opts.default_port - listen_table_insert(node_listen, "http", ip, port, - false, enable_ipv6) + if value.enable_http2 ~= nil and opts.disallow_enable_http2 then + util.die("ERROR: port level enable_http2 in " .. config_path .. " is deprecated " + .. "from 3.9 version, and you should use enable_http2 in " + .. "apisix level.", "\n") end - end - end - yaml_conf.apisix.node_listen = node_listen - local enable_http3_in_server_context = false - local ssl_listen = {} - -- listen in https, support multiple ports, support specific IP - for _, value in ipairs(yaml_conf.apisix.ssl.listen) do - local ip = value.ip - local port = value.port - local enable_ipv6 = false - local enable_http2 = value.enable_http2 - local enable_http3 = value.enable_http3 - - if ip == nil then - ip = "0.0.0.0" - if yaml_conf.apisix.enable_ipv6 then - enable_ipv6 = true + local enable_http3 = false + if opts.allow_http3 and value.enable_http3 ~= nil then + enable_http3 = value.enable_http3 end - end - if port == nil then - port = 9443 - end + if enable_http3 then + any_http3 = true + end - if enable_http2 ~= nil then - util.die("ERROR: port level enable_http2 in ssl.listen is deprecated" - .. "from 3.9 version, and you should use enable_http2 in " - .. "apisix level.", "\n") - end + local enable_ipv6 = opts.enable_ipv6 and value.ip == nil + listen_table_insert(list, opts.scheme, ip, port, enable_http3, enable_ipv6, + ip_port_to_check) + end - if enable_http3 == nil then - enable_http3 = false - end - if enable_http3 == true then - enable_http3_in_server_context = true - end + iterate_listen_values(conf, handle) - listen_table_insert(ssl_listen, "https", ip, port, - enable_http3, enable_ipv6) - end + return list, any_http3 + end + + local node_listen + node_listen = normalize_listen_conf(yaml_conf.apisix.node_listen, { + config_path = "node_listen", + default_port = 9080, + enable_ipv6 = yaml_conf.apisix.enable_ipv6, + disallow_enable_http2 = true, + }) + yaml_conf.apisix.node_listen = node_listen + + local ssl_listen + local enable_http3_in_server_context + ssl_listen, enable_http3_in_server_context = normalize_listen_conf( + yaml_conf.apisix.ssl.listen, { + config_path = "ssl.listen", + default_port = 9443, + enable_ipv6 = yaml_conf.apisix.enable_ipv6, + disallow_enable_http2 = true, + allow_http3 = true, + } + ) yaml_conf.apisix.ssl.listen = ssl_listen yaml_conf.apisix.enable_http3_in_server_context = enable_http3_in_server_context - -- enable ssl with place holder crt&key - yaml_conf.apisix.ssl.ssl_cert = "cert/ssl_PLACE_HOLDER.crt" - yaml_conf.apisix.ssl.ssl_cert_key = "cert/ssl_PLACE_HOLDER.key" + -- enable ssl with place holder crt&key + yaml_conf.apisix.ssl.ssl_cert = "cert/ssl_PLACE_HOLDER.crt" + yaml_conf.apisix.ssl.ssl_cert_key = "cert/ssl_PLACE_HOLDER.key" + + if yaml_conf.apisix.proxy_protocol then + local pp_listen_http = yaml_conf.apisix.proxy_protocol.listen_http + local pp_listen_https = yaml_conf.apisix.proxy_protocol.listen_https + + if not pp_listen_http and yaml_conf.apisix.proxy_protocol.listen_http_port then + stderr:write("WARNING: proxy_protocol.listen_http_port is deprecated; " .. + "use proxy_protocol.listen_http instead.\n") + pp_listen_http = yaml_conf.apisix.proxy_protocol.listen_http_port + end + + if not pp_listen_https and yaml_conf.apisix.proxy_protocol.listen_https_port then + stderr:write("WARNING: proxy_protocol.listen_https_port is deprecated; " .. + "use proxy_protocol.listen_https instead.\n") + pp_listen_https = yaml_conf.apisix.proxy_protocol.listen_https_port + end + + yaml_conf.apisix.proxy_protocol.listen_http = normalize_listen_conf( + pp_listen_http, { + config_path = "proxy_protocol.listen_http", + default_port = 9080, + enable_ipv6 = yaml_conf.apisix.enable_ipv6, + } + ) + yaml_conf.apisix.proxy_protocol.listen_https = normalize_listen_conf( + pp_listen_https, { + config_path = "proxy_protocol.listen_https", + default_port = 9081, + enable_ipv6 = yaml_conf.apisix.enable_ipv6, + } + ) + end - local tcp_enable_ssl - -- compatible with the original style which only has the addr - if enable_stream and yaml_conf.apisix.stream_proxy and yaml_conf.apisix.stream_proxy.tcp then + local tcp_enable_ssl + -- compatible with the original style which only has the addr + if enable_stream and yaml_conf.apisix.stream_proxy and yaml_conf.apisix.stream_proxy.tcp then local tcp = yaml_conf.apisix.stream_proxy.tcp for i, item in ipairs(tcp) do if type(item) ~= "table" then diff --git a/apisix/cli/schema.lua b/apisix/cli/schema.lua index 8a70bda8bca2..c2ed1dcd5a73 100644 --- a/apisix/cli/schema.lua +++ b/apisix/cli/schema.lua @@ -78,11 +78,110 @@ local config_schema = { proxy_protocol = { type = "object", properties = { + listen_http = { + anyOf = { + { + type = "integer", + }, + { + type = "object", + properties = { + ip = { + type = "string", + }, + port = { + type = "integer", + minimum = 1, + maximum = 65535, + }, + }, + required = {"port"}, + }, + { + type = "array", + minItems = 1, + items = { + anyOf = { + { + type = "integer", + minimum = 1, + maximum = 65535, + }, + { + type = "object", + properties = { + ip = { + type = "string", + }, + port = { + type = "integer", + minimum = 1, + maximum = 65535, + }, + }, + required = {"port"}, + }, + }, + }, + uniqueItems = true, + }, + }, + }, listen_http_port = { type = "integer", + minimum = 1, + maximum = 65535, }, - listen_https_port = { - type = "integer", + listen_https = { + anyOf = { + { + type = "integer", + minimum = 1, + maximum = 65535, + }, + { + type = "object", + properties = { + ip = { + type = "string", + }, + port = { + type = "integer", + minimum = 1, + maximum = 65535, + }, + }, + required = {"port"}, + }, + { + type = "array", + minItems = 1, + items = { + anyOf = { + { + type = "integer", + minimum = 1, + maximum = 65535, + }, + { + type = "object", + properties = { + ip = { + type = "string", + }, + port = { + type = "integer", + minimum = 1, + maximum = 65535, + }, + }, + required = {"port"}, + }, + }, + }, + uniqueItems = true, + }, + }, }, enable_tcp_pp = { type = "boolean", diff --git a/conf/config.yaml.example b/conf/config.yaml.example index 139e30edc367..3674d0cf9235 100644 --- a/conf/config.yaml.example +++ b/conf/config.yaml.example @@ -36,8 +36,25 @@ apisix: enable_http2: true # proxy_protocol: # PROXY Protocol configuration - # listen_http_port: 9181 # APISIX listening port for HTTP traffic with PROXY protocol. - # listen_https_port: 9182 # APISIX listening port for HTTPS traffic with PROXY protocol. + # listen_http_port: 9181 # [DEPRECATED] HTTP listen port for PROXY Protocol + # listen_https_port: 9182 # [DEPRECATED] HTTPS listen port for PROXY Protocol + # + # # Preferred: Specify multiple ports or IP/port combinations for HTTP traffic with PROXY Protocol. + # listen_http: + # - port: 9181 # Default binds to all IPs (`0.0.0.0` for IPv4 and `[::]` for IPv6, if enabled). + # - ip: 127.0.0.1 # Bind to a specific IPv4 address. + # port: 9183 + # - ip: ::1 # Bind to a specific IPv6 address (enable_ipv6 must be true). + # port: 9184 + # + # # Same for HTTPS traffic with PROXY Protocol. + # listen_https: + # - port: 9182 # Default binds to all IPs. + # - ip: 0.0.0.0 # Explicitly bind only to all IPv4 addresses. + # port: 9185 + # - ip: :: # Explicitly bind only to all IPv6 addresses. + # port: 9186 + # # enable_tcp_pp: true # Enable the PROXY protocol when stream_proxy.tcp is set. # enable_tcp_pp_to_upstream: true # Enable the PROXY protocol. diff --git a/t/cli/test_main.sh b/t/cli/test_main.sh index 62c128c94380..f629d067ff12 100755 --- a/t/cli/test_main.sh +++ b/t/cli/test_main.sh @@ -177,6 +177,154 @@ fi echo "passed: support specific IP listen in http and https" +# check LEGACY proxy protocol listen for http/https (ipv4 & ipv6) +echo " +apisix: + proxy_protocol: + listen_http_port: 9080 + listen_https_port: 9081 +" > conf/config.yaml + +make init + +count_pp_http_ipv4=`grep -c "listen 0.0.0.0:908. default_server proxy_protocol" conf/nginx.conf || true` +if [ $count_pp_http_ipv4 -ne 1 ]; then + echo "failed: failed to LEGACY support proxy_protocol http port with ipv4" + exit 1 +fi + +count_pp_http_ipv6=`grep -c "listen \[::\]:908. default_server proxy_protocol" conf/nginx.conf || true` +if [ $count_pp_http_ipv6 -ne 1 ]; then + echo "failed: failed to LEGACY support proxy_protocol http port with ipv6" + exit 1 +fi + +count_pp_https_ipv4=`grep -c "listen 0.0.0.0:908. ssl default_server proxy_protocol" conf/nginx.conf || true` +if [ $count_pp_https_ipv4 -ne 1 ]; then + echo "failed: failed to LEGACY support proxy_protocol https ports with ipv4" + exit 1 +fi + +count_pp_https_ipv6=`grep -c "listen \[::\]:908. ssl default_server proxy_protocol" conf/nginx.conf || true` +if [ $count_pp_https_ipv6 -ne 1 ]; then + echo "failed: failed to LEGACY support proxy_protocol https ports with ipv6" + exit 1 +fi + +echo "passed: LEGACY proxy protocol ports for http/https with ipv4 & ipv6" + +# check proxy protocol listen for http/https (ipv4 & ipv6) +echo " +apisix: + proxy_protocol: + listen_http: + - 9080 + - 9081 + listen_https: + - 9082 + - 9083 +" > conf/config.yaml + +make init + +count_pp_http_ipv4=`grep -c "listen 0.0.0.0:908. default_server proxy_protocol" conf/nginx.conf || true` +if [ $count_pp_http_ipv4 -ne 2 ]; then + echo "failed: failed to support proxy_protocol http ports with ipv4" + exit 1 +fi + +count_pp_http_ipv6=`grep -c "listen \[::\]:908. default_server proxy_protocol" conf/nginx.conf || true` +if [ $count_pp_http_ipv6 -ne 2 ]; then + echo "failed: failed to support proxy_protocol http ports with ipv6" + exit 1 +fi + +count_pp_https_ipv4=`grep -c "listen 0.0.0.0:908. ssl default_server proxy_protocol" conf/nginx.conf || true` +if [ $count_pp_https_ipv4 -ne 2 ]; then + echo "failed: failed to support proxy_protocol https ports with ipv4" + exit 1 +fi + +count_pp_https_ipv6=`grep -c "listen \[::\]:908. ssl default_server proxy_protocol" conf/nginx.conf || true` +if [ $count_pp_https_ipv6 -ne 2 ]; then + echo "failed: failed to support proxy_protocol https ports with ipv6" + exit 1 +fi + +echo "passed: proxy protocol listen for http/https with ipv4 & ipv6" + +# check auto-bracketing for raw IPv6 input in proxy_protocol +echo " +apisix: + proxy_protocol: + listen_http: + - ip: ::1 + port: 9080 + - ip: 2001:db8::1 + port: 8080 +" > conf/config.yaml + +make init + +# Check auto-bracketing for ::1 +auto_bracket_ipv6=`grep -c "listen \\[::1\\]:9080 default_server proxy_protocol" conf/nginx.conf || true` +if [ $auto_bracket_ipv6 -ne 1 ]; then + echo "failed: auto-bracketing for ::1 IPv6 address in proxy_protocol" + exit 1 +fi + +# Check retention of correctly formatted IPv6 address [2001:db8::1] +formatted_ipv6_preserved=`grep -c "listen \\[2001:db8::1\\]:8080 default_server proxy_protocol" conf/nginx.conf || true` +if [ $formatted_ipv6_preserved -ne 1 ]; then + echo "failed: retention of formatted IPv6 address in proxy_protocol" + exit 1 +fi + +echo "passed: auto-bracketing and retention of IPv6 proxy_protocol addresses" + +# check support specific IPv6 IP listen in http and https +echo " +apisix: + node_listen: + - ip: ::1 + port: 9081 + ssl: + enable: true + listen: + - ip: ::1 + port: 9444 + enable_http3: true +" > conf/config.yaml + +make init + +count_http_ipv6_specific_ip=`grep -c "listen \[::1\]:9081" conf/nginx.conf || true` +if [ $count_http_ipv6_specific_ip -ne 1 ]; then + echo "failed: failed to support specific IPv6 IP listen in http" + exit 1 +fi + +count_https_ipv6_specific_ip=`grep -c "listen \[::1\]:9444 ssl" conf/nginx.conf || true` +if [ $count_https_ipv6_specific_ip -ne 1 ]; then + echo "failed: failed to support specific IPv6 IP listen in https" + exit 1 +fi + +count_https_ipv6_specific_ip_and_enable_quic=`grep -c "listen \[::1\]:9444 quic" conf/nginx.conf || true` +if [ $count_https_ipv6_specific_ip_and_enable_quic -ne 1 ]; then + echo "failed: failed to support specific IPv6 IP and enable quic listen in https" + exit 1 +fi + +count_https_ipv6_specific_ip_and_enable_http3=`grep -c "http3 on" conf/nginx.conf || true` +if [ $count_https_ipv6_specific_ip_and_enable_http3 -lt 1 ]; then + echo "failed: failed to enable http3 for specific IPv6 IP in https" + exit 1 +fi + +echo "passed: support specific IPv6 IP listen in http and https" + + # check deprecated enable_http2 in node_listen echo " apisix: