Merge branch 'apache:master' into master

Author: laz-xyr

.github/workflows/check-changelog.yml

@@ -0,0 +1,27 @@
+name: Check Changelog
+
+on:
+  push:
+    paths:
+      - 'CHANGELOG.md'
+      - 'ci/check_changelog_prs.ts'
+  pull_request:
+    paths:
+      - 'CHANGELOG.md'
+      - 'ci/check_changelog_prs.ts'
+
+jobs:
+  check-changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+
+      - name: Run check_changelog_prs script
+        working-directory: ci
+        run: |
+          curl -fsSL https://bun.sh/install | bash
+          export PATH="$HOME/.bun/bin:$PATH"
+          bun run check_changelog_prs.ts

.github/workflows/push-dev-image-on-commit.yml

@@ -49,7 +49,7 @@ jobs:
 
       - name: Login to Docker Hub
         if: github.ref == 'refs/heads/master'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

apisix/cli/file.lua

@@ -287,6 +287,8 @@ function _M.read_yaml_conf(apisix_home)
             default_conf.etcd = default_conf.deployment.etcd
             if default_conf.deployment.role_data_plane.config_provider == "yaml" then
                 default_conf.deployment.config_provider = "yaml"
+            elseif default_conf.deployment.role_data_plane.config_provider == "json" then
+                default_conf.deployment.config_provider = "json"
             elseif default_conf.deployment.role_data_plane.config_provider == "xds" then
                 default_conf.deployment.config_provider = "xds"
             end

apisix/cli/schema.lua

@@ -405,7 +405,7 @@ local deployment_schema = {
             role_data_plane = {
                 properties = {
                     config_provider = {
-                        enum = {"etcd", "yaml", "xds"}
+                        enum = {"etcd", "yaml", "json", "xds"}
                     },
                 },
                 required = {"config_provider"}

apisix/core.lua

@@ -24,7 +24,16 @@ end
 local config_provider = local_conf.deployment and local_conf.deployment.config_provider
                       or "etcd"
 log.info("use config_provider: ", config_provider)
-local config = require("apisix.core.config_" .. config_provider)
+
+local config
+-- Currently, we handle JSON parsing in config_yaml, so special processing is needed here.
+if config_provider == "json" then
+    config = require("apisix.core.config_yaml")
+    config.file_type = "json"
+else
+    config = require("apisix.core.config_" .. config_provider)
+end
+
 config.type = config_provider
 
 

apisix/core/config_yaml.lua

@@ -46,7 +46,6 @@ local ngx          = ngx
 local re_find      = ngx.re.find
 local process      = require("ngx.process")
 local worker_id    = ngx.worker.id
-local apisix_yaml_path = profile:yaml_path("apisix")
 local created_obj  = {}
 local shared_dict
 local status_report_shared_dict_name = "status-report"
@@ -56,6 +55,9 @@ local _M = {
     local_conf = config_local.local_conf,
     clear_local_cache = config_local.clear_cache,
 
+    -- yaml or json
+    file_type = "yaml",
+
     ERR_NO_SHARED_DICT = "failed prepare standalone config shared dict, this will degrade "..
                     "to event broadcasting, and if a worker crashes, the configuration "..
                     "cannot be restored from other workers and shared dict"
@@ -69,10 +71,68 @@ local mt = {
     end
 }
 
-
 local apisix_yaml
 local apisix_yaml_mtime
 
+local config_yaml = {
+    path = profile:yaml_path("apisix"),
+    type = "yaml",
+    parse = function(self)
+        local f, err = io.open(self.path, "r")
+        if not f then
+            return nil, "failed to open file " .. self.path .. " : " .. err
+        end
+
+        f:seek('end', -10)
+        local end_flag = f:read("*a")
+        local found_end_flag = re_find(end_flag, [[#END\s*$]], "jo")
+
+        if not found_end_flag then
+            f:close()
+            return nil, "missing valid end flag in file " .. self.path
+        end
+
+        f:seek('set')
+        local raw_config = f:read("*a")
+        f:close()
+
+        return yaml.load(raw_config), nil
+    end
+}
+
+local config_json = {
+    -- `-5` to remove the "yaml" suffix
+    path = config_yaml.path:sub(1, -5) .. "json",
+    type = "json",
+    parse = function(self)
+        local f, err = io.open(self.path, "r")
+        if not f then
+            return nil, "failed to open file " .. self.path .. " : " .. err
+        end
+        local raw_config = f:read("*a")
+        f:close()
+
+        local config, err = json.decode(raw_config)
+        if err then
+            return nil, "failed to decode json: " .. err
+        end
+        return config, nil
+    end
+}
+
+local config_file_table = {
+    yaml = config_yaml,
+    json = config_json
+}
+
+
+local config_file = setmetatable({}, {
+    __index = function(_, key)
+        return config_file_table[_M.file_type][key]
+    end
+})
+
+
 local function sync_status_to_shdict(status)
     if process.type() ~= "worker" then
         return
@@ -112,13 +172,13 @@ local function is_use_admin_api()
 end
 
 
-local function read_apisix_yaml(premature, pre_mtime)
+local function read_apisix_config(premature, pre_mtime)
     if premature then
         return
     end
-    local attributes, err = lfs.attributes(apisix_yaml_path)
+    local attributes, err = lfs.attributes(config_file.path)
     if not attributes then
-        log.error("failed to fetch ", apisix_yaml_path, " attributes: ", err)
+        log.error("failed to fetch ", config_file.path, " attributes: ", err)
         return
     end
 
@@ -127,36 +187,15 @@ local function read_apisix_yaml(premature, pre_mtime)
         return
     end
 
-    local f, err = io.open(apisix_yaml_path, "r")
-    if not f then
-        log.error("failed to open file ", apisix_yaml_path, " : ", err)
-        return
-    end
-
-    f:seek('end', -10)
-    local end_flag = f:read("*a")
-    -- log.info("flag: ", end_flag)
-    local found_end_flag = re_find(end_flag, [[#END\s*$]], "jo")
-
-    if not found_end_flag then
-        f:close()
-        log.warn("missing valid end flag in file ", apisix_yaml_path)
-        return
-    end
-
-    f:seek('set')
-    local yaml_config = f:read("*a")
-    f:close()
-
-    local apisix_yaml_new = yaml.load(yaml_config)
-    if not apisix_yaml_new then
-        log.error("failed to parse the content of file " .. apisix_yaml_path)
+    local config_new, err = config_file:parse()
+    if err then
+        log.error("failed to parse the content of file ", config_file.path, ": ", err)
         return
     end
 
-    update_config(apisix_yaml_new, last_modification_time)
+    update_config(config_new, last_modification_time)
 
-    log.warn("config file ", apisix_yaml_path, " reloaded.")
+    log.warn("config file ", config_file.path, " reloaded.")
 end
 
 
@@ -171,7 +210,7 @@ local function sync_data(self)
     else
         if not apisix_yaml_mtime then
             log.warn("wait for more time")
-            return nil, "failed to read local file " .. apisix_yaml_path
+            return nil, "failed to read local file " .. config_file.path
         end
         conf_version = apisix_yaml_mtime
     end
@@ -395,15 +434,15 @@ local function _automatic_fetch(premature, self)
         local ok, ok2, err = pcall(sync_data, self)
         if not ok then
             err = ok2
-            log.error("failed to fetch data from local file " .. apisix_yaml_path .. ": ",
+            log.error("failed to fetch data from local file " .. config_file.path .. ": ",
                       err, ", ", tostring(self))
             ngx_sleep(3)
             break
 
         elseif not ok2 and err then
             if err ~= "timeout" and err ~= "Key not found"
                and self.last_err ~= err then
-                log.error("failed to fetch data from local file " .. apisix_yaml_path .. ": ",
+                log.error("failed to fetch data from local file " .. config_file.path .. ": ",
                           err, ", ", tostring(self))
             end
 
@@ -477,7 +516,7 @@ function _M.new(key, opts)
         end
 
         if err then
-            log.error("failed to fetch data from local file ", apisix_yaml_path, ": ",
+            log.error("failed to fetch data from local file ", config_file.path, ": ",
                       err, ", ", key)
         end
 
@@ -517,7 +556,7 @@ function _M.init()
         return true
     end
 
-    read_apisix_yaml()
+    read_apisix_config()
     return true
 end
 
@@ -531,7 +570,7 @@ function _M.init_worker()
     end
 
     -- sync data in each non-master process
-    ngx.timer.every(1, read_apisix_yaml)
+    ngx.timer.every(1, read_apisix_config)
 
     return true
 end

apisix/discovery/kubernetes/init.lua

@@ -633,44 +633,61 @@ function _M.init_worker()
 end
 
 
-function _M.dump_data()
+local function dump_endpoints_from_dict(endpoint_dict)
+    local keys, err = endpoint_dict:get_keys()
+    if err then
+        core.log.error("get keys from discovery dict failed: ", err)
+        return
+    end
 
-    local eps = {}
-    for _, conf in ipairs(local_conf.discovery.kubernetes) do
+    if not keys or #keys == 0 then
+        return
+    end
 
-        local id = conf.id
-        local endpoint_dict = get_endpoint_dict(id)
-        local keys, err = endpoint_dict:get_keys()
-        if err then
-            error(err)
-            break
+    local endpoints = {}
+    for i = 1, #keys do
+        local key = keys[i]
+        -- skip key with suffix #version
+        if key:sub(-#"#version") ~= "#version" then
+            local value = endpoint_dict:get(key)
+            core.table.insert(endpoints, {
+                name = key,
+                value = value
+            })
         end
+    end
 
-        if keys then
-            local k8s = {}
-            for i = 1, #keys do
-
-                local key = keys[i]
-                --skip key with suffix #version
-                if key:sub(-#"#version") ~= "#version" then
-                    local value = endpoint_dict:get(key)
+    return endpoints
+end
 
-                    core.table.insert(k8s, {
-                        name = key,
-                        value = value
-                    })
-                end
-            end
+function _M.dump_data()
+    local discovery_conf = local_conf.discovery.kubernetes
+    local eps = {}
 
+    if #discovery_conf == 0 then
+        -- Single mode: discovery_conf is a single configuration object
+        local endpoint_dict = get_endpoint_dict()
+        local endpoints = dump_endpoints_from_dict(endpoint_dict)
+        if endpoints then
             core.table.insert(eps, {
-                id = conf.id,
-                endpoints = k8s
+                endpoints = endpoints
             })
-
+        end
+    else
+        -- Multiple mode: discovery_conf is an array of configuration objects
+        for _, conf in ipairs(discovery_conf) do
+            local endpoint_dict = get_endpoint_dict(conf.id)
+            local endpoints = dump_endpoints_from_dict(endpoint_dict)
+            if endpoints then
+                core.table.insert(eps, {
+                    id = conf.id,
+                    endpoints = endpoints
+                })
+            end
         end
     end
 
-    return {config = local_conf.discovery.kubernetes, endpoints = eps}
+    return {config = discovery_conf, endpoints = eps}
 end