feat: added max/resp_body_bytes attr to logger plugins (#13034)

Author: janiussyafiq

apisix/plugins/clickhouse-logger.lua

@@ -14,7 +14,6 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
-
 local bp_manager_mod  = require("apisix.utils.batch-processor-manager")
 local log_util        = require("apisix.utils.log-util")
 local plugin          = require("apisix.plugin")
@@ -57,7 +56,9 @@ local schema = {
             items = {
                 type = "array"
             }
-        }
+        },
+        max_req_body_bytes = {type = "integer", minimum = 1, default = 524288},
+        max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288},
     },
     oneOf = {
         {required = {"endpoint_addr", "user", "password", "database", "logtable"}},
@@ -174,6 +175,9 @@ local function send_http_data(conf, log_message)
 end
 
 
+_M.access = log_util.check_and_read_req_body
+
+
 function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end

apisix/plugins/elasticsearch-logger.lua

@@ -14,7 +14,6 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
-
 local core            = require("apisix.core")
 local http            = require("resty.http")
 local log_util        = require("apisix.utils.log-util")
@@ -104,6 +103,8 @@ local schema = {
                 type = "array"
             }
         },
+        max_req_body_bytes = { type = "integer", minimum = 1, default = 524288 },
+        max_resp_body_bytes = { type = "integer", minimum = 1, default = 524288 },
     },
     encrypt_fields = {"auth.password"},
     oneOf = {
@@ -214,6 +215,7 @@ local function get_logger_entry(conf, ctx)
         core.json.encode(entry) .. "\n"
 end
 
+
 local function fetch_and_update_es_version(conf)
     if conf._version then
         return
@@ -290,12 +292,16 @@ function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end
 
-function _M.access(conf)
+
+function _M.access(conf, ctx)
     -- fetch_and_update_es_version will call ES server only the first time
     -- so this should not amount to considerable overhead
     fetch_and_update_es_version(conf)
+
+    log_util.check_and_read_req_body(conf, ctx)
 end
 
+
 function _M.log(conf, ctx)
     local metadata = plugin.plugin_metadata(plugin_name)
     local max_pending_entries = metadata and metadata.value and

apisix/plugins/file-logger.lua

@@ -14,6 +14,7 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
+
 local log_util     =   require("apisix.utils.log-util")
 local core         =   require("apisix.core")
 local plugin       =   require("apisix.plugin")
@@ -49,6 +50,8 @@ local schema = {
                 type = "array"
             }
         },
+        max_req_body_bytes = {type = "integer", minimum = 1, default = 524288},
+        max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288},
         match = {
             type = "array",
             maxItems = 20,
@@ -204,10 +207,15 @@ local function write_file_data(conf, log_message)
     end
 end
 
+
+_M.access = log_util.check_and_read_req_body
+
+
 function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end
 
+
 function _M.log(conf, ctx)
     local entry = log_util.get_log_entry(plugin_name, conf, ctx)
     if entry == nil then

apisix/plugins/http-logger.lua

@@ -14,7 +14,6 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
-
 local bp_manager_mod  = require("apisix.utils.batch-processor-manager")
 local plugin          = require("apisix.plugin")
 local log_util        = require("apisix.utils.log-util")
@@ -51,6 +50,8 @@ local schema = {
                 type = "array"
             }
         },
+        max_req_body_bytes = {type = "integer", minimum = 1, default = 524288},
+        max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288},
         concat_method = {type = "string", default = "json",
                          enum = {"json", "new_line"}},
         ssl_verify = {type = "boolean", default = false},
@@ -168,6 +169,9 @@ local function send_http_data(conf, log_message)
 end
 
 
+_M.access = log_util.check_and_read_req_body
+
+
 function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end

apisix/plugins/kafka-logger.lua

@@ -14,7 +14,6 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
-local expr     = require("resty.expr.v1")
 local core     = require("apisix.core")
 local log_util = require("apisix.utils.log-util")
 local producer = require ("resty.kafka.producer")
@@ -24,7 +23,7 @@ local plugin = require("apisix.plugin")
 local math     = math
 local pairs    = pairs
 local type     = type
-local req_read_body = ngx.req.read_body
+
 local plugin_name = "kafka-logger"
 local batch_processor_manager = bp_manager_mod.new("kafka logger")
 
@@ -220,30 +219,7 @@ local function send_kafka_data(conf, log_message, prod)
 end
 
 
-function _M.access(conf, ctx)
-    if conf.include_req_body then
-        local should_read_body = true
-        if conf.include_req_body_expr then
-            if not conf.request_expr then
-                local request_expr, err = expr.new(conf.include_req_body_expr)
-                if not request_expr then
-                    core.log.error('generate request expr err ', err)
-                    return
-                end
-                conf.request_expr = request_expr
-            end
-
-            local result = conf.request_expr:eval(ctx.var)
-
-            if not result then
-                should_read_body = false
-            end
-        end
-        if should_read_body then
-            req_read_body()
-        end
-    end
-end
+_M.access = log_util.check_and_read_req_body
 
 
 function _M.body_filter(conf, ctx)

apisix/plugins/loggly.lua

@@ -77,6 +77,8 @@ local schema = {
                 type = "array"
             }
         },
+        max_req_body_bytes = {type = "integer", minimum = 1, default = 524288},
+        max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288},
         tags = {
             type = "array",
             minItems = 1,
@@ -177,6 +179,9 @@ function _M.check_schema(conf, schema_type)
 end
 
 
+_M.access = log_util.check_and_read_req_body
+
+
 function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end

apisix/plugins/loki-logger.lua

@@ -14,7 +14,6 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
-
 local bp_manager_mod  = require("apisix.utils.batch-processor-manager")
 local log_util        = require("apisix.utils.log-util")
 local core            = require("apisix.core")
@@ -106,6 +105,8 @@ local schema = {
                 type = "array"
             }
         },
+        max_req_body_bytes = {type = "integer", minimum = 1, default = 524288},
+        max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288},
     },
     required = {"endpoint_addrs"}
 }
@@ -193,6 +194,9 @@ local function send_http_data(conf, log)
 end
 
 
+_M.access = log_util.check_and_read_req_body
+
+
 function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end

apisix/plugins/rocketmq-logger.lua

@@ -68,6 +68,8 @@ local schema = {
                 type = "array"
             }
         },
+        max_req_body_bytes = {type = "integer", minimum = 1, default = 524288},
+        max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288},
     },
     encrypt_fields = {"secret_key"},
     required = {"nameserver_list", "topic"}
@@ -138,6 +140,9 @@ local function send_rocketmq_data(conf, log_message, prod)
 end
 
 
+_M.access = log_util.check_and_read_req_body
+
+
 function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end

apisix/plugins/skywalking-logger.lua

@@ -14,7 +14,6 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
-
 local bp_manager_mod  = require("apisix.utils.batch-processor-manager")
 local plugin          = require("apisix.plugin")
 local log_util        = require("apisix.utils.log-util")
@@ -55,6 +54,8 @@ local schema = {
                 type = "array"
             }
         },
+        max_req_body_bytes = {type = "integer", minimum = 1, default = 524288},
+        max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288},
     },
     required = {"endpoint_addr"},
 }
@@ -139,6 +140,9 @@ local function send_http_data(conf, log_message)
 end
 
 
+_M.access = log_util.check_and_read_req_body
+
+
 function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end

apisix/plugins/sls-logger.lua

@@ -48,6 +48,8 @@ local schema = {
                 type = "array"
             }
         },
+        max_req_body_bytes = {type = "integer", minimum = 1, default = 524288},
+        max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288},
         timeout = {type = "integer", minimum = 1, default= 5000},
         log_format = {type = "object"},
         host = {type = "string"},
@@ -158,6 +160,9 @@ local function handle_log(entries)
 end
 
 
+_M.access = log_util.check_and_read_req_body
+
+
 function _M.body_filter(conf, ctx)
     log_util.collect_body(conf, ctx)
 end