Merge pull request ceph#61282 from oshrey16/wip-lua-oshrey

rgw/lua: add configurable runtime limit for the lua state

Author: yuvalif

doc/radosgw/lua-scripting.rst

@@ -21,9 +21,10 @@ The request context script can also access fields in the request and modify cert
 The data context script can access the content of the object as well as the request fields and the `Global RGW Table`_. 
 All Lua language features can be used in all contexts.
 An execution of a script in a context can use up to 500K byte of memory. This include all libraries used by Lua, but not the memory which is managed by the RGW itself, and may be accessed from Lua.
-To change this default value, use the ``rgw_lua_max_memory_per_state`` configuration parameter. Note that the basic overhead of Lua with its standard libraries is ~32K bytes. To disable the limit, use zero or a negative number.
+To change this default value, use the ``rgw_lua_max_memory_per_state`` configuration parameter. Note that the basic overhead of Lua with its standard libraries is ~32K bytes. To disable the limit, use zero.
+By default, the execution of a Lua script is limited to a maximum runtime of 1000 milliseconds. This limit can be changed using the ``rgw_lua_max_runtime_per_state`` configuration parameter. If a Lua script exceeds this runtime, it will be terminated. To disable the runtime limit, use zero.
 
-By default, all Lua standard libraries are available in the script, however, in order to allow for other Lua modules to be used in the script, we support adding packages to an allowlist:
+By default, all Lua standard libraries are available in the script, however, in order to allow for additional Lua modules to be used in the script, we support adding packages to an allowlist:
 
   - Adding a Lua package to the allowlist, or removing a packge from it does not install or remove it. For the changes to take affect a "reload" command should be called.
   - In addition all packages in the allowlist are being re-installed using the luarocks package manager on radosgw restart.

src/common/options/rgw.yaml.in

@@ -4260,3 +4260,14 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_lua_max_runtime_per_state
+  type: uint
+  level: advanced
+  desc: Maximum runtime for each Lua state in milliseconds
+  long_desc: Sets the maximum runtime for each Lua state in milliseconds.
+    If exceeded, the script will be terminated. Defaults to 1000 milliseconds (1 second).
+    If set to zero, there is no limit.
+  default: 1000
+  services:
+  - rgw
+  with_legacy: true

src/rgw/rgw_lua.cc

@@ -56,7 +56,8 @@ std::string to_string(context ctx)
 
 bool verify(const std::string& script, std::string& err_msg) 
 {
-  lua_state_guard lguard(0, nullptr); // no memory limit, sice we don't execute the script
+  // no memory and runtime limit, since we don't execute the script
+  lua_state_guard lguard(0, 0, nullptr);
   auto L = lguard.get();
   try {
     open_standard_libs(L);

src/rgw/rgw_lua_background.cc

@@ -124,7 +124,8 @@ const BackgroundMapValue& Background::get_table_value(const std::string& key) co
 void Background::run() {
   ceph_pthread_setname("lua_background");
   const DoutPrefixProvider* const dpp = &dp;
-  lua_state_guard lguard(cct->_conf->rgw_lua_max_memory_per_state, dpp);
+  lua_state_guard lguard(cct->_conf->rgw_lua_max_memory_per_state,
+                         cct->_conf->rgw_lua_max_runtime_per_state, dpp);
   auto L = lguard.get();
   if (!L) {
     ldpp_dout(dpp, 1) << "Failed to create state for Lua background thread" << dendl;

src/rgw/rgw_lua_data_filter.cc

@@ -74,7 +74,8 @@ struct BufferlistMetaTable : public EmptyMetaTable {
 };
 
 int RGWObjFilter::execute(bufferlist& bl, off_t offset, const char* op_name) const {
-  lua_state_guard lguard(s->cct->_conf->rgw_lua_max_memory_per_state, s);
+  lua_state_guard lguard(s->cct->_conf->rgw_lua_max_memory_per_state,
+                         s->cct->_conf->rgw_lua_max_runtime_per_state, s);
   auto L = lguard.get();
   if (!L) {
     ldpp_dout(s, 1) << "Failed to create state for Lua data context" << dendl;

src/rgw/rgw_lua_request.cc

@@ -789,7 +789,8 @@ int execute(
     RGWOp* op,
     const std::string& script)
 {
-  lua_state_guard lguard(s->cct->_conf->rgw_lua_max_memory_per_state, s);
+  lua_state_guard lguard(s->cct->_conf->rgw_lua_max_memory_per_state,
+                         s->cct->_conf->rgw_lua_max_runtime_per_state, s);
   auto L = lguard.get();
   if (!L) {
     ldpp_dout(s, 1) << "Failed to create state for Lua request context" << dendl;

src/rgw/rgw_lua_utils.cc

@@ -136,12 +136,21 @@ lua_State* newstate(int max_memory) {
 }
 
 // lua_state_guard ctor
-lua_state_guard::lua_state_guard(std::size_t  _max_memory, const DoutPrefixProvider* _dpp) :
-  max_memory(_max_memory),
-  dpp(_dpp),
-  state(newstate(_max_memory)) {
-  if (state &&  perfcounter) {
-    perfcounter->inc(l_rgw_lua_current_vms, 1);
+lua_state_guard::lua_state_guard(std::size_t _max_memory,
+                                 std::uint64_t _max_runtime,
+                                 const DoutPrefixProvider* _dpp)
+    : max_memory(_max_memory),
+      max_runtime(std::chrono::milliseconds(_max_runtime)),
+      start_time(ceph::real_clock::now()),
+      dpp(_dpp),
+      state(newstate(_max_memory)) {
+  if (state) {
+    if (max_runtime.count() > 0) {
+      set_runtime_hook();
+    }
+    if (perfcounter) {
+      perfcounter->inc(l_rgw_lua_current_vms, 1);
+    }
   }
 }
 
@@ -162,6 +171,8 @@ lua_state_guard::~lua_state_guard() {
     // dont limit memory during cleanup
     *remaining_memory = 0;
   }
+  // clear any runtime hooks
+  lua_sethook(L, nullptr, 0, 0);
   try {
     lua_close(L);
   } catch (const std::runtime_error& e) {
@@ -177,5 +188,36 @@ lua_state_guard::~lua_state_guard() {
   }
 }
 
+void lua_state_guard::runtime_hook(lua_State* L, lua_Debug* ar) {
+  auto now = ceph::real_clock::now();
+  lua_getfield(L, LUA_REGISTRYINDEX, max_runtime_key);
+  auto max_runtime =
+      *static_cast<std::chrono::milliseconds*>(lua_touserdata(L, -1));
+  lua_getfield(L, LUA_REGISTRYINDEX, start_time_key);
+  auto start_time =
+      *static_cast<ceph::real_clock::time_point*>(lua_touserdata(L, -1));
+  lua_pop(L, 2);
+  auto elapsed =
+      std::chrono::duration_cast<std::chrono::milliseconds>(now - start_time);
+
+  if (elapsed > max_runtime) {
+    std::string err = "Lua runtime limit exceeded: total elapsed time is " +
+                      std::to_string(elapsed.count()) + " ms";
+    luaL_error(L, "%s", err.c_str());
+  }
+}
+
+void lua_state_guard::set_runtime_hook() {
+  lua_pushlightuserdata(state,
+                        const_cast<std::chrono::milliseconds*>(&max_runtime));
+  lua_setfield(state, LUA_REGISTRYINDEX, max_runtime_key);
+  lua_pushlightuserdata(state,
+                        const_cast<ceph::real_clock::time_point*>(&start_time));
+  lua_setfield(state, LUA_REGISTRYINDEX, start_time_key);
+
+  // Check runtime after each line or every 1000 VM instructions
+  lua_sethook(state, runtime_hook, LUA_MASKLINE | LUA_MASKCOUNT, 1000);
+}
+
 } // namespace rgw::lua
 

src/rgw/rgw_lua_utils.h

@@ -9,9 +9,11 @@
 #include <string_view>
 #include <ctime>
 #include <lua.hpp>
+#include <chrono>
 
 #include "include/common_fwd.h"
 #include "rgw_perf_counters.h"
+#include <common/ceph_time.h>
 
 // a helper type traits structs for detecting std::variant
 template<class>
@@ -67,16 +69,27 @@ void stack_dump(lua_State* L);
 
 class lua_state_guard {
   const std::size_t max_memory;
+  const std::chrono::milliseconds max_runtime;
+  const ceph::real_clock::time_point start_time;
   const DoutPrefixProvider* const dpp;
   lua_State* const state;
-public:
-  lua_state_guard(std::size_t _max_memory, const DoutPrefixProvider* _dpp);
+
+  static void runtime_hook(lua_State* L, lua_Debug* ar);
+  void set_runtime_hook();
+
+ public:
+  lua_state_guard(std::size_t _max_memory, std::uint64_t _max_runtime,
+                  const DoutPrefixProvider* _dpp);
   ~lua_state_guard();
   lua_State* get() { return state; }
 };
 
 int dostring(lua_State* L, const char* str);
 
+// keys for the lua registry
+static constexpr const char* max_runtime_key = "runtimeguard_max_runtime";
+static constexpr const char* start_time_key = "runtimeguard_start_time";
+
 constexpr const int MAX_LUA_VALUE_SIZE = 1000;
 constexpr const int MAX_LUA_KEY_ENTRIES = 100000;
 

src/test/rgw/test_rgw_lua.cc

@@ -1613,6 +1613,50 @@ TEST(TestRGWLua, MemoryLimit)
   ASSERT_NE(rc, 0);
 }
 
+TEST(TestRGWLua, LuaRuntimeLimit)
+{
+  std::string script = "print(\"hello world\")";
+  
+  DEFINE_REQ_STATE;
+
+  // runtime should be sufficient
+  s.cct->_conf->rgw_lua_max_runtime_per_state = 1000; // 1 second runtime limit
+  int rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+  ASSERT_EQ(rc, 0);
+
+  // no runtime limit
+  s.cct->_conf->rgw_lua_max_runtime_per_state = 0;
+  rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+  ASSERT_EQ(rc, 0);
+  
+  // script should exceed the runtime limit
+  script = R"(
+    local t = 0
+    for i = 1, 1e8 do
+      t = t + i
+    end
+  )";
+ 
+  s.cct->_conf->rgw_lua_max_runtime_per_state = 10; // 10 milliseconds runtime limit
+  rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+  ASSERT_NE(rc, 0);
+
+  s.cct->_conf->rgw_lua_max_runtime_per_state = 0; // no runtime limit
+  rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+  ASSERT_EQ(rc, 0);
+
+  // script should exceed the runtime limit
+    script = R"(
+    for i = 1, 10 do
+      os.execute("sleep 1")
+    end
+  )";
+ 
+  s.cct->_conf->rgw_lua_max_runtime_per_state = 5000; // 5 seconds runtime limit
+  rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+  ASSERT_NE(rc, 0);
+}
+
 TEST(TestRGWLua, DifferentContextUser)
 {
   const std::string script = R"(