Implement remaining Lua runtime libraries (#1075)

* punch an allow list in for Lua functions

* implement lua function allow listing

* test to make sure we're exporting exactly what Redis does by default, and we can configure it reasonably; implements a number of workarounds to accomplish this, as the 5.1 <-> 5.4 Lua version difference also had some library changes

* implement cjson.encode and decode

* implement bit.xxx operations in Lua scripts; the rules around these are pretty sloppy, so ran tests against Redis first to verify matching behavior

* fix typo in AllowedFunctions

* implement message pack support (cmsgpack.pack and cmsgpack.unpack)

* implement struct.pack/unpack/size; these are provided by Lua and are a superset of what Redis wants; technically we allow more than Redis does, but I think that's a reasonable tradeoff for implementation complexity

* allow redis functions to be allowed/denied rather than unconditionally included

* fix formatting

* Move intermediate binary blobs onto ScratchBufferManager

* formatting

* strange this doesn't error locally; removing preview use of params ReadOnlySpan<T>

* test for #1079, fails in main and passes here

* tests (and fixes) for math.frexp and math.ldexp

* tests for table.maxn and loadstring

* prevent LuaAllowedFunctions from importing functions not on the default import list

* enforce maximum depth for cjson functions

* enforce maximum encoding depth for cmsgpack.pack

* hey, a reliable-ish repro for the linux longjmp issue; disable these exception tests on Linux until that refactor happens

* condition more exception tests on 'we're on Windows' until the refactor comes in

* knock out a missing TODO; remove List<byte> and use scratch buffer instead

* Update test/Garnet.test/LuaScriptRunnerTests.cs

Co-authored-by: Copilot <[email protected]>

* add more conditions to skip exceptions in .NET 9 or non-Windows builds; consolidate condition checks; restore non-error-y parts of tests in .NET 9 and non-Windows builds

---------

Co-authored-by: Copilot <[email protected]>

Author: kevin-montrose

benchmark/BDN.benchmark/Lua/LuaParams.cs

@@ -29,7 +29,7 @@ public LuaParams(LuaMemoryManagementMode mode, bool memoryLimit, TimeSpan? timeo
         /// Get the equivalent <see cref="LuaOptions"/>.
         /// </summary>
         public LuaOptions CreateOptions()
-        => new(Mode, MemoryLimit ? "2m" : "", Timeout ?? System.Threading.Timeout.InfiniteTimeSpan, LuaLoggingMode.Enable);
+        => new(Mode, MemoryLimit ? "2m" : "", Timeout ?? System.Threading.Timeout.InfiniteTimeSpan, LuaLoggingMode.Enable, []);
 
         /// <summary>
         /// String representation

benchmark/BDN.benchmark/Operations/OperationsBase.cs

@@ -56,7 +56,7 @@ public virtual void GlobalSetup()
                 QuietMode = true,
                 EnableLua = true,
                 DisablePubSub = true,
-                LuaOptions = new(LuaMemoryManagementMode.Native, "", Timeout.InfiniteTimeSpan, LuaLoggingMode.Enable),
+                LuaOptions = new(LuaMemoryManagementMode.Native, "", Timeout.InfiniteTimeSpan, LuaLoggingMode.Enable, []),
             };
 
             if (Params.useAof)

libs/host/Configuration/Options.cs

@@ -560,6 +560,27 @@ internal sealed class Options
         [Option("lua-logging-mode", Required = false, HelpText = "Behavior of redis.log(...) when called from Lua scripts.  Defaults to Enable.")]
         public LuaLoggingMode LuaLoggingMode { get; set; }
 
+        // Parsing is a tad tricky here as JSON wants to set to empty at certain points
+        //
+        // A bespoke union-on-set gets the desired semantics.
+        private readonly HashSet<string> luaAllowedFunctions = [];
+
+        [OptionValidation]
+        [Option("lua-allowed-functions", Separator = ',', Required = false, HelpText = "If set, restricts the functions available in Lua scripts to given list.")]
+        public IEnumerable<string> LuaAllowedFunctions
+        {
+            get => luaAllowedFunctions;
+            set
+            {
+                if (value == null)
+                {
+                    return;
+                }
+
+                luaAllowedFunctions.UnionWith(value);
+            }
+        }
+
         [FilePathValidation(false, true, false)]
         [Option("unixsocket", Required = false, HelpText = "Unix socket address path to bind server to")]
         public string UnixSocketPath { get; set; }
@@ -811,7 +832,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
                 LoadModuleCS = LoadModuleCS,
                 FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault(),
                 SkipRDBRestoreChecksumValidation = SkipRDBRestoreChecksumValidation.GetValueOrDefault(),
-                LuaOptions = EnableLua.GetValueOrDefault() ? new LuaOptions(LuaMemoryManagementMode, LuaScriptMemoryLimit, LuaScriptTimeoutMs == 0 ? Timeout.InfiniteTimeSpan : TimeSpan.FromMilliseconds(LuaScriptTimeoutMs), LuaLoggingMode, logger) : null,
+                LuaOptions = EnableLua.GetValueOrDefault() ? new LuaOptions(LuaMemoryManagementMode, LuaScriptMemoryLimit, LuaScriptTimeoutMs == 0 ? Timeout.InfiniteTimeSpan : TimeSpan.FromMilliseconds(LuaScriptTimeoutMs), LuaLoggingMode, LuaAllowedFunctions, logger) : null,
                 UnixSocketPath = UnixSocketPath,
                 UnixSocketPermission = unixSocketPermissions
             };

libs/host/defaults.conf

@@ -380,6 +380,9 @@
 	/* Allow redis.log(...) to write to the Garnet logs */
 	"LuaLoggingMode": "Enable",
 
+	/* Allow all built in and redis.* functions by default */
+	"LuaAllowedFunctions": null,
+
 	/* Unix socket address path to bind the server to */
 	"UnixSocketPath": null,
 

libs/server/ArgSlice/ScratchBufferManager.cs

@@ -114,6 +114,27 @@ public ArgSlice CreateArgSlice(string str)
             return retVal;
         }
 
+        public ReadOnlySpan<byte> UTF8EncodeString(string str)
+        {
+            // We'll always need AT LEAST this many bytes
+            ExpandScratchBufferIfNeeded(str.Length);
+
+            var space = FullBuffer()[scratchBufferOffset..];
+
+            // Attempt to fit in the existing buffer first
+            if (!Encoding.UTF8.TryGetBytes(str, space, out var written))
+            {
+                // If that fails, figure out exactly how much space we need
+                var neededBytes = Encoding.UTF8.GetByteCount(str);
+                ExpandScratchBufferIfNeeded(neededBytes);
+
+                space = FullBuffer()[scratchBufferOffset..];
+                written = Encoding.UTF8.GetBytes(str, space);
+            }
+
+            return space[..written];
+        }
+
         /// <summary>
         /// Create an ArgSlice that includes a header of specified size, followed by RESP Bulk-String formatted versions of the specified ArgSlice values (arg1 and arg2)
         /// </summary>

libs/server/Lua/LuaOptions.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.Runtime.InteropServices;
 using Microsoft.Extensions.Logging;
 
@@ -18,6 +19,7 @@ public sealed class LuaOptions
         public string MemoryLimit = "";
         public TimeSpan Timeout = System.Threading.Timeout.InfiniteTimeSpan;
         public LuaLoggingMode LogMode = LuaLoggingMode.Silent;
+        public HashSet<string> AllowedFunctions = [];
 
         /// <summary>
         /// Construct options with default options.
@@ -30,12 +32,13 @@ public LuaOptions(ILogger logger = null)
         /// <summary>
         /// Construct options with specific settings.
         /// </summary>
-        public LuaOptions(LuaMemoryManagementMode memoryMode, string memoryLimit, TimeSpan timeout, LuaLoggingMode logMode, ILogger logger = null) : this(logger)
+        public LuaOptions(LuaMemoryManagementMode memoryMode, string memoryLimit, TimeSpan timeout, LuaLoggingMode logMode, IEnumerable<string> allowedFunctions, ILogger logger = null) : this(logger)
         {
             MemoryManagementMode = memoryMode;
             MemoryLimit = memoryLimit;
             Timeout = timeout;
             LogMode = logMode;
+            AllowedFunctions = new HashSet<string>(allowedFunctions, StringComparer.OrdinalIgnoreCase);
         }
 
         /// <summary>