Interpreter: Add a limit to how much we try to grow memory, to avoid DOS (#3227)

growMemory() now also returns whether we succeeded.

Without this it could eventually start to swap etc., which is annoying.

Author: kripken

src/shell-interface.h

@@ -217,8 +217,14 @@ struct ShellExternalInterface : ModuleInstance::ExternalInterface {
 
   void tableStore(Address addr, Name entry) override { table[addr] = entry; }
 
-  void growMemory(Address /*oldSize*/, Address newSize) override {
+  bool growMemory(Address /*oldSize*/, Address newSize) override {
+    // Apply a reasonable limit on memory size, 1GB, to avoid DOS on the
+    // interpreter.
+    if (newSize > 1024 * 1024 * 1024) {
+      return false;
+    }
     memory.resize(newSize);
+    return true;
   }
 
   void trap(const char* why) override {

src/tools/wasm-ctor-eval.cpp

@@ -283,7 +283,7 @@ struct CtorEvalExternalInterface : EvallingModuleInstance::ExternalInterface {
   // called during initialization, but we don't keep track of a table
   void tableStore(Address addr, Name value) override {}
 
-  void growMemory(Address /*oldSize*/, Address newSize) override {
+  bool growMemory(Address /*oldSize*/, Address newSize) override {
     throw FailToEvalException("grow memory");
   }
 

src/wasm-interpreter.h

@@ -1685,7 +1685,7 @@ template<typename GlobalManager, typename SubType> class ModuleInstanceBase {
                                LiteralList& arguments,
                                Type result,
                                SubType& instance) = 0;
-    virtual void growMemory(Address oldSize, Address newSize) = 0;
+    virtual bool growMemory(Address oldSize, Address newSize) = 0;
     virtual void trap(const char* why) = 0;
     virtual void throwException(Literal exnref) = 0;
 
@@ -2406,8 +2406,13 @@ template<typename GlobalManager, typename SubType> class ModuleInstanceBase {
       if (newSize > instance.wasm.memory.max) {
         return fail;
       }
-      instance.externalInterface->growMemory(
-        instance.memorySize * Memory::kPageSize, newSize * Memory::kPageSize);
+      if (!instance.externalInterface->growMemory(
+            instance.memorySize * Memory::kPageSize,
+            newSize * Memory::kPageSize)) {
+        // We failed to grow the memory in practice, even though it was valid
+        // to try to do so.
+        return fail;
+      }
       instance.memorySize = newSize;
       return ret;
     }