Add std.encodeUTF8, std.decodeUTF8 (#577)

Author: sparkprime

core/desugarer.cpp

@@ -34,7 +34,7 @@ struct BuiltinDecl {
     std::vector<UString> params;
 };
 
-static unsigned long max_builtin = 35;
+static unsigned long max_builtin = 37;
 BuiltinDecl jsonnet_builtin_decl(unsigned long builtin)
 {
     switch (builtin) {
@@ -74,6 +74,8 @@ BuiltinDecl jsonnet_builtin_decl(unsigned long builtin)
         case 33: return {U"asciiUpper", {U"str"}};
         case 34: return {U"join", {U"sep", U"arr"}};
         case 35: return {U"parseJson", {U"str"}};
+        case 36: return {U"encodeUTF8", {U"str"}};
+        case 37: return {U"decodeUTF8", {U"arr"}};
         default:
             std::cerr << "INTERNAL ERROR: Unrecognized builtin function: " << builtin << std::endl;
             std::abort();

core/vm.cpp

@@ -73,6 +73,7 @@ enum FrameKind {
     FRAME_UNARY,                // e in -e
     FRAME_BUILTIN_JOIN_STRINGS, // When executing std.join over strings, used to hold intermediate state.
     FRAME_BUILTIN_JOIN_ARRAYS,  // When executing std.join over arrays, used to hold intermediate state.
+    FRAME_BUILTIN_DECODE_UTF8,  // When executing std.decodeUTF8, used to hold intermediate state.
 };
 
 /** A frame on the stack.
@@ -135,6 +136,9 @@ struct Frame {
     UString str;
     bool first;
 
+    /** Used for accumulating bytes */
+    std::string bytes;
+
     /** The context is used in error messages to attempt to find a reasonable name for the
      * object, function, or thunk value being executed.  If it is a thunk, it is filled
      * with the value when the frame terminates.
@@ -875,6 +879,8 @@ class Interpreter {
         builtins["asciiUpper"] = &Interpreter::builtinAsciiUpper;
         builtins["join"] = &Interpreter::builtinJoin;
         builtins["parseJson"] = &Interpreter::builtinParseJson;
+        builtins["encodeUTF8"] = &Interpreter::builtinEncodeUTF8;
+        builtins["decodeUTF8"] = &Interpreter::builtinDecodeUTF8;
     }
 
     /** Clean up the heap, stack, stash, and builtin function ASTs. */
@@ -1308,6 +1314,65 @@ class Interpreter {
         return nullptr;
     }
 
+    const AST *builtinEncodeUTF8(const LocationRange &loc, const std::vector<Value> &args)
+    {
+        validateBuiltinArgs(loc, "encodeUTF8", args, {Value::STRING});
+
+        std::string byteString = encode_utf8(static_cast<HeapString *>(args[0].v.h)->value);
+
+        scratch = makeArray({});
+        auto &elements = static_cast<HeapArray *>(scratch.v.h)->elements;
+        for (const auto c : byteString) {
+            auto *th = makeHeap<HeapThunk>(idArrayElement, nullptr, 0, nullptr);
+            elements.push_back(th);
+            th->fill(makeNumber(uint8_t(c)));
+        }
+        return nullptr;
+    }
+
+    const AST *decodeUTF8(void)
+    {
+        Frame &f = stack.top();
+        const auto& elements = static_cast<HeapArray*>(f.val.v.h)->elements;
+        while (f.elementId < elements.size()) {
+            auto *th = elements[f.elementId];
+            if (th->filled) {
+                auto b = th->content;
+                if (b.t != Value::NUMBER) {
+                    std::stringstream ss;
+                    ss << "Element " << f.elementId << " of the provided array was not a number";
+                    throw makeError(stack.top().location, ss.str());
+                } else {
+                    double d = b.v.d;
+                    if (d < 0 || d > 255 || d != int(d)) {
+                        std::stringstream ss;
+                        ss << "Element " << f.elementId << " of the provided array was not an integer in range [0,255]";
+                        throw makeError(stack.top().location, ss.str());
+                    }
+                    f.bytes.push_back(uint8_t(d));
+                }
+                f.elementId++;
+            } else {
+                stack.newCall(f.location, th, th->self, th->offset, th->upValues);
+                return th->body;
+            }
+        }
+        scratch = makeString(decode_utf8(f.bytes));
+        return nullptr;
+    }
+
+    const AST *builtinDecodeUTF8(const LocationRange &loc, const std::vector<Value> &args)
+    {
+        validateBuiltinArgs(loc, "decodeUTF8", args, {Value::ARRAY});
+
+        Frame &f = stack.top();
+        f.kind = FRAME_BUILTIN_DECODE_UTF8;
+        f.val = args[0]; // arr
+        f.bytes.clear();
+        f.elementId = 0;
+        return decodeUTF8();
+    }
+
     const AST *builtinTrace(const LocationRange &loc, const std::vector<Value> &args)
     {
         if(args[0].t != Value::STRING) {
@@ -2855,6 +2920,14 @@ class Interpreter {
                     }
                 } break;
 
+                 case FRAME_BUILTIN_DECODE_UTF8: {
+                    auto *ast = decodeUTF8();
+                    if (ast != nullptr) {
+                        ast_ = ast;
+                        goto recurse;
+                    }
+                } break;
+
                 default:
                     std::cerr << "INTERNAL ERROR: Unknown FrameKind:  " << f.kind << std::endl;
                     std::abort();

doc/ref/stdlib.html

@@ -863,6 +863,11 @@ <h4 id="parseHex">std.parseHex(str)</h4>
   <div class="hgroup-inline">
     <div class="panel">
       <h4 id="parseJson">std.parseJson(str)</h4>
+      <p>
+        Parses a JSON string.
+      </p>
+      <em>Available in upcoming release.</em>
+      <p>Example: <code>std.parseJson('{"foo": "bar"}')</code> yields <code>{"foo": "bar"}</code>.</p>
     </div>
     <div style="clear: both"></div>
   </div>
@@ -871,11 +876,32 @@ <h4 id="parseJson">std.parseJson(str)</h4>
 <div class="hgroup">
   <div class="hgroup-inline">
     <div class="panel">
+      <h4 id="encodeUTF8">std.encodeUTF8(str)</h4>
+      <em>Available in upcoming release.</em>
       <p>
-        Parses a JSON string.
+        Encode a string using <a href="https://en.wikipedia.org/wiki/UTF-8">UTF8</a>. Returns an array of numbers representing bytes.
       </p>
+    </div>
+    <div style="clear: both"></div>
+  </div>
+</div>
+
+<div class="hgroup">
+  <div class="hgroup-inline">
+    <div class="panel">
+      <h4 id="decodeUTF8">std.decodeUTF8(arr)</h4>
+    </div>
+    <div style="clear: both"></div>
+  </div>
+</div>
+
+<div class="hgroup">
+  <div class="hgroup-inline">
+    <div class="panel">
       <em>Available in upcoming release.</em>
-      <p>Example: <code>std.parseJson('{"foo": "bar"}')</code> yields <code>{"foo": "bar"}</code>.</p>
+      <p>
+        Decode an array of numbers representing bytes using <a href="https://en.wikipedia.org/wiki/UTF-8">UTF8</a>. Returns a string.
+      </p>
     </div>
     <div style="clear: both"></div>
   </div>

test_suite/error.decodeUTF8_float.jsonnet

@@ -0,0 +1 @@
+std.decodeUTF8([17.5])

test_suite/error.decodeUTF8_float.jsonnet.golden

@@ -0,0 +1,2 @@
+RUNTIME ERROR: Element 0 of the provided array was not an integer in range [0,255]
+	error.decodeUTF8_float.jsonnet:1:1-23	

test_suite/error.decodeUTF8_nan.jsonnet

@@ -0,0 +1 @@
+std.decodeUTF8(['foo'])

test_suite/error.decodeUTF8_nan.jsonnet.golden

@@ -0,0 +1,2 @@
+RUNTIME ERROR: Element 0 of the provided array was not a number
+	error.decodeUTF8_nan.jsonnet:1:1-24	

test_suite/stdlib.jsonnet

@@ -880,4 +880,19 @@ std.assertEqual(std.find('a', ['a']), [0]) &&
 std.assertEqual(std.find('a', ['a', ['a'], 'b', 'a']), [0, 3]) &&
 std.assertEqual(std.find(['a'], [['a']]), [0]) &&
 
+std.assertEqual(std.encodeUTF8(''), []) &&
+std.assertEqual(std.encodeUTF8('A'), [65]) &&
+std.assertEqual(std.encodeUTF8('AAA'), [65, 65, 65]) &&
+std.assertEqual(std.encodeUTF8('§'), [194, 167]) &&
+std.assertEqual(std.encodeUTF8('Zażółć gęślą jaźń'), [90, 97, 197, 188, 195, 179, 197, 130, 196, 135, 32, 103, 196, 153, 197, 155, 108, 196, 133, 32, 106, 97, 197, 186, 197, 132]) &&
+std.assertEqual(std.encodeUTF8('😃'), [240, 159, 152, 131]) &&
+
+std.assertEqual(std.decodeUTF8([]), '') &&
+std.assertEqual(std.decodeUTF8([65]), 'A') &&
+std.assertEqual(std.decodeUTF8([65, 65, 65]), 'AAA') &&
+std.assertEqual(std.decodeUTF8([(function(x) 65)(42)]), 'A') &&
+std.assertEqual(std.decodeUTF8([65 + 1 - 1]), 'A') &&
+std.assertEqual(std.decodeUTF8([90, 97, 197, 188, 195, 179, 197, 130, 196, 135, 32, 103, 196, 153, 197, 155, 108, 196, 133, 32, 106, 97, 197, 186, 197, 132]), 'Zażółć gęślą jaźń') &&
+std.assertEqual(std.decodeUTF8([240, 159, 152, 131]), '😃') &&
+
 true