Add importbin statement

Add `importbin`.  Like `importstr` but returns an array of
numbers (integers 0-255).

Note! This changes the `JsonnetImportCallback` function type
signature, and requires modification to existing code that uses custom
import callbacks.  The included python bindings have been updated.

Author: anguslees

case_studies/micromanage/validate.py

@@ -38,8 +38,8 @@ def err(path, msg, note=None):
     raise ConfigError('%s: %s' % (render_path(path), msg), note)
 
 _KEYWORDS = {
-    'import', 'importstr', 'function', 'self', 'super', 'assert', 'if', 'then',
-    'else', 'for', 'in', 'local', 'tailstrict', 'true', 'false', 'null', 'error',
+    'import', 'importstr', 'importbin', 'function', 'self', 'super', 'assert', 'if',
+    'then', 'else', 'for', 'in', 'local', 'tailstrict', 'true', 'false', 'null', 'error',
 }
 
 def _isidentifier(name):

core/ast.h

@@ -45,6 +45,7 @@ enum ASTType {
     AST_FUNCTION,
     AST_IMPORT,
     AST_IMPORTSTR,
+    AST_IMPORTBIN,
     AST_INDEX,
     AST_IN_SUPER,
     AST_LITERAL_BOOLEAN,
@@ -80,6 +81,7 @@ static inline std::string ASTTypeToString(ASTType type)
         case AST_FUNCTION: return "AST_FUNCTION";
         case AST_IMPORT: return "AST_IMPORT";
         case AST_IMPORTSTR: return "AST_IMPORTSTR";
+        case AST_IMPORTBIN: return "AST_IMPORTBIN";
         case AST_INDEX: return "AST_INDEX";
         case AST_IN_SUPER: return "AST_IN_SUPER";
         case AST_LITERAL_BOOLEAN: return "AST_LITERAL_BOOLEAN";
@@ -455,6 +457,15 @@ struct Importstr : public AST {
     }
 };
 
+/** Represents importbin "file". */
+struct Importbin : public AST {
+    LiteralString *file;
+    Importbin(const LocationRange &lr, const Fodder &open_fodder, LiteralString *file)
+        : AST(lr, AST_IMPORTBIN, open_fodder), file(file)
+    {
+    }
+};
+
 /** Represents both e[e] and the syntax sugar e.f.
  *
  * One of index and id will be nullptr before desugaring.  After desugaring id will be nullptr.
@@ -647,8 +658,8 @@ struct ObjectField {
 
     ObjectField(enum Kind kind, const Fodder &fodder1, const Fodder &fodder2,
                 const Fodder &fodder_l, const Fodder &fodder_r, enum Hide hide, bool super_sugar,
-                bool method_sugar, AST *expr1, const Identifier *id, const LocationRange &id_lr, 
-                const ArgParams &params, bool trailing_comma, const Fodder &op_fodder, AST *expr2, 
+                bool method_sugar, AST *expr1, const Identifier *id, const LocationRange &id_lr,
+                const ArgParams &params, bool trailing_comma, const Fodder &op_fodder, AST *expr2,
                 AST *expr3, const Fodder &comma_fodder)
         : kind(kind),
           fodder1(fodder1),

core/desugarer.cpp

@@ -772,6 +772,12 @@ class Desugarer {
             desugar(file, obj_level);
             ast->file = dynamic_cast<LiteralString *>(file);
 
+        } else if (auto *ast = dynamic_cast<Importbin *>(ast_)) {
+            // TODO(dcunnin): Abstract this into a template function if it becomes more common.
+            AST *file = ast->file;
+            desugar(file, obj_level);
+            ast->file = dynamic_cast<LiteralString *>(file);
+
         } else if (auto *ast = dynamic_cast<InSuper *>(ast_)) {
             desugar(ast->element, obj_level);
 

core/formatter.cpp

@@ -67,7 +67,7 @@ static AST *left_recursive_deep(AST *ast_)
  * \param fodder The fodder to print
  * \param space_before Whether a space should be printed before any other output.
  * \param separate_token If the last fodder was an interstitial, whether a space should follow it.
- * \param final Whether fodder is the last one in 
+ * \param final Whether fodder is the last one in
  */
 void fodder_fill(std::ostream &o, const Fodder &fodder, bool space_before, bool separate_token, bool final)
 {
@@ -414,6 +414,10 @@ class Unparser {
             o << "importstr";
             unparse(ast->file, true);
 
+        } else if (auto *ast = dynamic_cast<const Importbin *>(ast_)) {
+            o << "importbin";
+            unparse(ast->file, true);
+
         } else if (auto *ast = dynamic_cast<const InSuper *>(ast_)) {
             unparse(ast->element, true);
             fill(ast->inFodder, true, true);
@@ -1784,6 +1788,11 @@ class FixIndentation {
             Indent new_indent = newIndent(open_fodder(ast->file), indent, column + 1);
             expr(ast->file, new_indent, true);
 
+        } else if (auto *ast = dynamic_cast<Importbin *>(ast_)) {
+            column += 9;  // importbin
+            Indent new_indent = newIndent(open_fodder(ast->file), indent, column + 1);
+            expr(ast->file, new_indent, true);
+
         } else if (auto *ast = dynamic_cast<InSuper *>(ast_)) {
             expr(ast->element, indent, space_before);
             fill(ast->inFodder, true, true, indent.lineUp);

core/lexer.cpp

@@ -190,6 +190,7 @@ static const std::map<std::string, Token::Kind> keywords = {
     {"if", Token::IF},
     {"import", Token::IMPORT},
     {"importstr", Token::IMPORTSTR},
+    {"importbin", Token::IMPORTBIN},
     {"in", Token::IN},
     {"local", Token::LOCAL},
     {"null", Token::NULL_LIT},

core/lexer.h

@@ -263,6 +263,7 @@ struct Token {
         IF,
         IMPORT,
         IMPORTSTR,
+        IMPORTBIN,
         IN,
         LOCAL,
         NULL_LIT,
@@ -346,6 +347,7 @@ struct Token {
             case IF: return "if";
             case IMPORT: return "import";
             case IMPORTSTR: return "importstr";
+            case IMPORTBIN: return "importbin";
             case IN: return "in";
             case LOCAL: return "local";
             case NULL_LIT: return "null";

core/lexer_test.cpp

@@ -309,6 +309,7 @@ TEST(Lexer, TestKeywords)
     testLex("if", "if", {Token(Token::Kind::IF, "if")}, "");
     testLex("import", "import", {Token(Token::Kind::IMPORT, "import")}, "");
     testLex("importstr", "importstr", {Token(Token::Kind::IMPORTSTR, "importstr")}, "");
+    testLex("importbin", "importbin", {Token(Token::Kind::IMPORTBIN, "importbin")}, "");
     testLex("in", "in", {Token(Token::Kind::IN, "in")}, "");
     testLex("local", "local", {Token(Token::Kind::LOCAL, "local")}, "");
     testLex("null", "null", {Token(Token::Kind::NULL_LIT, "null")}, "");

core/libjsonnet.cpp

@@ -45,12 +45,20 @@ static void memory_panic(void)
 static char *from_string(JsonnetVm *vm, const std::string &v)
 {
     char *r = jsonnet_realloc(vm, nullptr, v.length() + 1);
-    std::strcpy(r, v.c_str());
+    std::memcpy(r, v.c_str(), v.length() + 1);
     return r;
 }
 
-static char *default_import_callback(void *ctx, const char *dir, const char *file,
-                                     char **found_here_cptr, int *success);
+static char *from_string_nonull(JsonnetVm *vm, const std::string &v, size_t *buflen)
+{
+    char *r = jsonnet_realloc(vm, nullptr, v.length());
+    std::memcpy(r, v.data(), v.length());
+    *buflen = v.length();
+    return r;
+}
+
+static int default_import_callback(void *ctx, const char *dir, const char *file,
+                                   char **found_here_cptr, char **buf, size_t *buflen);
 
 const char *jsonnet_json_extract_string(JsonnetVm *vm, const struct JsonnetJsonValue *v)
 {
@@ -229,8 +237,8 @@ static enum ImportStatus try_path(const std::string &dir, const std::string &rel
     return IMPORT_STATUS_OK;
 }
 
-static char *default_import_callback(void *ctx, const char *dir, const char *file,
-                                     char **found_here_cptr, int *success)
+static int default_import_callback(void *ctx, const char *dir, const char *file,
+                                   char **found_here_cptr, char **buf, size_t *buflen)
 {
     auto *vm = static_cast<JsonnetVm *>(ctx);
 
@@ -243,25 +251,23 @@ static char *default_import_callback(void *ctx, const char *dir, const char *fil
     // If not found, try library search path.
     while (status == IMPORT_STATUS_FILE_NOT_FOUND) {
         if (jpaths.size() == 0) {
-            *success = 0;
             const char *err = "no match locally or in the Jsonnet library paths.";
-            char *r = jsonnet_realloc(vm, nullptr, std::strlen(err) + 1);
-            std::strcpy(r, err);
-            return r;
+            *buf = from_string_nonull(vm, err, buflen);
+            return 1;  // failure
         }
         status = try_path(jpaths.back(), file, input, found_here, err_msg);
         jpaths.pop_back();
     }
 
     if (status == IMPORT_STATUS_IO_ERROR) {
-        *success = 0;
-        return from_string(vm, err_msg);
-    } else {
-        assert(status == IMPORT_STATUS_OK);
-        *success = 1;
-        *found_here_cptr = from_string(vm, found_here);
-        return from_string(vm, input);
+        *buf = from_string_nonull(vm, err_msg, buflen);
+        return 1;  // failure
     }
+
+    assert(status == IMPORT_STATUS_OK);
+    *found_here_cptr = from_string(vm, found_here);
+    *buf = from_string_nonull(vm, input, buflen);
+    return 0;  // success
 }
 
 #define TRY try {

core/parser.cpp

@@ -584,6 +584,7 @@ class Parser {
             case Token::IN:
             case Token::IMPORT:
             case Token::IMPORTSTR:
+            case Token::IMPORTBIN:
             case Token::LOCAL:
             case Token::PAREN_R:
             case Token::SEMICOLON:
@@ -860,6 +861,23 @@ class Parser {
                 }
             }
 
+            case Token::IMPORTBIN: {
+                pop();
+                AST *body = parse(MAX_PRECEDENCE);
+                if (body->type == AST_LITERAL_STRING) {
+                    auto *lit = static_cast<LiteralString *>(body);
+                    if (lit->tokenKind == LiteralString::BLOCK) {
+                        throw StaticError(lit->location,
+                                          "Cannot use text blocks in import statements.");
+                    }
+                    return alloc->make<Importbin>(span(begin, body), begin.fodder, lit);
+                } else {
+                    std::stringstream ss;
+                    ss << "computed imports are not allowed.";
+                    throw StaticError(body->location, ss.str());
+                }
+            }
+
             case Token::LOCAL: {
                 pop();
                 Local::Binds binds;
@@ -934,7 +952,7 @@ class Parser {
                     op_precedence = APPLY_PRECEDENCE;
                     break;
 
-                default: 
+                default:
                     // This happens when we reach EOF or the terminating token of an outer context.
                     return lhs;
             }

core/parser_test.cpp

@@ -118,6 +118,7 @@ TEST(Parser, TestTuple)
 
     testParse("import 'foo.jsonnet'");
     testParse("importstr 'foo.text'");
+    testParse("importbin 'foo.text'");
 
     testParse("{a: b} + {c: d}");
     testParse("{a: b}{c: d}");
@@ -315,6 +316,9 @@ TEST(Parser, TestInvalidImport)
     testParseError("importstr (a b)",
                    R"_(test:1:14: expected token ")" but got (IDENTIFIER, "b"))_");
     testParseError("importstr (a+b)", "test:1:11-16: computed imports are not allowed.");
+    testParseError("importbin (a b)",
+                   R"_(test:1:14: expected token ")" but got (IDENTIFIER, "b"))_");
+    testParseError("importbin (a+b)", "test:1:11-16: computed imports are not allowed.");
 }
 
 TEST(Parser, TestInvalidOperator)