[Tolk] Correctly handle variadic ints at serialization

Occasionally, `varint32` was not handled at packing to/from cells
and worked like a regular `int32`.
This MR fixes the issue, and also:
- adds missing varintN types
- fix intN/uintN limits
- rename bytesN to bitsN, since "bits" are more frequent in practice

Author: tolk-vm

crypto/smartcont/tolk-stdlib/common.tolk

@@ -227,8 +227,8 @@ struct PackOptions {
     /// however, if you guarantee that a slice is valid (for example, it comes from trusted sources),
     /// set this option to true to disable runtime checks;
     /// note: `int32` and other are always validated for overflow without any extra gas,
-    /// so this flag controls only rarely used `bytesN` / `bitsN` types
-    skipBitsNFieldsValidation: bool = false,
+    /// so this flag controls only rarely used `bitsN` type
+    skipBitsNValidation: bool = false,
 }
 
 /// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions.

tolk-tester/tests/intN-tests.tolk

@@ -80,7 +80,7 @@ fun test3(op: int32, qid: uint64) {
     if (op == qid) {}
     if ((op as int32?)! == qid) {}
     if (op == (qid as uint64)) {}
-    if ((op as uint257?)! != (qid as int256?)!) {}
+    if ((op as int257?)! != (qid as uint256?)!) {}
     __expect_type(op << qid, "int");
 
     takeAnyInt(op);
@@ -227,7 +227,7 @@ fun main() {
     assign0(mutate t.1 as uint16);
     assign0(mutate t.2 as int);
     __expect_type(t.0 as int1, "int1");
-    __expect_type(t.0 as uint257, "uint257");
+    __expect_type(t.0 as int257, "int257");
     __expect_type(0 as int32, "int32");
     __expect_type((0 as int32) as uint64?, "uint64?");
     __expect_type(null as (int8, [uint16?])?, "(int8, [uint16?])?");

tolk-tester/tests/lazy-load-tests.tolk

@@ -217,6 +217,11 @@ struct OneIntOneRef {
     r: cell;
 }
 
+struct WithVariadicInts {
+    i1: varuint32;
+    i2: varuint32;
+}
+
 
 @noinline
 fun contract.getFakeData(seqno: int): Cell<WalletStorage> {
@@ -655,7 +660,7 @@ fun demo135(skip: bool) {
     cc.bin16 = stringHexToSlice("00000000") as bits16;
     try {
         return cc.toCell({
-            skipBitsNFieldsValidation: skip
+            skipBitsNValidation: skip
         }).beginParse().remainingBitsCount();
     } catch (excno) {
         return excno;
@@ -939,6 +944,13 @@ fun demo156(s: slice): int {
     }
 }
 
+@method_id(157)
+fun demo157() {
+    var c: Cell<WithVariadicInts> = WithVariadicInts { i1: 1 << 100, i2: 1 << 200 }.toCell();
+    val d = lazy c.load();
+    return d.i2 == (1 << 200);
+}
+
 
 @method_id(200)
 fun demo200() {
@@ -1280,7 +1292,7 @@ fun demo310() {
         f9: "" as bits200,
         f10: ("" as bits400, ()),
         f11: 0,
-    }.toCell({skipBitsNFieldsValidation: true});
+    }.toCell({skipBitsNValidation: true});
     var st = lazy cell.load();
     var sm = lazy cell.load();
     var sk = lazy cell.load();
@@ -1625,6 +1637,7 @@ fun main() {
 @testcase | 155 |                 | 555
 @testcase | 156 | x{0110}         | 16
 @testcase | 156 | x{FF10}         | -100
+@testcase | 157 |                 | -1
 
 @testcase | 200 |                 | 1
 @testcase | 201 |                 | 1 2 -1

tolk-tester/tests/pack-unpack-2.tolk

@@ -334,6 +334,30 @@ struct DifferentMix3 {
     pairm: (int32, int64)?;
 }
 
+/*
+    ui16: VarUInteger 16
+    i16: VarInteger 16
+    ui32: VarUInteger 32
+    i32: VarInteger 32
+ */
+
+struct WithVariadicInts {
+    ui16: varuint16;
+    i16: varint16;
+    ui32: varuint32;
+    i32: varint32;
+}
+
+/*
+    Test MIN_INT and MAX_INT
+ */
+
+struct EdgeCaseInts {
+    maxUint: uint256 = +115792089237316195423570985008687907853269984665640564039457584007913129639935;
+    maxInt:   int257 = +115792089237316195423570985008687907853269984665640564039457584007913129639935;
+    minInt:   int257 = -115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1;
+}
+
 /*
     Test: write with builder, read with other struct
     (type `builder` is available for writing, but not for reading)
@@ -670,6 +694,27 @@ fun test_mutatingRemainder() {
     return obj.rest.remainingBitsCount();
 }
 
+@method_id(228)
+fun test_VariadicIntegers() {
+    var t: WithVariadicInts = {
+        ui16: (1 << 120) - 1,
+        i16: -(1 << 119),
+        ui32: (1 << 248) - 1,
+        i32: -(1 << 247),
+    };
+    run<WithVariadicInts>(t, stringHexToSlice("ffffffffffffffffffffffffffffffff800000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe000000000000000000000000000000000000000000000000000000000000020_"));
+    run<WithVariadicInts>({ui16: 0, i16: 0, ui32: 0, i32: 0}, stringHexToSlice("000020_"));
+
+    var c = t.toCell().load();
+    return (c.ui16 == t.ui16) & (c.i16 == t.i16) & (c.ui32 == t.ui32) & (c.i32 == t.i32)
+}
+
+@method_id(229)
+fun test_EdgeCaseIntegers() {
+    var edge: EdgeCaseInts = {};
+    var manual = beginCell().storeUint(edge.maxUint, 256).storeInt(edge.maxInt, 257).storeInt(edge.minInt, 257);
+    return edge.toCell().hash() == manual.endCell().hash()
+}
 
 fun main() {
     var t: JustInt32 = { value: 10 };
@@ -709,4 +754,6 @@ fun main() {
 @testcase | 225 | | 5 40 65535 8 50000000
 @testcase | 226 | | 1 16 0
 @testcase | 227 | | 16
+@testcase | 228 | | -1
+@testcase | 229 | | -1
  */

tolk-tester/tests/pack-unpack-5.tolk

@@ -22,6 +22,7 @@ fun test1() {
         coins.estimatePackSize(),
         bits6.estimatePackSize(),
         bytes8.estimatePackSize(),
+        varint32.estimatePackSize(),
     );
 }
 
@@ -204,7 +205,7 @@ fun main() {
 }
 
 /**
-@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ]
+@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] [ 5 253 0 0 ]
 @testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ]
 @testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ]
 @testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ]

tolk-tester/tests/pack-unpack-6.tolk

@@ -50,13 +50,13 @@ fun test5() {
 
 @method_id(106)
 fun test6() {
-    return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNFieldsValidation: true}).hash() & 0xFFFF;
+    return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNValidation: true}).hash() & 0xFFFF;
 }
 
 @method_id(107)
 fun test7(believe: bool) {
     try {
-        return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNFieldsValidation: believe}).hash() & 0xFFFF;
+        return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNValidation: believe}).hash() & 0xFFFF;
     } catch (excno) {
         return excno;
     }

tolk/builtins.cpp

@@ -1036,6 +1036,25 @@ static AsmOp compile_fetch_int(std::vector<VarDescr>& res, std::vector<VarDescr>
   return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch);
 }
 
+// fun slice.__loadVarInt(mutate self, bits: int, unsigned: bool): int
+static AsmOp compile_fetch_varint(std::vector<VarDescr>& res, std::vector<VarDescr>& args, SrcLocation loc) {
+  tolk_assert(args.size() == 3 && res.size() == 2);
+  // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic
+  tolk_assert(args[1].is_int_const() && args[2].is_int_const());
+  long n_bits = args[1].int_const->to_long();
+  long is_unsigned = args[2].int_const->to_long();
+
+  args[1].unused();
+  args[2].unused();
+  if (n_bits == 16) {
+    return exec_op(loc, is_unsigned ? "LDVARUINT16" : "LDVARINT16", 1, 2);
+  }
+  if (n_bits == 32) {
+    return exec_op(loc, is_unsigned ? "LDVARUINT32" : "LDVARINT32", 1, 2);
+  }
+  tolk_assert(false);
+}
+
 // fun builder.storeInt  (mutate self, x: int, len: int): self   asm(x b len) "STIX";
 // fun builder.storeUint (mutate self, x: int, len: int): self   asm(x b len) "STUX";
 static AsmOp compile_store_int(std::vector<VarDescr>& res, std::vector<VarDescr>& args, SrcLocation loc, bool sgnd) {
@@ -1063,6 +1082,25 @@ static AsmOp compile_store_int(std::vector<VarDescr>& res, std::vector<VarDescr>
   return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1);
 }
 
+// fun builder.__storeVarInt (mutate self, x: int, bits: int, unsigned: bool): self
+static AsmOp compile_store_varint(std::vector<VarDescr>& res, std::vector<VarDescr>& args, SrcLocation loc) {
+  tolk_assert(args.size() == 4 && res.size() == 1);
+  // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic
+  tolk_assert(args[2].is_int_const() && args[3].is_int_const());
+  long n_bits = args[2].int_const->to_long();
+  long is_unsigned = args[3].int_const->to_long();
+
+  args[2].unused();
+  args[3].unused();
+  if (n_bits == 16) {
+    return exec_op(loc, is_unsigned ? "STVARUINT16" : "STVARINT16", 2, 1);
+  }
+  if (n_bits == 32) {
+    return exec_op(loc, is_unsigned ? "STVARUINT32" : "STVARINT32", 2, 1);
+  }
+  tolk_assert(false);
+}
+
 // fun builder.storeBool(mutate self, value: bool): self   asm( -> 1 0) "1 STI";
 static AsmOp compile_store_bool(std::vector<VarDescr>& res, std::vector<VarDescr>& args, SrcLocation loc) {
   tolk_assert(args.size() == 2 && res.size() == 1);
@@ -1388,6 +1426,13 @@ void define_builtins() {
   define_builtin_func("__InMessage.getInMsgParam", ParamsInt1, Int, nullptr,
                                 compile_calc_InMessage_getInMsgParam,
                                 0);
+  define_builtin_method("builder.__storeVarInt", Builder, {Builder, Int, Int, Bool}, Unit, nullptr,
+                                compile_store_varint,   // not exposed to stdlib, used in auto-serialization
+                                FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf);
+  define_builtin_method("slice.__loadVarInt", Slice, {Slice, Int, Bool}, Int, nullptr,
+                                compile_fetch_varint,   // not exposed to stdlib, used in auto-serialization
+                                FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf,
+                                {}, {1, 0});
 
   // compile-time only functions, evaluated essentially at compile-time, no runtime implementation
   // they are placed in stdlib and marked as `builtin`

tolk/pack-unpack-api.cpp

@@ -58,7 +58,7 @@ class PackUnpackAvailabilityChecker {
     if (any_type->try_as<TypeDataIntN>()) {
       return {};
     }
-    if (any_type->try_as<TypeDataBytesN>()) {
+    if (any_type->try_as<TypeDataBitsN>()) {
       return {};
     }
     if (any_type == TypeDataCoins::create()) {
@@ -345,7 +345,7 @@ std::vector<var_idx_t> generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation
       ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_field));
     } else {
       std::vector ir_gap_or_tail = loaded_state->get_ir_loaded_aside_field(hidden_field);
-      if (hidden_field->declared_type->unwrap_alias()->try_as<TypeDataBytesN>()) {
+      if (hidden_field->declared_type->unwrap_alias()->try_as<TypeDataBitsN>()) {
         ctx.storeSlice(ir_gap_or_tail[0]);
       } else {
         ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_gap_or_tail));

tolk/pack-unpack-serializers.cpp

@@ -65,7 +65,7 @@ PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector<var_idx_t>
   , f_storeUint(lookup_function("builder.storeUint"))
   , ir_builder(std::move(ir_builder))
   , ir_builder0(this->ir_builder[0])
-  , option_skipBitsNFieldsValidation(ir_options[0]) {
+  , option_skipBitsNValidation(ir_options[0]) {
 }
 
 void PackContext::storeInt(var_idx_t ir_idx, int len) const {
@@ -264,16 +264,51 @@ struct S_IntN final : ISerializer {
   }
 };
 
-struct S_BytesN final : ISerializer {
+struct S_VariadicIntN final : ISerializer {
+  const int n_bits;           // only 16 and 32 available
+  const bool is_unsigned;
+
+  explicit S_VariadicIntN(int n_bits, bool is_unsigned)
+    : n_bits(n_bits), is_unsigned(is_unsigned) {}
+
+  void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector<var_idx_t>&& rvect) override {
+    FunctionPtr f_storeVarInt = lookup_function("builder.__storeVarInt");
+    std::vector args = { ctx->ir_builder0, rvect[0], code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") };
+    code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), f_storeVarInt);
+  }
+
+  std::vector<var_idx_t> unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override {
+    FunctionPtr f_loadVarInt = lookup_function("slice.__loadVarInt");
+    std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") };
+    std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-varint)");
+    code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, result[0]}, std::move(args), f_loadVarInt);
+    return result;
+  }
+
+  void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override {
+    // no TVM instruction to skip, just load but don't use the result
+    unpack(ctx, code, loc);
+  }
+
+  PackSize estimate(const EstimateContext* ctx) override {
+    if (n_bits == 32) {
+      return PackSize(5, 253);
+    } else {
+      return PackSize(4, 124);    // same as `coins`
+    }
+  }
+};
+
+struct S_BitsN final : ISerializer {
   const int n_bits;
 
-  explicit S_BytesN(int n_width, bool is_bits)
+  explicit S_BitsN(int n_width, bool is_bits)
     : n_bits(is_bits ? n_width : n_width * 8) {}
 
   void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector<var_idx_t>&& rvect) override {
     tolk_assert(rvect.size() == 1);
 
-    Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNFieldsValidation});
+    Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNValidation});
     {
       code.push_set_cur(if_disabled_by_user.block0);
       code.close_pop_cur(loc);
@@ -400,10 +435,8 @@ struct S_Coins final : ISerializer {
   }
 
   void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override {
-    FunctionPtr f_loadCoins = lookup_function("slice.loadCoins");
-    std::vector args = ctx->ir_slice;
-    std::vector dummy_loaded = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)");
-    code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadCoins);
+    // no TVM instruction to skip, just load but don't use the result
+    unpack(ctx, code, loc);
   }
 
   PackSize estimate(const EstimateContext* ctx) override {
@@ -428,9 +461,7 @@ struct S_Address final : ISerializer {
     // we can't do just
     // ctx->skipBits(2 + 1 + 8 + 256);
     // because it may be addr_none or addr_extern; there is no "skip address" in TVM, so just load it
-    FunctionPtr f_loadAddress = lookup_function("slice.loadAddress");
-    std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(tmp-addr)");
-    code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress);
+    unpack(ctx, code, loc);
   }
 
   PackSize estimate(const EstimateContext* ctx) override {
@@ -1088,10 +1119,13 @@ std::vector<PackOpcode> auto_generate_opcodes_for_union(TypePtr union_type, std:
 
 static std::unique_ptr<ISerializer> get_serializer_for_type(TypePtr any_type) {
   if (const auto* t_intN = any_type->try_as<TypeDataIntN>()) {
+    if (t_intN->is_variadic) {
+      return std::make_unique<S_VariadicIntN>(t_intN->n_bits, t_intN->is_unsigned);
+    }
     return std::make_unique<S_IntN>(t_intN->n_bits, t_intN->is_unsigned);
   }
-  if (const auto* t_bytesN = any_type->try_as<TypeDataBytesN>()) {
-    return std::make_unique<S_BytesN>(t_bytesN->n_width, t_bytesN->is_bits);
+  if (const auto* t_bitsN = any_type->try_as<TypeDataBitsN>()) {
+    return std::make_unique<S_BitsN>(t_bitsN->n_width, t_bitsN->is_bits);
   }
   if (any_type == TypeDataCoins::create()) {
     return std::make_unique<S_Coins>();

tolk/pack-unpack-serializers.h

@@ -64,7 +64,7 @@ class PackContext {
 public:
   const std::vector<var_idx_t> ir_builder;
   const var_idx_t ir_builder0;
-  const var_idx_t option_skipBitsNFieldsValidation;
+  const var_idx_t option_skipBitsNValidation;
 
   PackContext(CodeBlob& code, SrcLocation loc, std::vector<var_idx_t> ir_builder, const std::vector<var_idx_t>& ir_options);