[Tolk] Rewrite lexer, spaces are not mandatory anymore

A new lexer is noticeably faster and memory efficient
(although splitting a file to tokens is negligible in a whole pipeline).

But the purpose of rewriting lexer was not just to speed up,
but to allow writing code without spaces:
`2+2` is now 4, not a valid identifier as earlier.

The variety of symbols allowed in identifier has greatly reduced
and is now similar to other languages.

SrcLocation became 8 bytes on stack everywhere.

Command-line flags were also reworked:
- the input for Tolk compiler is only a single file now, it's parsed, and parsing continues while new #include are resolved
- flags like -A -P and so on are no more needed, actually

Author: tolk-vm

crypto/fift/utils.cpp

@@ -211,20 +211,39 @@ td::Result<fift::SourceLookup> create_mem_source_lookup(std::string main, std::s
                               fift_dir);
 }
 
-td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code, std::string fift_dir, bool is_raw) {
+td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code) {
   std::stringstream ss;
   std::string sb;
   sb.reserve(asm_code.size() + 100);
-  sb.append("\"Asm.fif\" include\n ");
-  sb.append(is_raw ? "<{" : "");
+  sb.append("\"Asm.fif\" include\n <{\n");
   sb.append(asm_code.data(), asm_code.size());
-  sb.append(is_raw ? "}>c" : "");
-  sb.append(" boc>B \"res\" B>file");
+  sb.append("\n}>c boc>B \"res\" B>file");
 
-  TRY_RESULT(source_lookup, create_source_lookup(std::move(sb), true, true, true, false, false, false, false, fift_dir));
+  TRY_RESULT(source_lookup, create_source_lookup(std::move(sb), true, true, true, false, false, false, false));
   TRY_RESULT(res, run_fift(std::move(source_lookup), &ss));
   TRY_RESULT(boc, res.read_file("res"));
   return vm::std_boc_deserialize(std::move(boc.data));
 }
 
+td::Result<CompiledProgramOutput> compile_asm_program(std::string&& program_code, const std::string& fift_dir) {
+  std::string main_fif;
+  main_fif.reserve(program_code.size() + 100);
+  main_fif.append(program_code.data(), program_code.size());
+  main_fif.append(R"( dup hashB B>X      $>B "hex" B>file)");   // write codeHashHex to a file
+  main_fif.append(R"(     boc>B B>base64 $>B "boc" B>file)");   // write codeBoc64 to a file
+
+  std::stringstream fift_output_stream;
+  TRY_RESULT(source_lookup, create_source_lookup(std::move(main_fif), true, true, false, false, false, false, false, fift_dir));
+  TRY_RESULT(res, run_fift(std::move(source_lookup), &fift_output_stream));
+
+  TRY_RESULT(boc, res.read_file("boc"));
+  TRY_RESULT(hex, res.read_file("hex"));
+
+  return CompiledProgramOutput{
+    std::move(program_code),
+    std::move(boc.data),
+    std::move(hex.data),
+  };
+}
+
 }  // namespace fift

crypto/fift/utils.h

@@ -26,11 +26,21 @@ struct FiftOutput {
   SourceLookup source_lookup;
   std::string output;
 };
+
+// given a valid Fift code PROGRAM{ ... }END>c, compile_asm_program() returns this output
+// now it's used primarily for wasm output (see tolk-js, for example)
+struct CompiledProgramOutput {
+  std::string fiftCode;
+  std::string codeBoc64;
+  std::string codeHashHex;
+};
+
 td::Result<fift::SourceLookup> create_mem_source_lookup(std::string main, std::string fift_dir = "",
                                                         bool need_preamble = true, bool need_asm = true,
                                                         bool need_ton_util = true, bool need_lisp = true,
                                                         bool need_w3_code = true);
 td::Result<FiftOutput> mem_run_fift(std::string source, std::vector<std::string> args = {}, std::string fift_dir = "");
 td::Result<FiftOutput> mem_run_fift(SourceLookup source_lookup, std::vector<std::string> args);
-td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code, std::string fift_dir = "", bool is_raw = true);
+td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code);
+td::Result<CompiledProgramOutput> compile_asm_program(std::string&& program_code, const std::string& fift_dir);
 }  // namespace fift

crypto/funcfiftlib/funcfiftlib.cpp

@@ -37,10 +37,10 @@
 
 td::Result<std::string> compile_internal(char *config_json) {
   TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json)))
-  auto &obj = input_json.get_object();
+  td::JsonObject& config = input_json.get_object();
 
-  TRY_RESULT(opt_level, td::get_json_object_int_field(obj, "optLevel", false));
-  TRY_RESULT(sources_obj, td::get_json_object_field(obj, "sources", td::JsonValue::Type::Array, false));
+  TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optLevel", false));
+  TRY_RESULT(sources_obj, td::get_json_object_field(config, "sources", td::JsonValue::Type::Array, false));
 
   auto &sources_arr = sources_obj.get_array();
 
@@ -52,29 +52,25 @@ td::Result<std::string> compile_internal(char *config_json) {
 
   funC::opt_level = std::max(0, opt_level);
   funC::program_envelope = true;
+  funC::asm_preamble = true;
   funC::verbosity = 0;
   funC::indent = 1;
 
   std::ostringstream outs, errs;
-  auto compile_res = funC::func_proceed(sources, outs, errs);
-
-  if (compile_res != 0) {
-    return td::Status::Error(std::string("Func compilation error: ") + errs.str());
+  int funC_res = funC::func_proceed(sources, outs, errs);
+  if (funC_res != 0) {
+    return td::Status::Error("FunC compilation error: " + errs.str());
   }
 
-  TRY_RESULT(code_cell, fift::compile_asm(outs.str(), "/fiftlib/", false));
-  TRY_RESULT(boc, vm::std_boc_serialize(code_cell));
+  TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/"));
 
   td::JsonBuilder result_json;
-  auto result_obj = result_json.enter_object();
-  result_obj("status", "ok");
-  result_obj("codeBoc", td::base64_encode(boc));
-  result_obj("fiftCode", outs.str());
-  result_obj("codeHashHex", code_cell->get_hash().to_hex());
-  result_obj.leave();
-
-  outs.clear();
-  errs.clear();
+  auto obj = result_json.enter_object();
+  obj("status", "ok");
+  obj("fiftCode", std::move(fift_res.fiftCode));
+  obj("codeBoc", std::move(fift_res.codeBoc64));
+  obj("codeHashHex", std::move(fift_res.codeHashHex));
+  obj.leave();
 
   return result_json.string_builder().as_cslice().str();
 }

crypto/smartcont/mathlib.tolk

@@ -572,9 +572,9 @@ int atanh_f261(int x, int n) inline_ref {
     s -= 1;
   }
   x += t;
-  int 2x = 2 * x;
-  int y = lshift256divr(2x, (x >> 1) - t);
-  ;; y = 2x - (mulrshiftr256(2x, y) ~>> 2);  ;; this line could improve precision on very rare occasions
+  int `2x` = 2 * x;
+  int y = lshift256divr(`2x`, (x >> 1) - t);
+  ;; y = `2x` - (mulrshiftr256(2x, y) ~>> 2);  ;; this line could improve precision on very rare occasions
   return (atanh_f258(y, 36), s);
 }
 

tolk/CMakeLists.txt

@@ -1,10 +1,9 @@
 cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
 
 set(TOLK_SOURCE
-        srcread.cpp
+        src-file.cpp
         lexer.cpp
         symtable.cpp
-        keywords.cpp
         unify-types.cpp
         parse-tolk.cpp
         abscode.cpp

tolk/abscode.cpp

@@ -24,29 +24,19 @@ namespace tolk {
  * 
  */
 
-TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, const SrcLocation* loc)
-    : v_type(_type), idx(_idx), cls(_cls), coord(0) {
+TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, SrcLocation loc)
+    : v_type(_type), idx(_idx), cls(_cls), coord(0), where(loc) {
   if (sym) {
     name = sym->sym_idx;
     sym->value->idx = _idx;
   }
-  if (loc) {
-    where = std::make_unique<SrcLocation>(*loc);
-  }
   if (!_type) {
     v_type = TypeExpr::new_hole();
   }
-  if (cls == _Named) {
-    undefined = true;
-  }
 }
 
-void TmpVar::set_location(const SrcLocation& loc) {
-  if (where) {
-    *where = loc;
-  } else {
-    where = std::make_unique<SrcLocation>(loc);
-  }
+void TmpVar::set_location(SrcLocation loc) {
+  where = loc;
 }
 
 void TmpVar::dump(std::ostream& os) const {
@@ -469,10 +459,10 @@ void CodeBlob::print(std::ostream& os, int flags) const {
   if ((flags & 8) != 0) {
     for (const auto& var : vars) {
       var.dump(os);
-      if (var.where && (flags & 1) != 0) {
-        var.where->show(os);
+      if (var.where.is_defined() && (flags & 1) != 0) {
+        var.where.show(os);
         os << " defined here:\n";
-        var.where->show_context(os);
+        var.where.show_context(os);
       }
     }
   }
@@ -483,7 +473,7 @@ void CodeBlob::print(std::ostream& os, int flags) const {
   os << "-------- END ---------\n\n";
 }
 
-var_idx_t CodeBlob::create_var(int cls, TypeExpr* var_type, SymDef* sym, const SrcLocation* location) {
+var_idx_t CodeBlob::create_var(int cls, TypeExpr* var_type, SymDef* sym, SrcLocation location) {
   vars.emplace_back(var_cnt, cls, var_type, sym, location);
   if (sym) {
     sym->value->idx = var_cnt;
@@ -501,7 +491,7 @@ bool CodeBlob::import_params(FormalArgList arg_list) {
     SymDef* arg_sym;
     SrcLocation arg_loc;
     std::tie(arg_type, arg_sym, arg_loc) = par;
-    list.push_back(create_var(arg_sym ? (TmpVar::_In | TmpVar::_Named) : TmpVar::_In, arg_type, arg_sym, &arg_loc));
+    list.push_back(create_var(arg_sym ? (TmpVar::_In | TmpVar::_Named) : TmpVar::_In, arg_type, arg_sym, arg_loc));
   }
   emplace_back(loc, Op::_Import, list);
   in_var_cnt = var_cnt;

tolk/analyzer.cpp

@@ -36,7 +36,7 @@ int CodeBlob::split_vars(bool strict) {
   for (int j = 0; j < var_cnt; j++) {
     TmpVar& var = vars[j];
     if (strict && var.v_type->minw != var.v_type->maxw) {
-      throw ParseError{var.where.get(), "variable does not have fixed width, cannot manipulate it"};
+      throw ParseError{var.where, "variable does not have fixed width, cannot manipulate it"};
     }
     std::vector<TypeExpr*> comp_types;
     int k = var.v_type->extract_components(comp_types);
@@ -45,7 +45,7 @@ int CodeBlob::split_vars(bool strict) {
     if (k != 1) {
       var.coord = ~((n << 8) + k);
       for (int i = 0; i < k; i++) {
-        auto v = create_var(vars[j].cls, comp_types[i], 0, vars[j].where.get());
+        auto v = create_var(vars[j].cls, comp_types[i], 0, vars[j].where);
         tolk_assert(v == n + i);
         tolk_assert(vars[v].idx == v);
         vars[v].name = vars[j].name;
@@ -54,7 +54,7 @@ int CodeBlob::split_vars(bool strict) {
       n += k;
       ++changes;
     } else if (strict && var.v_type->minw != 1) {
-      throw ParseError{var.where.get(),
+      throw ParseError{var.where,
                             "cannot work with variable or variable component of width greater than one"};
     }
   }