[Tolk] Correctly handle import without ".tolk" extension

tolk-vm · tolk-vm · commit 72e0ae29a2fa · 2025-07-07T04:23:08.000+03:00
Also, output abs paths in error messages.
diff --git a/tolk-tester/tests/imports-tests.tolk b/tolk-tester/tests/imports-tests.tolk
@@ -0,0 +1,18 @@
+import "imports/some-math"
+import "./imports/use-dicts.ext"
+
+fun main() {
+    prepareDict_3_30_4_40_5_x(4);
+    return someAdd(1, 2);
+}
+
+/**
+@testcase | 0 | | 3
+
+@fif_codegen
+"""
+  main() PROC:<{
+    3 PUSHINT
+  }>
+"""
+ */
diff --git a/tolk-tester/tests/imports/has-tinf-err.tolk b/tolk-tester/tests/imports/has-tinf-err.tolk
@@ -0,0 +1,4 @@
+
+fun asdf() {
+    var i: int = "";
+}
diff --git a/tolk-tester/tests/imports/some-helpers.tolk b/tolk-tester/tests/imports/some-helpers.tolk
@@ -0,0 +1 @@
+import "./some-storage"
diff --git a/tolk-tester/tests/imports/some-storage.tolk b/tolk-tester/tests/imports/some-storage.tolk
@@ -0,0 +1 @@
+struct SomeStorage {}
diff --git a/tolk-tester/tests/imports/use-dicts.ext.tolk b/tolk-tester/tests/imports/use-dicts.ext.tolk
diff --git a/tolk-tester/tests/invalid-symbol/err-2205.tolk b/tolk-tester/tests/invalid-symbol/err-2205.tolk
@@ -3,6 +3,6 @@ import "../imports/invalid-no-import.tolk";
 
 /**
 @compilation_should_fail
-@stderr ../imports/invalid-no-import.tolk:2:13
+@stderr tests/imports/invalid-no-import.tolk:2:13
 @stderr Using a non-imported symbol `someAdd`
  */
diff --git a/tolk-tester/tests/invalid-symbol/err-2539.tolk b/tolk-tester/tests/invalid-symbol/err-2539.tolk
@@ -1,9 +1,9 @@
 import "@stdlib/tvm-dicts"
-import "../imports/use-dicts-err.tolk"
+import "../imports/use-dicts-err"
 
 /**
 @compilation_should_fail
-@stderr ../imports/use-dicts-err.tolk:2:22
+@stderr tests/imports/use-dicts-err.tolk:2:22
 @stderr Using a non-imported symbol `createEmptyDict`
-@stderr Forgot to import "@stdlib/tvm-dicts"?
+@stderr hint: forgot to import "@stdlib/tvm-dicts"?
  */
diff --git a/tolk-tester/tests/invalid-symbol/err-2540.tolk b/tolk-tester/tests/invalid-symbol/err-2540.tolk
@@ -0,0 +1,11 @@
+import "../imports/some-helpers"
+
+fun usingNonImported() {
+    var s: SomeStorage = {};
+}
+
+/**
+@compilation_should_fail
+@stderr Using a non-imported symbol `SomeStorage`
+@stderr forgot to import "some-storage.tolk"?
+ */
diff --git a/tolk-tester/tests/invalid-typing/err-6722.tolk b/tolk-tester/tests/invalid-typing/err-6722.tolk
@@ -0,0 +1,7 @@
+import "../imports/./has-tinf-err"
+
+/**
+@compilation_should_fail
+@stderr imports/has-tinf-err.tolk:3:18
+@stderr can not assign `slice` to variable of type `int`
+ */
diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk
@@ -1,4 +1,4 @@
-import "imports/use-dicts.tolk"
+import "imports/use-dicts.ext.tolk"
 
 fun simpleAllConst() {
     return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true);
diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h
@@ -247,7 +247,7 @@ class ASTStringifier final : public ASTVisitor {
       case ast_import_directive:
         return static_cast<std::string>(v->as<ast_import_directive>()->get_file_leaf()->str_val);
       case ast_tolk_file:
-        return v->as<ast_tolk_file>()->file->rel_filename;
+        return v->as<ast_tolk_file>()->file->realpath;
       default:
         return {};
     }
diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h
@@ -56,7 +56,7 @@ struct CompilerSettings {
 
   std::string output_filename;
   std::string boc_output_filename;
-  std::string stdlib_folder;    // a path to tolk-stdlib/; files imported via @stdlib/xxx are there
+  std::string stdlib_folder;    // path to tolk-stdlib/; note: from tolk-js it's empty! tolk-js reads files via js callback
 
   FsReadCallback read_callback;
 
diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp
@@ -52,10 +52,9 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con
     for (AnyV v_toplevel : file->ast->as<ast_tolk_file>()->get_toplevel_declarations()) {
       if (auto v_import = v_toplevel->try_as<ast_import_directive>()) {
         std::string imported_str = v_import->get_file_name();
-        size_t cur_slash_pos = file->rel_filename.rfind('/');
-        std::string rel_filename = cur_slash_pos == std::string::npos || imported_str[0] == '@'
+        std::string rel_filename = imported_str[0] == '@'
           ? std::move(imported_str)
-          : file->rel_filename.substr(0, cur_slash_pos + 1) + imported_str;
+          : file->extract_dirname() + imported_str;
 
         const SrcFile* imported = G.all_src_files.locate_and_register_source_file(rel_filename, v_import->loc);
         file->imports.push_back(SrcFile::ImportDirective{imported});
diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp
@@ -125,11 +125,11 @@ void pipeline_generate_fif_output_to_std_cout() {
   std::cout << "// automatically generated from ";
   bool need_comma = false;
   for (const SrcFile* file : G.all_src_files) {
-    if (!file->is_stdlib_file()) {
+    if (!file->is_stdlib_file) {
       if (need_comma) {
         std::cout << ", ";
       }
-      std::cout << file->rel_filename;
+      std::cout << file->extract_short_name();
       need_comma = true;
     }
   }
diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp
@@ -64,22 +64,6 @@ static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V<ast_identifier>
   }
 }
 
-static void check_import_exists_when_using_sym(FunctionPtr cur_f, AnyV v_usage, const Symbol* used_sym) {
-  SrcLocation sym_loc = used_sym->loc;
-  if (!v_usage->loc.is_symbol_from_same_or_builtin_file(sym_loc)) {
-    const SrcFile* declared_in = sym_loc.get_src_file();
-    bool has_import = false;
-    for (const SrcFile::ImportDirective& import : v_usage->loc.get_src_file()->imports) {
-      if (import.imported_file == declared_in) {
-        has_import = true;
-      }
-    }
-    if (!has_import) {
-      throw ParseError(cur_f, v_usage->loc, "Using a non-imported symbol `" + used_sym->name + "`. Forgot to import \"" + declared_in->rel_filename + "\"?");
-    }
-  }
-}
-
 struct NameAndScopeResolver {
   std::vector<std::unordered_map<uint64_t, const Symbol*>> scopes;
 
@@ -180,7 +164,9 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody {
 
     // for global functions, global vars and constants, `import` must exist
     if (!sym->try_as<LocalVarPtr>()) {
-      check_import_exists_when_using_sym(cur_f, v, sym);
+      if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) {
+        sym->check_import_exists_when_used_from(cur_f, v->loc);
+      }
     }
   }
 
diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp
@@ -168,6 +168,9 @@ class TypeNodesVisitorResolver {
         }
         if (const Symbol* sym = lookup_global_symbol(text)) {
           if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) {
+            if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) {
+              sym->check_import_exists_when_used_from(cur_f, v->loc);
+            }
             return custom_type;
           }
         }
diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp
@@ -24,41 +24,44 @@ namespace tolk {
 
 static_assert(sizeof(SrcLocation) == 8);
 
-const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& abs_filename) const {
+const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& realpath) const {
+  // files with the same realpath are considered equal
   for (const SrcFile* file : all_src_files) {
-    if (file->abs_filename == abs_filename) {
+    if (file->realpath == realpath) {
       return file;
     }
   }
   return nullptr;
 }
 
-const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from) {
-  td::Result<std::string> path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, rel_filename.c_str());
+const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& filename, SrcLocation included_from) {
+  bool is_stdlib = filename.size() > 8 && filename.starts_with("@stdlib/");
+
+  td::Result<std::string> path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, filename.c_str());
   if (path.is_error()) {
     if (included_from.is_defined()) {
       throw ParseError(included_from, "Failed to import: " + path.move_as_error().message().str());
     }
-    throw Fatal("Failed to locate " + rel_filename + ": " + path.move_as_error().message().str());
+    throw Fatal("Failed to locate " + filename + ": " + path.move_as_error().message().str());
   }
 
-  std::string abs_filename = path.move_as_ok();
-  if (const SrcFile* file = find_file(abs_filename)) {
+  std::string realpath = path.move_as_ok();
+  if (const SrcFile* file = find_file(realpath)) {
     return file;
   }
 
-  td::Result<std::string> text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, abs_filename.c_str());
+  td::Result<std::string> text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, realpath.c_str());
   if (text.is_error()) {
     if (included_from.is_defined()) {
       throw ParseError(included_from, "Failed to import: " + text.move_as_error().message().str());
     }
-    throw Fatal("Failed to read " + rel_filename + ": " + text.move_as_error().message().str());
+    throw Fatal("Failed to read " + realpath + ": " + text.move_as_error().message().str());
   }
 
   int file_id = static_cast<int>(all_src_files.size());   // SrcFile::file_id is the index in all files
-  SrcFile* created = new SrcFile(file_id, rel_filename, std::move(abs_filename), text.move_as_ok());
+  SrcFile* created = new SrcFile(file_id, is_stdlib, std::move(realpath), text.move_as_ok());
   if (G.is_verbosity(1)) {
-    std::cerr << "register file_id " << created->file_id << " " << created->abs_filename << std::endl;
+    std::cerr << "register file_id " << created->file_id << " " << created->realpath << std::endl;
   }
   all_src_files.push_back(created);
   return created;
@@ -72,11 +75,6 @@ SrcFile* AllRegisteredSrcFiles::get_next_unparsed_file() {
   return const_cast<SrcFile*>(all_src_files[++last_parsed_file_id]);
 }
 
-bool SrcFile::is_stdlib_file() const {
-  std::string_view rel(rel_filename);
-  return rel.size() > 10 && rel.substr(0, 8) == "@stdlib/"; // common.tolk, tvm-dicts.tolk, etc
-}
-
 bool SrcFile::is_offset_valid(int offset) const {
   return offset >= 0 && offset < static_cast<int>(text.size());
 }
@@ -116,9 +114,30 @@ SrcFile::SrcPosition SrcFile::convert_offset(int offset) const {
   return SrcPosition{offset, line_idx + 1, char_idx + 1, line_str};
 }
 
+std::string SrcFile::extract_short_name() const {
+  size_t last_slash = realpath.find_last_of("/\\");
+  if (last_slash == std::string::npos) {
+    return realpath;
+  }
+  std::string short_name = realpath.substr(last_slash + 1);    // "file.tolk" (no path)
+
+  if (is_stdlib_file) {   // not "common.tolk", but "@stdlib/common"
+    return "@stdlib/" + short_name.substr(0, short_name.size() - 5);
+  }
+  return short_name;
+}
+
+std::string SrcFile::extract_dirname() const {
+  size_t last_slash = realpath.find_last_of("/\\");
+  if (last_slash == std::string::npos) {
+    return "";
+  }
+  return realpath.substr(0, last_slash + 1);
+}
+
 
 std::ostream& operator<<(std::ostream& os, const SrcFile* src_file) {
-  return os << (src_file ? src_file->rel_filename : "unknown-location");
+  return os << (src_file ? src_file->realpath : "unknown-location");
 }
 
 std::ostream& operator<<(std::ostream& os, const Fatal& fatal) {
@@ -233,7 +252,7 @@ void ParseError::show(std::ostream& os) const {
     // print "location: line1 \n (spaces) line2 \n ..."
     std::string_view message = this->message;
     std::string loc_text = loc.to_string();
-    std::string loc_spaces(std::min(static_cast<int>(loc_text.size()), 30), ' ');
+    std::string loc_spaces(std::min(static_cast<int>(loc_text.size()), 9), ' ');
     size_t start = 0, end;
     os << loc_text << ": error: ";
     while ((end = message.find('\n', start)) != std::string::npos) {
@@ -248,7 +267,7 @@ void ParseError::show(std::ostream& os) const {
     }
   }
   if (current_function) {
-    os << "    // in function `" << current_function->as_human_readable() << "`" << std::endl;
+    os << std::endl << "    // in function `" << current_function->as_human_readable() << "`" << std::endl;
   }
   loc.show_context(os);
 }
diff --git a/tolk/src-file.h b/tolk/src-file.h
@@ -35,25 +35,26 @@ struct SrcFile {
   };
 
   int file_id;                          // an incremental counter through all parsed files
-  std::string rel_filename;             // relative to cwd
-  std::string abs_filename;             // absolute from root
+  bool is_stdlib_file;                  // is a part of Tolk distribution, imported via "@stdlib/..."
+  std::string realpath;                 // what "realpath" returned to locate (either abs path or what tolk-js returns)
   std::string text;                     // file contents loaded into memory, every Token::str_val points inside it
   AnyV ast = nullptr;                   // when a file has been parsed, its ast_tolk_file is kept here
   std::vector<ImportDirective> imports; // to check strictness (can't use a symbol without importing its file)
 
-  SrcFile(int file_id, std::string rel_filename, std::string abs_filename, std::string&& text)
+  SrcFile(int file_id, bool is_stdlib_file, std::string realpath, std::string&& text)
     : file_id(file_id)
-    , rel_filename(std::move(rel_filename))
-    , abs_filename(std::move(abs_filename))
+    , is_stdlib_file(is_stdlib_file)
+    , realpath(std::move(realpath))
     , text(std::move(text)) { }
 
   SrcFile(const SrcFile& other) = delete;
   SrcFile &operator=(const SrcFile&) = delete;
 
-  bool is_stdlib_file() const;
-
   bool is_offset_valid(int offset) const;
   SrcPosition convert_offset(int offset) const;
+
+  std::string extract_short_name() const;
+  std::string extract_dirname() const;
 };
 
 
@@ -102,9 +103,9 @@ class AllRegisteredSrcFiles {
 
 public:
   const SrcFile* get_file(int file_id) const { return all_src_files.at(file_id); }
-  const SrcFile* find_file(const std::string& abs_filename) const;
+  const SrcFile* find_file(const std::string& realpath) const;
 
-  const SrcFile* locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from);
+  const SrcFile* locate_and_register_source_file(const std::string& filename, SrcLocation included_from);
   SrcFile* get_next_unparsed_file();
 
   auto begin() const { return all_src_files.begin(); }
diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp
@@ -21,6 +21,19 @@
 
 namespace tolk {
 
+void Symbol::check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const {
+  const SrcFile* declared_in = loc.get_src_file();
+  bool has_import = false;
+  for (const SrcFile::ImportDirective& import : used_loc.get_src_file()->imports) {
+    if (import.imported_file == declared_in) {
+      has_import = true;
+    }
+  }
+  if (!has_import) {
+    throw ParseError(cur_f, used_loc, "Using a non-imported symbol `" + name + "`\nhint: forgot to import \"" + declared_in->extract_short_name() + "\"?");
+  }
+}
+
 std::string FunctionData::as_human_readable() const {
   if (!is_generic_function()) {
     return name;  // if it's generic instantiation like `f<int>`, its name is "f<int>", not "f"
diff --git a/tolk/symtable.h b/tolk/symtable.h
@@ -43,6 +43,8 @@ struct Symbol {
 #endif
     return dynamic_cast<ConstTPtr>(this);
   }
+
+  void check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const;
 };
 
 struct LocalVarData final : Symbol {
diff --git a/tolk/tolk-main.cpp b/tolk/tolk-main.cpp
@@ -149,19 +149,15 @@ static std::string auto_discover_stdlib_folder() {
 td::Result<std::string> fs_read_callback(CompilerSettings::FsReadCallbackKind kind, const char* query) {
   switch (kind) {
     case CompilerSettings::FsReadCallbackKind::Realpath: {
-      td::Result<std::string> res_realpath;
-      if (query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8)) {
-        // import "@stdlib/filename" or import "@stdlib/filename.tolk"
-        std::string path = G.settings.stdlib_folder + static_cast<std::string>(query + 7);
-        if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) {
-          path += ".tolk";
-        }
-        res_realpath = td::realpath(td::CSlice(path.c_str()));
-      } else {
-        // import "relative/to/cwd/path.tolk"
-        res_realpath = td::realpath(td::CSlice(query));
-      }
+      bool is_stdlib = query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8);
+      std::string path = is_stdlib
+            ? G.settings.stdlib_folder + static_cast<std::string>(query + 7)
+            : static_cast<std::string>(query);
 
+      if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) {
+        path += ".tolk";
+      }
+      td::Result<std::string> res_realpath = td::realpath(td::CSlice(path.c_str()));
       if (res_realpath.is_error()) {
         // note, that for non-existing files, `realpath()` on Linux/Mac returns an error,
         // whereas on Windows, it returns okay, but fails after, on reading, with a message "cannot open file"
diff --git a/tolk/tolk-wasm.cpp b/tolk/tolk-wasm.cpp
@@ -57,7 +57,7 @@ static td::Result<std::string> compile_internal(char *config_json) {
   std::cerr.rdbuf(errs.rdbuf());
   int exit_code = tolk_proceed(entrypoint_filename);
   if (exit_code != 0) {
-    return td::Status::Error("Tolk compilation error: " + errs.str());
+    return td::Status::Error(errs.str());
   }
 
   TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/"));

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
++
 +fun asdf() {
 +    var i: int = "";
 +}
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import "imports/use-dicts.tolk"`
	`1`	`+import "imports/use-dicts.ext.tolk"`
`2`	`2`
`3`	`3`	`fun simpleAllConst() {`
`4`	`4`	`return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true);`