Pmakhnev/tolk source maps

crypto/fift/lib/Asm.fif

@@ -184,7 +184,7 @@ x{6E} @Defop ISNULL
 x{6F0} @Defop(4u) TUPLE
 x{6F00} @Defop NIL
 x{6F01} @Defop SINGLE
-x{6F02} dup @Defop PAIR @Defop CONS 
+x{6F02} dup @Defop PAIR @Defop CONS
 x{6F03} @Defop TRIPLE
 x{6F1} @Defop(4u) INDEX
 x{6F10} dup @Defop FIRST @Defop CAR
@@ -251,7 +251,7 @@ x{7F} @Defop TRUE
     { dup 16 fits
       { <b x{81} s, swap 16 i, }
       { 11 { dup 259 > abort"integer too large" 8 + 2dup fits } until
-        <b x{82} s, over 3 >> 2- 5 u, -rot i, 
+        <b x{82} s, over 3 >> 2- 5 u, -rot i,
       } cond
     } cond
   } cond
@@ -260,6 +260,11 @@ x{7F} @Defop TRUE
 x{83FF} @Defop PUSHNAN
 { <b x{84} s, swap 1- 8 u, @addopb } : PUSHPOW2DEC
 { <b x{85} s, swap 1- 8 u, @addopb } : PUSHNEGPOW2
+
+// special instruction for debugging purposes
+// doesn't exists in TVM
+{ <b x{F955} s, swap 16 u, @addopb } : DEBUGMARK
+
 //
 // other constants
 x{88} @Defop(ref) PUSHREF
@@ -917,24 +922,24 @@ recursive IFELSE-cont2 {
 { 1 { swap @normal? swap IFELSE-cont2 } does @doafter<{ } : @doifelse
 { 1 { swap @normal? IFELSE-cont2 } does @doafter<{ } : @doifnotelse
 {
-  { dup `else eq? 
+  { dup `else eq?
     { drop @doifelse }
     { dup `else: eq?
       { drop IFJMP-cont }
       { @normal? IF-cont
-      } cond 
-    } cond 
-  } @doafter<{ 
+      } cond
+    } cond
+  } @doafter<{
 } : IF:<{
 {
-  { dup `else eq? 
+  { dup `else eq?
     { drop @doifnotelse }
     { dup `else: eq?
       { drop IFNOTJMP-cont }
       { @normal? IFNOT-cont
-      } cond 
-    } cond 
-  } @doafter<{ 
+      } cond
+    } cond
+  } @doafter<{
 } : IFNOT:<{
 
 x{E304} @Defop CONDSEL
@@ -962,12 +967,12 @@ x{EB} dup @Defop AGAINEND @Defop AGAIN:
 { }> PUSHCONT UNTIL } : }>UNTIL
 { { @normal? PUSHCONT UNTIL } @doafter<{ } : UNTIL:<{
 { PUSHCONT { @normal? PUSHCONT WHILE } @doafter<{ } : @dowhile
-{ 
-  { dup `do eq? 
-    { drop @dowhile } 
-    { `do: eq? not abort"`}>DO<{` expected" PUSHCONT WHILEEND 
-    } cond 
-  } @doafter<{ 
+{
+  { dup `do eq?
+    { drop @dowhile }
+    { `do: eq? not abort"`}>DO<{` expected" PUSHCONT WHILEEND
+    } cond
+  } @doafter<{
 } : WHILE:<{
 { }> PUSHCONT AGAIN } : }>AGAIN
 { { @normal? PUSHCONT AGAIN } @doafter<{ } : AGAIN:<{
@@ -987,11 +992,11 @@ x{E31B} dup @Defop AGAINENDBRK @Defop AGAINBRK:
 { { @normal? PUSHCONT UNTILBRK } @doafter<{ } : UNTILBRK:<{
 { PUSHCONT { @normal? PUSHCONT WHILEBRK } @doafter<{ } : @dowhile
 {
-  { dup `do eq? 
-    { drop @dowhile } 
+  { dup `do eq?
+    { drop @dowhile }
     { `do: eq? not abort"`}>DO<{` expected" PUSHCONT WHILEENDBRK
-    } cond 
-  } @doafter<{ 
+    } cond
+  } @doafter<{
 } : WHILEBRK:<{
 { }> PUSHCONT AGAINBRK } : }>AGAINBRK
 { { @normal? PUSHCONT AGAINBRK } @doafter<{ } : AGAINBRK:<{
@@ -1017,8 +1022,8 @@ x{ED1F} @Defop BLESSVARARGS
 { c4 PUSHCTR } : PUSHROOT
 { c4 POPCTR } : POPROOT
 x{ED6} dup @Defop(c) SETCONTCTR @Defop(c) SETCONT
-x{ED7} @Defop(c) SETRETCTR 
-x{ED8} @Defop(c) SETALTCTR 
+x{ED7} @Defop(c) SETRETCTR
+x{ED8} @Defop(c) SETALTCTR
 x{ED9} dup @Defop(c) POPSAVE @Defop(c) POPCTRSAVE
 x{EDA} dup @Defop(c) SAVE @Defop(c) SAVECTR
 x{EDB} dup @Defop(c) SAVEALT @Defop(c) SAVEALTCTR
@@ -1075,7 +1080,7 @@ x{EDFB} @Defop SAMEALTSAVE
 { dup sbits
    { @addop }
    {
-    dup srefs // 
+    dup srefs //
       { ref@ CALLREF }
       { drop }
       cond
@@ -1112,9 +1117,9 @@ x{F2FF} @Defop TRY
 x{F3} @Defop(4u,4u) TRYARGS
 { `catch @endblk } : }>CATCH<{
 { PUSHCONT { @normal? PUSHCONT TRY } @doafter<{ } : @trycatch
-{ 
+{
   { `catch eq? not abort"`}>CATCH<{` expected" @trycatch
-  } @doafter<{ 
+  } @doafter<{
 } : TRY:<{
 //
 // dictionary manipulation
@@ -1264,7 +1269,7 @@ x{F4A1} @Defop DICTUGETJMP
 x{F4A2} @Defop DICTIGETEXEC
 x{F4A3} @Defop DICTUGETEXEC
 { dup sbitrefs tuck 1 > swap 1 <> or abort"not a dictionary" swap 1 u@ over <> abort"not a dictionary" } : @chkdicts
-{ dup null? tuck { <s } ifnot drop not } : @chkdict 
+{ dup null? tuck { <s } ifnot drop not } : @chkdict
 { over @chkdict
   { swap <b x{F4A6_} s, swap ref, swap 10 u, @addopb }
   { nip swap NEWDICT swap PUSHINT }

crypto/fift/utils.cpp

@@ -227,7 +227,7 @@ td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code) {
 
 td::Result<CompiledProgramOutput> compile_asm_program(std::string&& program_code, const std::string& fift_dir) {
   std::string main_fif;
-  main_fif.reserve(program_code.size() + 100);
+  main_fif.reserve(program_code.size() + 200);
   main_fif.append(program_code.data(), program_code.size());
   main_fif.append(R"( dup hashB B>X      $>B "hex" B>file)");   // write codeHashHex to a file
   main_fif.append(R"(     boc>B B>base64 $>B "boc" B>file)");   // write codeBoc64 to a file

tolk/CMakeLists.txt

@@ -31,6 +31,7 @@ set(TOLK_SOURCE
         pipe-ast-to-legacy.cpp
         pipe-find-unused-symbols.cpp
         pipe-generate-fif-output.cpp
+        pipe-generate-source-map.cpp
         type-system.cpp
         smart-casts-cfg.cpp
         generics-helpers.cpp
@@ -43,6 +44,7 @@ set(TOLK_SOURCE
         stack-transform.cpp
         optimize.cpp
         codegen.cpp
+        debug-info.cpp
         tolk.cpp
 )
 

tolk/abscode.cpp

@@ -254,6 +254,10 @@ void Op::show(std::ostream& os, const std::vector<TmpVar>& vars, std::string pfx
       show_var_list(os, left, vars);
       os << " := " << str_const << std::endl;
       break;
+    case _DebugInfo:
+      os << pfx << dis << "DEBUGINFO ";
+      os << source_map_entry_idx << std::endl;
+      break;
     case _Import:
       os << pfx << dis << "IMPORT ";
       show_var_list(os, left, vars);
@@ -394,41 +398,41 @@ void CodeBlob::print(std::ostream& os, int flags) const {
   os << "-------- END ---------\n\n";
 }
 
-std::vector<var_idx_t> CodeBlob::create_var(TypePtr var_type, SrcLocation loc, std::string name) {
+std::vector<var_idx_t> CodeBlob::create_var(TypePtr var_type, SrcLocation loc, std::string name, TypePtr parent_type) {
   std::vector<var_idx_t> ir_idx;
   int stack_w = var_type->get_width_on_stack();
   ir_idx.reserve(stack_w);
   if (const TypeDataStruct* t_struct = var_type->try_as<TypeDataStruct>()) {
     for (int i = 0; i < t_struct->struct_ref->get_num_fields(); ++i) {
       StructFieldPtr field_ref = t_struct->struct_ref->get_field(i);
       std::string sub_name = name.empty() || t_struct->struct_ref->get_num_fields() == 1 ? name : name + "." + field_ref->name;
-      std::vector<var_idx_t> nested = create_var(field_ref->declared_type, loc, std::move(sub_name));
+      std::vector<var_idx_t> nested = create_var(field_ref->declared_type, loc, std::move(sub_name), parent_type);
       ir_idx.insert(ir_idx.end(), nested.begin(), nested.end());
     }
   } else if (const TypeDataTensor* t_tensor = var_type->try_as<TypeDataTensor>()) {
     for (int i = 0; i < t_tensor->size(); ++i) {
       std::string sub_name = name.empty() ? name : name + "." + std::to_string(i);
-      std::vector<var_idx_t> nested = create_var(t_tensor->items[i], loc, std::move(sub_name));
+      std::vector<var_idx_t> nested = create_var(t_tensor->items[i], loc, std::move(sub_name), parent_type);
       ir_idx.insert(ir_idx.end(), nested.begin(), nested.end());
     }
   } else if (const TypeDataAlias* t_alias = var_type->try_as<TypeDataAlias>()) {
-    ir_idx = create_var(t_alias->underlying_type, loc, std::move(name));
+    ir_idx = create_var(t_alias->underlying_type, loc, std::move(name), parent_type);
   } else if (const TypeDataUnion* t_union = var_type->try_as<TypeDataUnion>(); t_union && stack_w != 1) {
     std::string utag_name = name.empty() ? "'UTag" : name + ".UTag";
     if (t_union->or_null) {   // in stack comments, `a:(int,int)?` will be "a.0 a.1 a.UTag"
-      ir_idx = create_var(t_union->or_null, loc, std::move(name));
+      ir_idx = create_var(t_union->or_null, loc, std::move(name), parent_type);
     } else {                  // in stack comments, `a:int|slice` will be "a.USlot1 a.UTag"
       for (int i = 0; i < stack_w - 1; ++i) {
         std::string slot_name = name.empty() ? "'USlot" + std::to_string(i + 1) : name + ".USlot" + std::to_string(i + 1);
-        ir_idx.emplace_back(create_var(TypeDataUnknown::create(), loc, std::move(slot_name))[0]);
+        ir_idx.emplace_back(create_var(TypeDataUnknown::create(), loc, std::move(slot_name), var_type)[0]);
       }
     }
-    ir_idx.emplace_back(create_var(TypeDataInt::create(), loc, std::move(utag_name))[0]);
+    ir_idx.emplace_back(create_var(TypeDataInt::create(), loc, std::move(utag_name), parent_type)[0]);
   } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) {
 #ifdef TOLK_DEBUG
     tolk_assert(stack_w == 1);
 #endif
-    vars.emplace_back(var_cnt, var_type, std::move(name), loc);
+    vars.emplace_back(var_cnt, var_type, std::move(name), loc, parent_type);
     ir_idx.emplace_back(var_cnt);
     var_cnt++;
   }

tolk/analyzer.cpp

@@ -303,7 +303,7 @@ bool Op::std_compute_used_vars(bool disabled) {
 bool Op::compute_used_vars(const CodeBlob& code, bool edit) {
   tolk_assert(next);
   const VarDescrList& next_var_info = next->var_info;
-  if (cl == _Nop) {
+  if (cl == _Nop || cl == _DebugInfo) {
     return set_var_info_except(next_var_info, left);
   }
   switch (cl) {
@@ -528,6 +528,7 @@ bool prune_unreachable(std::unique_ptr<Op>& ops) {
     case Op::_UnTuple:
     case Op::_Import:
     case Op::_Let:
+    case Op::_DebugInfo:
       reach = true;
       break;
     case Op::_Return:
@@ -693,6 +694,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
   switch (cl) {
     case _Nop:
     case _Import:
+    case _DebugInfo:
       break;
     case _Return:
       values.set_unreachable();
@@ -887,6 +889,7 @@ bool Op::mark_noreturn() {
     case _SetGlob:
     case _GlobVar:
     case _CallInd:
+    case _DebugInfo:
       return set_noreturn(next->mark_noreturn());
     case _Return:
       return set_noreturn();

tolk/asmops.cpp

@@ -330,6 +330,21 @@ void AsmOpList::show_var_ext(std::ostream& os, std::pair<var_idx_t, const_idx_t>
   }
 }
 
+std::optional<std::tuple<TmpVar, std::string>> AsmOpList::get_var(const std::pair<var_idx_t, const_idx_t>& idx_pair) const {
+  const var_idx_t var_idx = idx_pair.first;
+  const const_idx_t const_idx = idx_pair.second;
+  if (!var_names_ || static_cast<size_t>(var_idx) >= var_names_->size()) {
+    return std::nullopt;
+  }
+  const auto var = var_names_->at(var_idx);
+  if (static_cast<size_t>(const_idx) < constants_.size() && constants_[const_idx].not_null()) {
+    const auto value = constants_[const_idx];
+    const auto value_str = value->to_dec_string();
+    return std::tie(var, value_str);
+  }
+  return std::tie(var, "");
+}
+
 void AsmOpList::out(std::ostream& os, int mode) const {
   std::size_t n = list_.size();
   for (std::size_t i = 0; i < n; i++) {

tolk/codegen.cpp

@@ -276,6 +276,29 @@ void Stack::rearrange_top(SrcLocation loc, var_idx_t top, bool last) {
 }
 
 bool Op::generate_code_step(Stack& stack) {
+  // we need to handle it here to correctly handle case `IFJMP { DROP }`
+  if (cl == _DebugInfo) {
+    std::ostringstream ops;
+    ops << source_map_entry_idx << " DEBUGMARK"; // pseudo instruction
+
+    // Append opcode to a list
+    if (const auto list_size = stack.o.list_.size(); list_size > 0) {
+      stack.o.insert(stack.o.list_.size(), loc, ops.str());
+    }
+
+    if (source_map_entry_idx < G.source_map.size()) {
+      auto& entry = G.source_map.at(source_map_entry_idx);
+
+      // Collect all available variables at this point
+      for (const auto index : stack.s) {
+        if (const auto var = stack.o.get_var(index); var.has_value()) {
+          const auto& [data, value] = *var;
+          entry.vars.push_back({data, value});
+        }
+      }
+    }
+  }
+
   stack.opt_show();
 
   // detect `throw 123` (actually _IntConst 123 + _Call __throw)
@@ -882,6 +905,10 @@ bool Op::generate_code_step(Stack& stack) {
       stack.o << AsmOp::Custom(loc, "TRY");
       return true;
     }
+    case _DebugInfo: {
+      // already handled above
+      return true;
+    }
     default:
       std::cerr << "fatal: unknown operation <??" << cl << ">\n";
       throw ParseError(loc, "unknown operation in generate_code()");

tolk/compiler-state.h

@@ -22,6 +22,7 @@
 #include <functional>
 #include <set>
 #include <string>
+#include <tolk.h>
 
 namespace tolk {
 
@@ -53,9 +54,11 @@ struct CompilerSettings {
   int optimization_level = 2;
   bool stack_layout_comments = true;
   bool tolk_src_as_line_comments = true;
+  bool collect_source_map = false;
 
   std::string output_filename;
   std::string boc_output_filename;
+  std::string source_map_output_filename;
   std::string stdlib_folder;    // path to tolk-stdlib/; note: from tolk-js it's empty! tolk-js reads files via js callback
 
   FsReadCallback read_callback;
@@ -106,6 +109,8 @@ struct CompilerState {
   std::vector<EnumDefPtr> all_enums;
   AllRegisteredSrcFiles all_src_files;
 
+  std::vector<SourceMapEntry> source_map;
+
   bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; }
 };