rdi: extend per-unit info with symbol ranges; build into baker

Author: ryanfleury

src/lib_rdi/rdi.h

@@ -762,6 +762,14 @@ X(RDI_U32, archive_file_path_node)\
 X(RDI_U32, build_path_node)\
 X(RDI_Language, language)\
 X(RDI_U32, line_table_idx)\
+X(RDI_U32, procedures_first_idx)\
+X(RDI_U32, procedures_count)\
+X(RDI_U32, global_variables_first_idx)\
+X(RDI_U32, global_variables_count)\
+X(RDI_U32, thread_variables_first_idx)\
+X(RDI_U32, thread_variables_count)\
+X(RDI_U32, constants_first_idx)\
+X(RDI_U32, constants_count)\
 
 #define RDI_LineTable_XList \
 X(RDI_U32, voffs_base_idx)\
@@ -1319,6 +1327,14 @@ RDI_U32 archive_file_path_node;
 RDI_U32 build_path_node;
 RDI_Language language;
 RDI_U32 line_table_idx;
+RDI_U32 procedures_first_idx;
+RDI_U32 procedures_count;
+RDI_U32 global_variables_first_idx;
+RDI_U32 global_variables_count;
+RDI_U32 thread_variables_first_idx;
+RDI_U32 thread_variables_count;
+RDI_U32 constants_first_idx;
+RDI_U32 constants_count;
 };
 
 typedef struct RDI_LineTable RDI_LineTable;

src/lib_rdi_make/rdi_make.c

@@ -51,23 +51,6 @@ rdim_memcpy_struct(dst, to_push);                                   \
 }                                                                     \
 rdim_memzero_struct(to_push);
 
-#define RDIM_IdxedChunkListShallowCopy(list_type, chunk_type, src, ...) \
-list_type dst = {0};\
-RDI_U64 base_idx = 0;\
-for(chunk_type *src_n = src->first; src_n != 0; src_n = src_n->next)\
-{\
-chunk_type *dst_n = rdim_push_array(arena, chunk_type, 1);\
-RDIM_SLLQueuePush(dst.first, dst.last, dst_n);\
-dst.total_count += src_n->count;\
-dst.chunk_count += 1;\
-dst_n->count = src_n->count;\
-dst_n->cap = dst_n->count;\
-dst_n->base_idx = base_idx;\
-dst_n->v = src_n->v;\
-base_idx += dst_n->count;\
-}\
-return dst;
-
 ////////////////////////////////
 //~ rjf: Basic Helpers
 
@@ -960,12 +943,6 @@ rdim_symbol_chunk_list_concat_in_place(RDIM_SymbolChunkList *dst, RDIM_SymbolChu
   RDIM_IdxedChunkListConcatInPlace(RDIM_SymbolChunkNode, dst, to_push);
 }
 
-RDI_PROC RDIM_SymbolChunkList
-rdim_symbol_chunk_list_shallow_copy(RDIM_Arena *arena, RDIM_SymbolChunkList *src)
-{
-  RDIM_IdxedChunkListShallowCopy(RDIM_SymbolChunkList, RDIM_SymbolChunkNode, src);
-}
-
 ////////////////////////////////
 //~ rjf: [Building] Inline Site Info Building
 

src/lib_rdi_make/rdi_make.h

@@ -1475,7 +1475,6 @@ RDI_PROC RDIM_UDTEnumVal *rdim_udt_push_enum_val(RDIM_Arena *arena, RDIM_UDTChun
 RDI_PROC RDIM_Symbol *rdim_symbol_chunk_list_push(RDIM_Arena *arena, RDIM_SymbolChunkList *list, RDI_U64 cap);
 RDI_PROC RDI_U64 rdim_idx_from_symbol(RDIM_Symbol *symbol);
 RDI_PROC void rdim_symbol_chunk_list_concat_in_place(RDIM_SymbolChunkList *dst, RDIM_SymbolChunkList *to_push);
-RDI_PROC RDIM_SymbolChunkList rdim_symbol_chunk_list_shallow_copy(RDIM_Arena *arena, RDIM_SymbolChunkList *src);
 
 ////////////////////////////////
 //~ rjf: [Building] Inline Site Info Building

src/rdi/rdi.mdesk

@@ -600,6 +600,14 @@ RDI_UnitMemberTable:
   {build_path_node               RDI_U32               ""}
   {language                      RDI_Language          ""}
   {line_table_idx                RDI_U32               ""}
+  {procedures_first_idx          RDI_U32               ""}
+  {procedures_count              RDI_U32               ""}
+  {global_variables_first_idx    RDI_U32               ""}
+  {global_variables_count        RDI_U32               ""}
+  {thread_variables_first_idx    RDI_U32               ""}
+  {thread_variables_count        RDI_U32               ""}
+  {constants_first_idx           RDI_U32               ""}
+  {constants_count               RDI_U32               ""}
 }
 
 @xlist RDI_Unit_XList:

src/rdi_from_dwarf/rdi_from_dwarf.c

@@ -3112,11 +3112,13 @@ d2r_convert(Arena *arena, D2R_ConvertParams *params)
   bake_params.udts             = g_d2r_shared.udts;
   bake_params.src_files        = g_d2r_shared.src_files;
   bake_params.line_tables      = g_d2r_shared.line_tables;
+#if 0 // TODO(rjf): @locpass
   bake_params.global_variables = g_d2r_shared.gvars;
   bake_params.thread_variables = g_d2r_shared.tvars;
   bake_params.procedures       = g_d2r_shared.procs;
   bake_params.scopes           = g_d2r_shared.scopes;
   bake_params.inline_sites     = g_d2r_shared.inline_sites;
+#endif
 
   scratch_end(scratch);
   return bake_params;

src/rdi_from_pdb/rdi_from_pdb.c

@@ -3294,7 +3294,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
         U64 sym_constants_chunk_cap = 2048;
         U64 sym_scopes_chunk_cap = 4096;
         U64 sym_inline_sites_chunk_cap = 2048;
-        RDIM_Unit *sym_unit = &all_units_ptr->first->v[sym_idx];
+        RDIM_Unit *sym_unit = &all_units_ptr->first->v[sym_idx > 0 ? sym_idx-1 : 0];
         RDIM_SymbolChunkList *sym_procedures = &sym_unit->procedures;
         RDIM_SymbolChunkList *sym_global_variables = &sym_unit->global_variables;
         RDIM_SymbolChunkList *sym_thread_variables = &sym_unit->thread_variables;

src/rdi_make/rdi_make_local.c

@@ -284,12 +284,25 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
   //////////////////////////////////////////////////////////////
   //- rjf: @rdim_bake_stage form joined symbol lists, from all units
   //
+  typedef struct UnitSymbolRanges UnitSymbolRanges;
+  struct UnitSymbolRanges
+  {
+    U64 procedures_first_idx;
+    U64 procedures_count;
+    U64 global_variables_first_idx;
+    U64 global_variables_count;
+    U64 thread_variables_first_idx;
+    U64 thread_variables_count;
+    U64 constants_first_idx;
+    U64 constants_count;
+  };
   RDIM_SymbolChunkList *all_global_variables = 0;
   RDIM_SymbolChunkList *all_thread_variables = 0;
   RDIM_SymbolChunkList *all_constants = 0;
   RDIM_SymbolChunkList *all_procedures = 0;
   RDIM_ScopeChunkList *all_scopes = 0;
   RDIM_InlineSiteChunkList *all_inline_sites = 0;
+  UnitSymbolRanges *unit_symbol_ranges = 0;
   if(lane_idx() == 0)
   {
     all_global_variables = push_array(scratch.arena, RDIM_SymbolChunkList, 1);
@@ -298,20 +311,28 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
     all_procedures       = push_array(scratch.arena, RDIM_SymbolChunkList, 1);
     all_scopes           = push_array(scratch.arena, RDIM_ScopeChunkList, 1);
     all_inline_sites     = push_array(scratch.arena, RDIM_InlineSiteChunkList, 1);
+    unit_symbol_ranges   = push_array(scratch.arena, UnitSymbolRanges, params->units.total_count);
+    U64 unit_idx = 0;
     for EachNode(unit_n, RDIM_UnitChunkNode, params->units.first)
     {
       for EachIndex(unit_n_idx, unit_n->count)
       {
         RDIM_Unit *unit = &unit_n->v[unit_n_idx];
-        RDIM_SymbolChunkList global_variables_shallow_copy = rdim_symbol_chunk_list_shallow_copy(scratch.arena, &unit->global_variables);
-        RDIM_SymbolChunkList thread_variables_shallow_copy = rdim_symbol_chunk_list_shallow_copy(scratch.arena, &unit->thread_variables);
-        RDIM_SymbolChunkList constants_shallow_copy = rdim_symbol_chunk_list_shallow_copy(scratch.arena, &unit->constants);
-        RDIM_SymbolChunkList procedures_shallow_copy = rdim_symbol_chunk_list_shallow_copy(scratch.arena, &unit->procedures);
-        rdim_symbol_chunk_list_concat_in_place(all_global_variables, &global_variables_shallow_copy);
-        rdim_symbol_chunk_list_concat_in_place(all_thread_variables, &thread_variables_shallow_copy);
-        rdim_symbol_chunk_list_concat_in_place(all_constants, &constants_shallow_copy);
-        rdim_symbol_chunk_list_concat_in_place(all_procedures, &procedures_shallow_copy);
-        // TODO(rjf): @locpass scopes, inline sites
+        unit_symbol_ranges[unit_idx].procedures_first_idx = all_procedures->total_count + 1;
+        unit_symbol_ranges[unit_idx].procedures_count = unit->procedures.total_count;
+        unit_symbol_ranges[unit_idx].global_variables_first_idx = all_global_variables->total_count + 1;
+        unit_symbol_ranges[unit_idx].global_variables_count = unit->global_variables.total_count;
+        unit_symbol_ranges[unit_idx].thread_variables_first_idx = all_thread_variables->total_count + 1;
+        unit_symbol_ranges[unit_idx].thread_variables_count = unit->thread_variables.total_count;
+        unit_symbol_ranges[unit_idx].constants_first_idx = all_constants->total_count + 1;
+        unit_symbol_ranges[unit_idx].constants_count = unit->constants.total_count;
+        rdim_symbol_chunk_list_concat_in_place(all_global_variables, &unit->global_variables);
+        rdim_symbol_chunk_list_concat_in_place(all_thread_variables, &unit->thread_variables);
+        rdim_symbol_chunk_list_concat_in_place(all_constants, &unit->constants);
+        rdim_symbol_chunk_list_concat_in_place(all_procedures, &unit->procedures);
+        rdim_scope_chunk_list_concat_in_place(all_scopes, &unit->scopes);
+        rdim_inline_site_chunk_list_concat_in_place(all_inline_sites, &unit->inline_sites);
+        unit_idx += 1;
       }
     }
   }
@@ -321,6 +342,7 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
   lane_sync_u64(&all_procedures, 0);
   lane_sync_u64(&all_scopes, 0);
   lane_sync_u64(&all_inline_sites, 0);
+  lane_sync_u64(&unit_symbol_ranges, 0);
 
   //////////////////////////////////////////////////////////////
   //- rjf: @rdim_bake_stage bake vmaps
@@ -3328,11 +3350,14 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
     //- rjf: bake units
     ProfScope("bake units")
     {
+      U64 base_unit_idx = 0;
       for EachNode(n, RDIM_UnitChunkNode, params->units.first)
       {
         Rng1U64 range = lane_range(n->count);
         for EachInRange(n_idx, range)
         {
+          U64 unit_idx = base_unit_idx + n_idx;
+          UnitSymbolRanges *symbol_ranges = &unit_symbol_ranges[unit_idx];
           RDIM_Unit *src = &n->v[n_idx];
           RDI_Unit *dst = &baked_units[n->base_idx + n_idx + 1];
           dst->unit_name_string_idx     = rdim_bake_idx_from_string(bake_strings, src->unit_name);
@@ -3343,7 +3368,16 @@ rdim_bake(Arena *arena, RDIM_BakeParams *params)
           dst->build_path_node          = rdim_bake_path_node_idx_from_string(path_tree, src->build_path);
           dst->language                 = src->language;
           dst->line_table_idx           = (RDI_U32)rdim_idx_from_line_table(src->line_table); // TODO(rjf): @u64_to_u32
-        }
+          dst->procedures_first_idx     = (RDI_U32)symbol_ranges->procedures_first_idx; // TODO(rjf): @u64_to_u32
+          dst->procedures_count         = (RDI_U32)symbol_ranges->procedures_count; // TODO(rjf): @u64_to_u32
+          dst->global_variables_first_idx = (RDI_U32)symbol_ranges->global_variables_first_idx; // TODO(rjf): @u64_to_u32
+          dst->global_variables_count     = (RDI_U32)symbol_ranges->global_variables_count; // TODO(rjf): @u64_to_u32
+          dst->thread_variables_first_idx = (RDI_U32)symbol_ranges->thread_variables_first_idx; // TODO(rjf): @u64_to_u32
+          dst->thread_variables_count     = (RDI_U32)symbol_ranges->thread_variables_count; // TODO(rjf): @u64_to_u32
+          dst->constants_first_idx        = (RDI_U32)symbol_ranges->constants_first_idx; // TODO(rjf): @u64_to_u32
+          dst->constants_count            = (RDI_U32)symbol_ranges->constants_count; // TODO(rjf): @u64_to_u32
+        }
+        base_unit_idx += n->count;
       }
     }