MX-17306 Header V3 fixes

Author: mradian1

README.md

@@ -43,4 +43,5 @@ pip install -r ./requirements-dev.txt --upgrade
 EXAMPLE USAGE
 ```
 python -m multiversx_cross_shard_analysis.gather_data --path /home/mihaela/Downloads/cross-shard-execution-anal-9afe696daf.zip
+python -m multiversx_cross_shard_analysis.headers_timeline_report --run-name cross-shard-execution-anal-6cc663f7af
 ```

multiversx_cross_shard_analysis/gather_data.py

@@ -26,6 +26,7 @@ def gather_data():
     # Generate reports
     mb_data = MiniblockData(handler.shard_data.miniblocks).get_data_for_round_report()
     out_folder = os.path.join(handler.run_name, "MiniblocksShardTimeline")
+    out_folder = os.path.join('Reports', out_folder)
     os.makedirs(out_folder, exist_ok=True)
 
     # generate PDFs per epoch
@@ -38,6 +39,7 @@ def gather_data():
 
     mb_data = MiniblockData(handler.shard_data.miniblocks).get_data_for_detail_report()
     out_folder = os.path.join(handler.run_name, "MiniblocksTimelineDetail")
+    out_folder = os.path.join('Reports', out_folder)
     os.makedirs(out_folder, exist_ok=True)
 
     for epoch in sorted(mb_data.keys()):
@@ -48,6 +50,7 @@ def gather_data():
 
     input_data = handler.shard_data.get_data_for_header_horizontal_report()
     out_folder = os.path.join(handler.run_name, "NonceTimeline")
+    out_folder = os.path.join('Reports', out_folder)
     os.makedirs(out_folder, exist_ok=True)
 
     for epoch in sorted(input_data.keys()):

multiversx_cross_shard_analysis/header_structures.py

@@ -58,6 +58,8 @@ def __init__(self):
         self.seen_miniblock_hashes = set()
 
     def add_node(self, node_data: HeaderData):
+        if node_data.header_dictionary['commited_headers'] == []:
+            node_data.header_dictionary['commited_headers'] = node_data.header_dictionary['proposed_headers'].copy()
         for header_status in node_data.header_dictionary.keys():
             for header in node_data.header_dictionary[header_status]:
                 shard_id = get_shard_id(header)
@@ -74,102 +76,16 @@ def add_node(self, node_data: HeaderData):
     def add_miniblocks(self, header: dict[str, Any], status: str):
         header_struct = Header(header, status)
 
-        for mention_type, mb in header_struct.miniblocks:
+        for mention_type, mb, metadata in header_struct.miniblocks:
             mb_hash = mb.get('hash')
             if mb_hash not in self.seen_miniblock_hashes:
                 self.seen_miniblock_hashes.add(mb_hash)
                 self.miniblocks[mb_hash] = mb.copy()
                 self.miniblocks[mb_hash]['mentioned'] = []
-            metadata = header_struct.metadata.copy()
+            # metadata = header_struct.metadata.copy()
             metadata["reserved"] = decode_reserved_field(mb.get("reserved", ""), mb.get("txCount", 0))
             self.miniblocks[mb_hash]['mentioned'].append((mention_type, metadata))
 
-    def get_data_for_header_vertical_report(self) -> dict[str, dict[int, Any]]:
-        miniblocks = MiniblockData(self.miniblocks)
-        report: dict[str, dict[int, Any]] = {}
-        last_epoch = None
-
-        for shard_id, header_data in self.parsed_headers.items():
-            header_group_count = 0
-            header_count = 0
-            header_group_name = ""
-            last_epoch = None
-
-            for header in sorted(header_data.header_dictionary['commited_headers'],
-                                 key=lambda x: get_value('nonce', x)):
-
-                epoch = get_value('epoch', header)
-
-                # reset counters when epoch changes
-                if epoch != last_epoch:
-                    header_group_count = 0
-                    header_count = 0
-                    header_group_name = ""
-                    last_epoch = epoch
-
-                # ensure epoch entry exists and contains all shards as keys
-                if epoch not in report:
-                    report[epoch] = {sid: {} for sid in self.parsed_headers.keys()}
-
-                if get_value('miniBlockHeaders', header) == []:
-                    continue
-
-                nonce = get_value('nonce', header)
-                round_num = get_value('round', header)
-                print(f"Processing header: epoch={epoch}, shard={shard_id}, nonce={nonce}, round={round_num}")
-
-                # build result for this header (only cross-shard miniblocks)
-                result: dict[int, list] = {}
-                for miniblock in [mb for mb in get_value('miniBlockHeaders', header) if mb.get('senderShardID') == shard_id and mb.get('receiverShardID') != mb.get('senderShardID')]:
-                    print(f"  Processing miniblock: hash={miniblock.get('hash')}, senderShardID={miniblock.get('senderShardID')}, receiverShardID={miniblock.get('receiverShardID')}")
-                    mb_hash = miniblock.get('hash')
-                    for mention_type, metadata in self.miniblocks[mb_hash]['mentioned']:
-                        # skip proposed mentions
-                        if 'proposed' in mention_type:
-                            continue
-
-                        rn = metadata['round']
-                        color = miniblocks.get_color_for_state(mention_type, miniblock['txCount'], metadata)
-                        shard_name = f'Shard {metadata["shard_id"]}' if metadata["shard_id"] != 4294967295 else "MetaShard"
-                        # append tuple (label, info, color)
-                        result.setdefault(rn, []).append((shard_name, mb_hash[:15] + '...', COLORS_MAPPING[color]))
-
-                # if result empty -> we don't include this nonce at all, don't count it
-                if not result:
-                    continue
-
-                # --- Add this nonce to the report and handle grouping ---
-                # if group start (every 5 actual added nonces)
-                group_size = 5
-                if header_count % group_size == 0:
-                    header_group_count += 1
-                    header_group_name = f"Nonces {nonce}"
-                    # initialize structure for this group
-                    report[epoch][shard_id][header_group_count] = {
-                        'group_name': header_group_name,
-                        'rounds': (round_num, round_num),
-                        'nonces': {}
-                    }
-                    print(f"Creating new header group: {header_group_name}")
-                else:
-                    # extend existing group's name
-                    header_group_name += f" - {nonce}"
-                    report[epoch][shard_id][header_group_count]['group_name'] = header_group_name
-
-                # store the nonce's data
-                report[epoch][shard_id][header_group_count]['nonces'][nonce] = result
-
-                # update group's rounds min/max based on result keys
-                min_r, max_r = report[epoch][shard_id][header_group_count]['rounds']
-                actual_min = min(result.keys())
-                actual_max = max(result.keys())
-                report[epoch][shard_id][header_group_count]['rounds'] = (min(min_r, actual_min), max(max_r, actual_max))
-
-                # increment header_count because we added this nonce
-                header_count += 1
-
-        return report
-
     def get_data_for_header_horizontal_report(self) -> dict[str, dict[int, Any]]:
         miniblocks = MiniblockData(self.miniblocks)
         report: dict[str, dict[int, Any]] = {}
@@ -193,7 +109,7 @@ def get_data_for_header_horizontal_report(self) -> dict[str, dict[int, Any]]:
 
                 # build result for this header (only cross-shard miniblocks)
                 result: dict[int, list] = {}
-                for miniblock in [mb for mb in get_value('miniBlockHeaders', header) if mb.get('senderShardID') == shard_id and mb.get('receiverShardID') != mb.get('senderShardID')]:
+                for miniblock in [mb for mb in get_value('miniBlockHeaders', header) if mb.get('senderShardID') == shard_id]:
                     mb_hash = miniblock.get('hash')
                     for mention_type, metadata in self.miniblocks[mb_hash]['mentioned']:
                         # skip proposed mentions
@@ -228,7 +144,7 @@ def get_data_for_header_horizontal_report(self) -> dict[str, dict[int, Any]]:
 class Header:
     def __init__(self, header: dict[str, Any], status: str):
         self.metadata: dict[str, Any] = self.get_header_metadata(header)
-        self.miniblocks: list[tuple[str, dict[str, Any]]] = self.get_miniblocks(header, status)
+        self.miniblocks: list[tuple[str, dict[str, Any], dict[str, Any]]] = self.get_miniblocks(header, status)
 
     # returns 'origin' or 'dest' based on miniblock senderShardID
     def get_miniblock_shard_type(self, miniblock_shard_id: int) -> str:
@@ -244,29 +160,34 @@ def get_header_metadata(self, header: dict[str, Any]) -> dict[str, Any]:
             "shard_id": header.get('shardID', 4294967295),
         }
 
-    def get_miniblocks(self, header: dict[str, Any], status: str) -> list[tuple[str, dict[str, Any]]]:
+    def get_miniblocks(self, header: dict[str, Any], status: str) -> list[tuple[str, dict[str, Any], dict[str, Any]]]:
         miniblocks = []
         if Header.isHeaderV2(header):
             header = header['header']
         for miniblock in header.get('miniBlockHeaders', []):
             miniblock_mention = self.get_miniblock_shard_type(miniblock["senderShardID"]) + f'_{status}'
-            miniblocks.append((miniblock_mention, miniblock))
+            miniblocks.append((miniblock_mention, miniblock, self.metadata.copy()))
         if Header.isMetaHeader(header):
             for shard_header in header['shardInfo']:
                 shard_metadata = self.get_header_metadata(shard_header)
                 for miniblock in shard_header.get('shardMiniBlockHeaders', []):
                     miniblock_mention = f'{meta}_{origin_shard if shard_metadata['shard_id'] == miniblock['senderShardID'] else dest_shard}_{status}'
-                    miniblocks.append((miniblock_mention, miniblock))
+                    miniblocks.append((miniblock_mention, miniblock, self.metadata.copy()))
                 if Header.isMetaHeaderV3(header):
                     for exec_result in shard_header.get('executionResults', []):
                         for miniblock in exec_result.get('miniBlockHeaders', []):
                             miniblock_mention = f'{meta}_{origin_shard if shard_metadata["shard_id"] == miniblock["senderShardID"] else dest_shard}_exec_{status}'
-                            miniblocks.append((miniblock_mention, miniblock))
+                            miniblocks.append((miniblock_mention, miniblock, self.metadata.copy()))
         if Header.isHeaderV3(header):
             for exec_result in header['executionResults']:
+                base_exec_result = exec_result.get('baseExecutionResult', {})
+                exec_result_metadata = self.metadata.copy()
+                exec_result_metadata['nonce'] = base_exec_result.get('headerNonce', 0)
+
                 for miniblock in exec_result.get('miniBlockHeaders', []):
                     miniblock_mention = self.get_miniblock_shard_type(miniblock["senderShardID"]) + f'_{status}_exec'
-                    miniblocks.append((miniblock_mention, miniblock))
+                    exec_result_metadata['exec_result_hash'] = miniblock.get('hash', '')
+                    miniblocks.append((miniblock_mention, miniblock, exec_result_metadata.copy()))
 
         return miniblocks
 

multiversx_cross_shard_analysis/headers_timeline_report.py

@@ -9,12 +9,14 @@
 from reportlab.lib.pagesizes import A4
 from reportlab.lib.styles import getSampleStyleSheet
 from reportlab.platypus import (Flowable, PageBreak, Paragraph,
-                                SimpleDocTemplate, Spacer, Table, TableStyle)
+                                SimpleDocTemplate, Spacer, LongTable, TableStyle)
 
 from multiversx_cross_shard_analysis.constants import COLORS_MAPPING, Colors
 from multiversx_cross_shard_analysis.header_structures import (HeaderData,
                                                                ShardData)
 
+from multiversx_cross_shard_analysis.miniblock_data import MiniblockData
+
 # -----------------------------
 # CONFIG (mirrors miniblock report)
 # -----------------------------
@@ -38,32 +40,35 @@
 # build stacked rectangles (same as miniblock version)
 # -----------------------------
 
-def build_stack_for_round(items: list[tuple[str, str, colors.Color]], col_width: float) -> Drawing:
-    rows = max(1, len(items))
-    total_h = rows * RECT_H
-    d = Drawing(col_width, total_h)
+def build_stack_rows(items: list[tuple[str, str, colors.Color]], col_width: float) -> list[Drawing]:
+    """
+    Instead of one giant Drawing, we return a list of small ones.
+    Each drawing represents one row in the vertical stack.
+    """
+    row_drawings = []
+
+    if len(items) == 0:
+        # Create a single "no data" row
+        d = Drawing(col_width, RECT_H)
+        rect_w = max(2, col_width - RECT_PADDING_X * 2) - 4
+        d.add(Rect(0, 2, rect_w, 12, fillColor=colors.whitesmoke, strokeColor=colors.grey))  # type: ignore
+        d.add(String(RECT_PADDING_X + 2, 6, "no data", fontSize=RECT_LABEL_FONT))
+        row_drawings.append(d)
+        return row_drawings
 
-    y = total_h - RECT_H
     for label, info, col in items:
+        # Create a small drawing for just this one item
+        d = Drawing(col_width, RECT_H)
         rect_w = max(2, col_width - RECT_PADDING_X * 2) - 4
 
-        d.add(Rect(0, y + 2, rect_w, RECT_H - 4, fillColor=col, strokeColor=colors.black))  # type: ignore
+        d.add(Rect(0, 2, rect_w, RECT_H - 4, fillColor=col, strokeColor=colors.black))  # type: ignore
 
         text_x = RECT_PADDING_X + 3
-        base_y = y + 4
+        d.add(String(text_x, 12, label, fontSize=RECT_LABEL_FONT))
+        d.add(String(text_x, 4, info, fontSize=RECT_INFO_FONT))
+        row_drawings.append(d)
 
-        d.add(String(text_x, base_y + 8, label, fontSize=RECT_LABEL_FONT))
-        d.add(String(text_x, base_y, info, fontSize=RECT_INFO_FONT))
-
-        y -= RECT_H
-
-    if len(items) == 0:
-        rect_w = max(2, col_width - RECT_PADDING_X * 2) - 4
-        mid = total_h / 2
-        d.add(Rect(0, mid - 6, rect_w, 12, fillColor=colors.whitesmoke, strokeColor=colors.grey))   # type: ignore
-        d.add(String(RECT_PADDING_X + 2, mid - 2, "no data", fontSize=RECT_LABEL_FONT))
-
-    return d
+    return row_drawings
 
 # -----------------------------
 # check for round gaps
@@ -96,29 +101,43 @@ def build_nonce_section(shard_id: int, nonce: int, rounds: list[int], data: dict
     num_cols = len(rounds)
     col_width = usable_width / max(1, num_cols)
 
+    # 1. Build the Header Row
     header = [Paragraph(f"<b>{r}</b>", styles["BodyText"]) for r in rounds]
 
-    cells = []
-    for r in rounds:
-        items = data.get(r, [])
-        drawing = build_stack_for_round(items, col_width)
-        cells.append(drawing)
-
-    tbl = Table(
-        [header, cells],
+    # 2. Transpose the stacks into rows
+    # We need to find the max height among all columns to normalize the row count
+    column_stacks = [build_stack_rows(data.get(r, []), col_width) for r in rounds]
+    max_rows = max(len(stack) for stack in column_stacks)
+
+    table_data = [header]
+
+    # Fill the table row by row
+    for i in range(max_rows):
+        row = []
+        for stack in column_stacks:
+            if i < len(stack):
+                row.append(stack[i])
+            else:
+                row.append("")  # Empty cell if this column has fewer items
+        table_data.append(row)
+
+    tbl = LongTable(
+        table_data,
         colWidths=[col_width] * num_cols,
         hAlign="LEFT",
+        splitByRow=True,  # This allows the table to break across pages between rows
     )
 
     tbl_style = [
         ("GRID", (0, 0), (-1, -1), 0.25, colors.grey),
         ("BACKGROUND", (0, 0), (-1, 0), colors.whitesmoke),
         ("ALIGN", (0, 0), (-1, 0), "CENTER"),
         ("VALIGN", (0, 1), (-1, -1), "TOP"),
+        ("TOPPADDING", (0, 0), (-1, -1), 0),  # Tighten padding for large lists
+        ("BOTTOMPADDING", (0, 0), (-1, -1), 0),
         ("FONTSIZE", (0, 0), (-1, 0), ROUND_HEADER_FONT),
     ]
 
-    # add red border if highlighted
     if highlight:
         tbl_style.append(("BOX", (0, 0), (-1, -1), 2, colors.red))
 
@@ -245,7 +264,8 @@ def build_nonce_timeline_pdf(shards_data: dict[int, dict[int, dict[int, list[Any
 }
 
 
-if __name__ == "__main__":
+def main():
+
     parser = argparse.ArgumentParser(description="Nonce timeline report generator")
 
     group = parser.add_mutually_exclusive_group(required=True)
@@ -298,7 +318,7 @@ def build_nonce_timeline_pdf(shards_data: dict[int, dict[int, dict[int, list[Any
         headers.miniblocks = data["miniblocks"]
 
     # process
-    input_data = headers.get_data_for_header_horizontal_report()
+    input_data = MiniblockData(headers.miniblocks).get_data_for_header_report()
 
     # output path
     out_folder = os.path.join(base_path, "NonceTimeline")
@@ -308,3 +328,7 @@ def build_nonce_timeline_pdf(shards_data: dict[int, dict[int, dict[int, list[Any
         outfile = os.path.join(out_folder, f"nonce_timeline_report_{epoch}.pdf")
         build_nonce_timeline_pdf(input_data[epoch], outname=outfile)
         print(f"Nonce timeline report for Epoch {epoch} generated: {outfile}")
+
+
+if __name__ == "__main__":
+    main()