trace generator tool (#277)

This PR introduces the Trace Generator Tool along with improvements in
the documentation at traces/stf_trace_archive/README.md and
traces/docker_stf_trace_gen/README.md. The main goal of this tool is to
create STF trace and it's metadata file for workload using docker.

The trace generator tool comes with three different modes
- **Macro** → uses `START_TRACE` and `STOP_TRACE` macros embedded in the
workload
- **Instruction Count (`insn_count`)** → traces a fixed number of
instructions after skipping some
- **Program Counter (`pc_count`)** → traces after a specific program
counter (PC) is reached

Other options included in the tool are:
* **`--isa ISA`**. Instruction set architecture (e.g., `rv64imafdc`).
* **`--dump`**. Create a trace file dump.
* **`--pk`**. Run Spike with **pk (proxy kernel)**.


This PR files follows this structure:
```text
README.md                  # Main documentation file
README.md                  # Trace generator documentation file
src/
├── data/                  # Core data models and classes
├── converters/                  # Converter classes
├── factories/              # Factory classes
├── utils/                 # Utility functions and helpers
└── generate_trace.py       # Main CLI entry point
```

### Quickstart

1. **Macro mode  (SPIKE only)**  
Trace using `START_TRACE` / `STOP_TRACE` markers inside the workload:
    ```bash
   python generate_trace.py --emulator spike macro workload.elf
    ```

2. **Instruction Count mode**
   Skip 1000 instructions, then trace 5000 instructions:

   ```bash
   python generate_trace.py --emulator qemu insn_count \
       --num-instructions 5000 --start-instruction 1000 workload.elf
   ```

3. **Program Counter mode (QEMU only)**
Start tracing after PC `0x80000000` is hit 5 times, trace 2000
instructions:

   ```bash
   python generate_trace.py --emulator qemu pc_count \
--num-instructions 2000 --start-pc 0x80000000 --pc-threshold 5
workload.elf
   ```

### Next Steps:
* Integrate `utils/docker_orchestrator.py` with all docker related
function created on PR #275
* Add a simpoint option, passing a simpoint file as input and generating
traces from it
* Validate and compare trace results generated by both Spike and QEMU
* Look at including a Macro mode for the qemu plugin
* Create test classes

Author: vinicius-r-silva

traces/docker_stf_trace_gen/Dockerfile

@@ -1,16 +1,17 @@
 FROM ubuntu:24.04
 
 # Set environment variables early
-ENV RISCV=/riscv \
-    QEMU_PLUGINS=/qemu/build/contrib/plugins \
-    PATH=$RISCV/bin:/opt/riscv/riscv32-elf/bin:/opt/riscv/riscv64-elf/bin:/opt/riscv/riscv32-glibc/bin:/SimPoint/bin:/qemu/build:$PATH \ 
-    DEBIAN_FRONTEND=noninteractive \
-    WORKDIR=/workspace \
-    WORKLOADS=/workloads \
-    OUTPUT=/output
+ENV RISCV=/riscv
+ENV QEMU_DIR=/qemu
+ENV QEMU_PLUGINS=/qemu/build/contrib/plugins
+ENV PATH=$RISCV/bin:/opt/riscv/riscv32-elf/bin:/opt/riscv/riscv64-elf/bin:/opt/riscv/riscv32-glibc/bin:/SimPoint/bin:/qemu/build:$PATH 
+ENV DEBIAN_FRONTEND=noninteractive
+ENV WORKDIR=/workspace
+ENV WORKLOADS=/workloads
+ENV OUTPUT=/output
 
 # Install dependencies and clean up in one layer
-RUN apt-get update && apt-get install -y \
+RUN apt update && apt install -y \
     autoconf \
     automake \
     autotools-dev \
@@ -53,22 +54,22 @@ RUN apt-get update && apt-get install -y \
     wget \
     zlib1g-dev \
     zstd \
-    && rm -rf /var/lib/apt/lists/* \
-    && apt-get clean
+    python3-yaml
 
 # Configure git for building
 RUN git config --global url."https://github.com/".insteadOf "[email protected]:" && \
     git config --global user.email "[email protected]" && \
     git config --global user.name "Docker Builder"
 
 # Create directory structure
-RUN mkdir -p /workloads /output /workspace $RISCV
+RUN mkdir -p /output
 
 # Clone repositories in RISCV directory
 WORKDIR $RISCV
-RUN git clone https://github.com/condorcomputing/condor.riscv-isa-sim.git --recurse-submodules && \
-    git clone https://github.com/sparcians/stf_tools && \
-    git clone https://github.com/riscv-software-src/riscv-pk.git
+RUN git clone https://github.com/condorcomputing/condor.riscv-isa-sim.git --recurse-submodules
+RUN git clone https://github.com/sparcians/stf_tools
+RUN git clone https://github.com/riscv-software-src/riscv-pk.git
+RUN git clone https://gitlab.com/ribeiro.v.silva/trace-gen.git
 
 # Clone QEMU and SimPoint
 WORKDIR /
@@ -87,15 +88,14 @@ RUN chmod +x $RISCV/get-tool.sh && \
     echo "Toolchain version:" && \
     riscv64-unknown-linux-gnu-gcc --version 2>/dev/null || echo "Toolchain setup pending"
 
-RUN mkdir -p /qemu/build
 # Build QEMU with plugins support
 WORKDIR /qemu/build
 RUN ../configure \
-        --target-list=riscv32-linux-user,riscv64-linux-user,riscv32-softmmu,riscv64-softmmu \
-        --enable-plugins \
-        --disable-docs \
-        --disable-gtk \
-        --disable-sdl 
+    --target-list=riscv32-linux-user,riscv64-linux-user,riscv32-softmmu,riscv64-softmmu \
+    --enable-plugins \
+    --disable-docs \
+    --disable-gtk \
+    --disable-sdl 
 RUN make -j$(nproc)
 RUN make install
 
@@ -116,6 +116,7 @@ RUN ../configure --prefix=$RISCV/condor.riscv-isa-sim/install
 RUN make -j$(nproc) 
 RUN make regress 
 RUN make install
+ENV STF_DIR=$RISCV/condor.riscv-isa-sim/stf_lib
 
 # Create mount points for runtime mounting
 # Environment and flow scripts will be mounted at runtime
@@ -127,9 +128,18 @@ RUN mkdir -p /workloads/environment /flow /outputs
 # - Host outputs -> /outputs
 
 RUN cp $RISCV/condor.riscv-isa-sim/install/bin/spike /usr/bin/
-WORKDIR /workspace 
 
+WORKDIR $RISCV/trace-gen
+RUN make 
+RUN make install
 
-CMD ["/bin/bash"]
+WORKDIR $RISCV/riscv-pk/build
+RUN mkdir $RISCV/pk
+RUN ../configure --prefix=$RISCV/pk --host=riscv64-unknown-elf
+RUN make -j$(nproc)
+RUN make install
+ENV PATH=$RISCV/pk:$PATH 
+
+WORKDIR /workspace 
 
-# need to mount Volumes and show it in the documenation when runnignthis 
+CMD ["/bin/bash"]

traces/docker_stf_trace_gen/README.md

@@ -298,12 +298,9 @@ QEMU advantage: 2.70x faster
 More in [doc/emulator-comparison](doc/emulator-comparison)
 
 
-### Trace Generation
-**Important**: QEMU and Spike use different trace formats:
-- **Spike**: Detailed STF (System Trace Format) traces for comprehensive analysis
-- **QEMU**: Simple assembly traces using `-d in_asm` output
+### STF Trace Generation
 
-QEMU cannot generate STF traces, making it useful for running large files and basic traces, while Spike provides detailed tracing capabilities.
+Read the [generate trace](generate_trace.md) file for details.
 
 ## Documentation
 

traces/docker_stf_trace_gen/build_workload.py

@@ -3,7 +3,7 @@
 import argparse
 from pathlib import Path
 from typing import List
-from utils.util import log, LogLevel, run_cmd, clean_dir, file_exists, write_file_lines
+from utils.util import Util, LogLevel
 from utils.config import BoardConfig
 
 DEFAULT_WORKLOADS = {
@@ -21,7 +21,7 @@ def __init__(self, board: str, arch: str, platform: str, bbv: bool, trace: bool)
         self.bbv = bbv
         self.trace = trace
         self.config = BoardConfig(board)
-        self.bin_dir = clean_dir(Path(f"/workloads/bin/{board}"))
+        self.bin_dir = Util.clean_dir(Path(f"/workloads/bin/{board}"))
         self.env_dir = Path(f"/workloads/environment/{board}")
         self.executables = []
 
@@ -41,14 +41,14 @@ def _get_flags(self, config: dict, workload_path: Path, workload_type: str, benc
     def build_environment(self, workload: str):
         """Compile environment runtime files."""
         if self.config.should_skip_environment(self.platform, workload):
-            log(LogLevel.INFO, f"Skipping environment build for {self.platform}")
+            Util.log(LogLevel.INFO, f"Skipping environment build for {self.platform}")
             return
         cc, cflags, _, _ = self._get_flags({}, workload, workload)
         for src in self.config.get_environment_files(workload):
             src_file = self.env_dir / src
             if src_file.exists():
                 obj = self.env_dir / f"{Path(src).stem}.o"
-                run_cmd([cc, "-c", *cflags, "-o", str(obj), str(src_file)])
+                Util.run_cmd([cc, "-c", *cflags, "-o", str(obj), str(src_file)])
 
     def build_common_files(self, workload_path: Path, workload_type: str) -> List[str]:
         """Compile common files for riscv-tests."""
@@ -61,29 +61,29 @@ def build_common_files(self, workload_path: Path, workload_type: str) -> List[st
             if c_file.name in skip:
                 continue
             obj = self.bin_dir / f"{c_file.stem}.o"
-            if run_cmd([cc, "-c", *cflags, "-o", str(obj), str(c_file)]):
+            if Util.run_cmd([cc, "-c", *cflags, "-o", str(obj), str(c_file)]):
                 obj_files.append(str(obj))
         return obj_files
 
     def build_benchmark(self, bench: str, workload_path: Path, workload_type: str, common_objs: List[str]):
         """Compile and link a single benchmark."""
-        log(LogLevel.INFO, f"Building {bench}")
+        Util.log(LogLevel.INFO, f"Building {bench}")
         bench_dir = workload_path / ("src" if workload_type == "embench-iot" else "benchmarks") / bench
         if not bench_dir.exists():
-            log(LogLevel.ERROR, f"Benchmark directory not found: {bench_dir}")
+            Util.log(LogLevel.ERROR, f"Benchmark directory not found: {bench_dir}")
 
         # Find source files
         source_exts = ['.c'] if workload_type == "embench-iot" else ['.c', '.S']
         sources = [f for ext in source_exts for f in bench_dir.glob(f"*{ext}")]
         if not sources:
-            log(LogLevel.ERROR, f"No sources found for {bench}")
+            Util.log(LogLevel.ERROR, f"No sources found for {bench}")
 
         # Compile sources
         cc, cflags, ldflags, config = self._get_flags({}, workload_path, workload_type, bench)
         obj_files = []
         for src in sources:
             obj = self.bin_dir / f"{src.stem}.o"
-            if run_cmd([cc, "-c", *cflags, "-o", str(obj), str(src)]):
+            if Util.run_cmd([cc, "-c", *cflags, "-o", str(obj), str(src)]):
                 obj_files.append(str(obj))
 
         # Compile additional sources for embench-iot
@@ -92,7 +92,7 @@ def build_benchmark(self, bench: str, workload_path: Path, workload_type: str, c
                 src_path = Path(src)
                 if src_path.exists():
                     obj = self.bin_dir / f"{src_path.stem}_support.o"
-                    if run_cmd([cc, "-c", *cflags, "-o", str(obj), str(src_path)]):
+                    if Util.run_cmd([cc, "-c", *cflags, "-o", str(obj), str(src_path)]):
                         obj_files.append(str(obj))
 
         # Link executable
@@ -104,7 +104,7 @@ def build_benchmark(self, bench: str, workload_path: Path, workload_type: str, c
             link_cmd.extend([f"-T{self.env_dir / config.get('linker_script', 'link.ld')}",
                              *[str(self.env_dir / f"{Path(f).stem}.o") for f in self.config.get_environment_files(workload_type)]])
         link_cmd.extend(config.get('libs', []))
-        if run_cmd(link_cmd):
+        if Util.run_cmd(link_cmd):
             self.executables.append(str(exe))
 
     def list_benchmarks(self, workload_path: Path, workload_type: str) -> List[str]:
@@ -118,18 +118,18 @@ def build_workload(self, workload: str, benchmark: str = None, custom_path: str
         """Build specified workload or benchmark."""
         workload_path = Path(custom_path or DEFAULT_WORKLOADS.get(workload, DEFAULT_WORKLOADS["riscv-tests"]))
         workload_type = workload if workload in DEFAULT_WORKLOADS else "custom"
-        if not file_exists(workload_path):
-            log(LogLevel.ERROR, f"Workload path not found: {workload_path}")
+        if not Util.file_exists(workload_path):
+            Util.log(LogLevel.ERROR, f"Workload path not found: {workload_path}")
 
-        log(LogLevel.INFO, f"Building {workload} for {self.arch}/{self.platform}/{self.board}")
+        Util.log(LogLevel.INFO, f"Building {workload} for {self.arch}/{self.platform}/{self.board}")
         self.build_environment(workload_type)
         common_objs = self.build_common_files(workload_path, workload_type)
         benchmarks = [benchmark] if benchmark else (["dhrystone"] if workload == "dhrystone" else self.list_benchmarks(workload_path, workload_type))
 
         for bench in benchmarks:
             self.build_benchmark(bench, workload_path, workload_type, common_objs)
 
-        log(LogLevel.INFO, f"Built {len(self.executables)} executables in {self.bin_dir}")
+        Util.log(LogLevel.INFO, f"Built {len(self.executables)} executables in {self.bin_dir}")
 
 def main():
     """Main entry point for building workloads."""
@@ -147,16 +147,16 @@ def main():
 
     if args.list:
         for name, path in DEFAULT_WORKLOADS.items():
-            if file_exists(path):
-                log(LogLevel.INFO, f"{name}: {path}")
+            if Util.file_exists(path):
+                Util.log(LogLevel.INFO, f"{name}: {path}")
                 builder = WorkloadBuilder(args.board, args.arch, args.platform, args.bbv, args.trace)
                 benchmarks = builder.list_benchmarks(Path(path), name)
                 if benchmarks:
-                    log(LogLevel.INFO, f"  Benchmarks: {', '.join(benchmarks[:10])}{'...' if len(benchmarks) > 10 else ''}")
+                    Util.log(LogLevel.INFO, f"  Benchmarks: {', '.join(benchmarks[:10])}{'...' if len(benchmarks) > 10 else ''}")
         return
 
     if not args.workload:
-        log(LogLevel.ERROR, "Workload required. Use --list to see available workloads")
+        Util.log(LogLevel.ERROR, "Workload required. Use --list to see available workloads")
     builder = WorkloadBuilder(args.board, args.arch, args.platform, args.bbv, args.trace)
     builder.build_workload(args.workload, args.benchmark, args.custom_path)
 

traces/docker_stf_trace_gen/converters/base.py

@@ -0,0 +1,8 @@
+from abc import ABC
+from typing import Any
+
+
+class BaseConverter(ABC):
+    @staticmethod
+    def convert(self, input: Any) -> Any:
+        raise NotImplementedError("This method should be overridden by subclasses.")

traces/docker_stf_trace_gen/converters/host_to_docker_path.py

@@ -0,0 +1,11 @@
+import os
+from converters.base import BaseConverter
+from data.consts import Const
+
+
+class HostToDockerPathConverter(BaseConverter):
+    @staticmethod
+    def convert(path: str) -> str:
+        parts = os.path.abspath(path).strip(os.sep).split(os.sep)
+        parts.insert(0, Const.DOCKER_TEMP_FOLDER)
+        return os.path.join(*parts)

traces/docker_stf_trace_gen/data/consts.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Const():
+    DOCKER_IMAGE_NAME = "riscv-perf-model:latest"
+    DOCKER_TEMP_FOLDER = "/host"
+    LIBSTFMEM = "/usr/lib/libstfmem.so"
+    STF_TOOLS = "/riscv/stf_tools/release/tools"
+    SPKIE_PK = "/riscv/riscv-pk/build/pk"

traces/docker_stf_trace_gen/data/metadata.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass
+from typing import Literal, Optional, Dict, Union
+
+
+@dataclass
+class Author:
+    email: str
+    name: Optional[str] = None
+    company: Optional[str] = None
+
+
+@dataclass
+class Workload:
+    filename: str
+    SHA256: str
+    execution_command: Optional[str]
+    elf_sections: Dict[str, str]
+
+
+@dataclass
+class IpModeInterval:
+    ip: int
+    ip_count: int
+    interval_lenght: int
+
+
+@dataclass
+class InstructionCountModeInterval:
+    start_instruction: int
+    interval_lenght: int
+
+
+@dataclass
+class Stf:
+    timestamp: str
+    stf_trace_info: Dict[str, str]
+    interval_mode: Literal["ip", "instructionCount", "macro", "fullyTrace"]
+    interval: Optional[Union[IpModeInterval, InstructionCountModeInterval]] = None
+
+
+@dataclass
+class Metadata:
+    author: Author
+    workload: Workload
+    stf: Stf
+    description: Optional[str] = None