WIP: Statistic data within all environment again

Author: Mes0903

statistic/README

@@ -1,6 +1,7 @@
 1: 13th Gen Intel(R) Core(TM) i7-13700
 2: Intel(R) Xeon(R) CPU E5-2640 v4 @ 2.40GHz
 3: Intel(R) Core(TM) i7-1065G7 CPU @ 1.30GHz
-4: arm (ThunderX2-99xx)
-5: mag
-6: M2 Pro Max
+4: 13th Gen Intel(R) Core(TM) i9-13900H
+5: arm (ThunderX2-99xx)
+6: mag
+7: M2 Pro Max

analyze.sh statistic/analyze_results.shanalyze.sh renamed to statistic/analyze_results.sh

@@ -24,8 +24,8 @@
 #
 # We specifically remove any ANSI color codes and ensure each line is a single line, so the output doesn't break.
 
-LOGDIR="logs"           # Directory containing the log files
-OUTFILE="results_summary.txt"
+LOGDIR="logs-4"           # Directory containing the log files
+OUTFILE="results_summary-4.txt"
 
 # Print header (11 columns)
 echo -e "SMP\treal_boot_time\ttimes_called\tns_per_call\tpredict_sec\tscale_factor\ttest_total_clocksource_ns\treal_total_clocksource_ns\tpercentage\treal_ns_per_call\tdiff_ns_per_call" > "$OUTFILE"

statistic/diff_boot_time.png

@@ -38,12 +38,13 @@
 import matplotlib.pyplot as plt
 
 ENV_FILES = [
-    "results_summary_1.txt",
-    "results_summary_2.txt",
-    "results_summary_3.txt",
-    "results_summary_4.txt",
-    "results_summary_5.txt",
-    "results_summary_6.txt"
+    "results_summary-1.txt",
+    "results_summary-2.txt",
+    "results_summary-3.txt",
+    "results_summary-4.txt",
+    "results_summary-5.txt",
+    "results_summary-6.txt",
+    "results_summary-6.txt"
 ]
 
 ENV_NAMES = [
@@ -53,20 +54,21 @@
     "4: Env4",
     "5: Env5",
     "6: Env6",
+    "7: Env7",
 ]
 
 # A dictionary for column labels (2..11):
 COL_LABELS = {
     2: "real_boot_time",
     3: "times_called",
-    4: "predict_ns_per_call",
-    5: "predict_boot_time",
+    4: "predict ns_per_call",
+    5: "predict boot time",
     6: "scale_factor",
-    7: "test_total_clocksource_ns",
-    8: "real_total_clocksource_ns",
-    9: "percentage",
-    10: "real_ns_per_call",
-    11: "diff_ns_per_call",
+    7: "test total clocksource ns",
+    8: "real total clocksource ns",
+    9: "percentage of clocksource",
+    10: "real ns_per_call",
+    11: "real - predict ns_per_call",
 }
 
 def main():
@@ -75,7 +77,7 @@ def main():
         label = COL_LABELS[col]
 
         plt.figure(figsize=(8, 6))
-        plt.title(f"{label} vs. SMP")
+        plt.title(f"{label}")
         plt.xlabel("SMP")
         plt.ylabel(label)
 

statistic/diff_ns_per_call.png

@@ -0,0 +1,149 @@
+import os
+import matplotlib.pyplot as plt
+
+def parse_results_summary(summary_path):
+    """
+    Reads a results_summary_N.txt file and returns a dict:
+       {smp_value: (percentage, predicted_ns_per_call, real_ns_per_call)}
+    """
+    smp_info = {}
+    with open(summary_path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+
+            parts = line.split()
+            # Skip the header line if it's present
+            if parts[0] == "SMP":
+                continue
+            
+            # Columns in summary file (1-based):
+            # 1: SMP
+            # 2: real_boot_time
+            # 3: times_called
+            # 4: ns_per_call        (predicted ns)
+            # 5: predict_sec
+            # 6: scale_factor
+            # 7: test_total_clocksource_ns
+            # 8: real_total_clocksource_ns
+            # 9: percentage
+            # 10: real_ns_per_call
+            # 11: diff_ns_per_call
+            #
+            # Zero-based index references:
+            # parts[0]  -> SMP
+            # parts[3]  -> ns_per_call (predicted)
+            # parts[8]  -> percentage
+            # parts[9]  -> real_ns_per_call
+            if len(parts) >= 10:
+                try:
+                    smp_val = int(parts[0])
+                    predicted_ns_per_call = float(parts[3])
+                    percentage = float(parts[8])
+                    smp_info[smp_val] = (percentage, predicted_ns_per_call)
+                except ValueError:
+                    continue
+
+    return smp_info
+
+def parse_and_scale_time_log(time_log_path, scale_factor):
+    scaled_values = []
+    with open(time_log_path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith("diff:"):
+                # e.g. "diff: 50176382"
+                parts = line.split(":")
+                if len(parts) == 2:
+                    try:
+                        val_ns = float(parts[1].strip())     # raw ns
+                        val_ms = val_ns / 1_000_000          
+                        scaled_val = val_ms * scale_factor   # scale by 'percentage'
+                        scaled_values.append(scaled_val)
+                    except ValueError:
+                        pass
+    return scaled_values
+
+def main():
+    base_statistic_dir = "."  # Your base folder for logs & summaries
+    num_summaries = 7                 # We have results_summary_1.txt ... results_summary_6.txt
+    num_smp = 32                      # Each summary covers SMP=1..32
+    
+    for i in range(1, num_summaries + 1):
+        summary_file = os.path.join(base_statistic_dir, f"results_summary-{i}.txt")
+        time_log_dir = os.path.join(base_statistic_dir, f"time_log-{i}")  # e.g. statistic/time_log-1
+        output_dir   = os.path.join(base_statistic_dir, f"plots-{i}")
+        
+        # Create output directory for plots
+        os.makedirs(output_dir, exist_ok=True)
+        
+        # 1) Parse the summary file => { SMP: (percentage, predicted_ns, real_ns) }
+        smp_data = parse_results_summary(summary_file)
+        print(f"[INFO] Parsed {len(smp_data)} SMP entries from {summary_file}")
+
+        # We'll store an average (of scaled diff) per SMP for a "trend" plot
+        averages = []
+        
+        for smp_val in range(1, num_smp + 1):
+            # Retrieve info for this SMP
+            if smp_val not in smp_data:
+                print(f"[WARNING] SMP={smp_val} not found in {summary_file}. Skipping.")
+                averages.append(0.0)
+                continue
+            
+            percentage, pred_ns_per_call = smp_data[smp_val]
+            
+            # 2) Parse & scale the time_log_{SMP}.txt
+            time_log_path = os.path.join(time_log_dir, f"time_log_{smp_val}.txt")
+            if not os.path.exists(time_log_path):
+                print(f"[WARNING] File not found: {time_log_path}")
+                averages.append(0.0)
+                continue
+            
+            scaled_values = parse_and_scale_time_log(time_log_path, percentage)
+            if not scaled_values:
+                print(f"[WARNING] No valid data in {time_log_path}")
+                averages.append(0.0)
+                continue
+            
+            # 3) Plot scaled diff values (blue line)
+            plt.figure(figsize=(8, 4))
+            plt.plot(scaled_values, color='blue', marker=None, linewidth=1, linestyle='-',
+                     label='real ns_per_call (average per 1e6 times)')
+            
+            # 4) Add horizontal lines for predicted (red) + real (green) ns_per_call
+            plt.axhline(y=pred_ns_per_call, color='red', linestyle='--',
+                        label='Predicted ns_per_call')
+            
+            plt.title(f"SMP={smp_val} --- real vs. predict ns_per_call\n"
+                      f"(percentage={percentage}, pred={pred_ns_per_call:.3f}ns)")
+            plt.xlabel("per 1e6 times called")
+            plt.ylabel("Nanoseconds (ns)")
+            plt.legend()
+            
+            # Save the figure for this SMP
+            plot_path = os.path.join(output_dir, f"time_log_{smp_val}_scaled.png")
+            plt.savefig(plot_path, dpi=150)
+            plt.close()
+            
+            # Compute average of scaled diff
+            avg_val = sum(scaled_values) / len(scaled_values)
+            averages.append(avg_val)
+        
+        # 5) Plot the average trend (per SMP)
+        smp_indices = list(range(1, num_smp + 1))
+        plt.figure(figsize=(8, 4))
+        plt.plot(smp_indices, averages, marker=None, linewidth=1)
+        plt.title(f"Average scaled diff (ns) across SMP=1..{num_smp} (Set {i})")
+        plt.xlabel("SMP #")
+        plt.ylabel("Average scaled diff (ns)")
+        
+        avg_trend_path = os.path.join(output_dir, "scaled_averages_trend.png")
+        plt.savefig(avg_trend_path, dpi=150)
+        plt.close()
+        
+        print(f"[INFO] Finished set {i}, results in {output_dir}")
+
+if __name__ == "__main__":
+    main()

statistic/draw.py statistic/draw_compare.pystatistic/draw.py renamed to statistic/draw_compare.py

@@ -0,0 +1,89 @@
+ DTC	minimal.dtb
+Ready to launch Linux kernel. Please be patient.
+failed to allocate TAP device: Operation not permitted
+[    0.000000] Linux version 6.1.99 (jserv@node1) (riscv32-buildroot-linux-gnu-gcc.br_real (Buildroot 2024.02.4) 12.3.0, GNU ld (GNU Binutils) 2.41) #1 SMP Thu Jul 18 13:04:10 CST 2024
+[    0.000000] Machine model: semu
+[    0.000000] earlycon: ns16550 at MMIO 0xf4000000 (options '')
+[    0.000000] printk: bootconsole [ns16550] enabled
+[    0.000000] Zone ranges:
+[    0.000000]   Normal   [mem 0x0000000000000000-0x000000001fffffff]
+[    0.000000] Movable zone start for each node
+[    0.000000] Early memory node ranges
+[    0.000000]   node   0: [mem 0x0000000000000000-0x000000001fffffff]
+[    0.000000] Initmem setup node 0 [mem 0x0000000000000000-0x000000001fffffff]
+[    0.000000] SBI specification v2.0 detected
+[    0.000000] SBI implementation ID=0x999 Version=0x1
+[    0.000000] SBI TIME extension detected
+[    0.000000] SBI IPI extension detected
+[    0.000000] SBI RFENCE extension detected
+[    0.000000] SBI SRST extension detected
+[    0.000000] SBI HSM extension detected
+[    0.000000] riscv: base ISA extensions aim
+[    0.000000] riscv: ELF capabilities aim
+[    0.000000] percpu: Embedded 10 pages/cpu s11604 r8192 d21164 u40960
+[    0.000000] Built 1 zonelists, mobility grouping on.  Total pages: 130048
+[    0.000000] Kernel command line: earlycon console=ttyS0
+[    0.000000] Dentry cache hash table entries: 65536 (order: 6, 262144 bytes, linear)
+[    0.000000] Inode-cache hash table entries: 32768 (order: 5, 131072 bytes, linear)
+[    0.000000] mem auto-init: stack:off, heap alloc:off, heap free:off
+[    0.000000] Memory: 506428K/524288K available (3578K kernel code, 345K rwdata, 873K rodata, 185K init, 140K bss, 17860K reserved, 0K cma-reserved)
+[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1
+[    0.000000] rcu: Hierarchical RCU implementation.
+[    0.000000] rcu: 	RCU restricting CPUs from NR_CPUS=32 to nr_cpu_ids=1.
+[    0.000000] rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.
+[    0.000000] rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=1
+[    0.000000] NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0
+[    0.000000] riscv-intc: 32 local interrupts mapped
+[    0.000000] plic: interrupt-controller@0: mapped 31 interrupts with 1 handlers for 1 contexts.
+[    0.000000] rcu: srcu_init: Setting srcu_struct sizes based on contention.
+[    0.000000] riscv-timer: riscv_timer_init_dt: Registering clocksource cpuid [0] hartid [0]
+[    0.000000] clocksource: riscv_clocksource: mask: 0xffffffffffffffff max_cycles: 0xefdb196da, max_idle_ns: 440795204367 ns
+[    0.000000] sched_clock: 64 bits at 65MHz, resolution 15ns, wraps every 2199023255550ns
+[    0.000010] Console: colour dummy device 80x25
+[    0.000010] Calibrating delay loop (skipped), value calculated using timer frequency.. 130.00 BogoMIPS (lpj=260000)
+[    0.000010] pid_max: default: 32768 minimum: 301
+[    0.000020] Mount-cache hash table entries: 1024 (order: 0, 4096 bytes, linear)
+[    0.000020] Mountpoint-cache hash table entries: 1024 (order: 0, 4096 bytes, linear)
+[    0.000050] ASID allocator disabled (0 bits)
+[    0.000050] rcu: Hierarchical SRCU implementation.
+[    0.000060] rcu: 	Max phase no-delay instances is 1000.
+[    0.000060] smp: Bringing up secondary CPUs ...
+[    0.000060] smp: Brought up 1 node, 1 CPU
+[    0.000070] devtmpfs: initialized
+[    0.000100] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
+[    0.000100] futex hash table entries: 256 (order: 2, 16384 bytes, linear)
+[    0.000130] NET: Registered PF_NETLINK/PF_ROUTE protocol family
+[    0.000160] platform soc@F0000000: Fixed dependency cycle(s) with /soc@F0000000/interrupt-controller@0
+[    0.000351] clocksource: Switched to clocksource riscv_clocksource
+[    0.000591] NET: Registered PF_INET protocol family
+[    0.000601] IP idents hash table entries: 8192 (order: 4, 65536 bytes, linear)
+[    0.000672] tcp_listen_portaddr_hash hash table entries: 512 (order: 0, 4096 bytes, linear)
+[    0.000672] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear)
+[    0.000672] TCP established hash table entries: 4096 (order: 2, 16384 bytes, linear)
+[    0.000682] TCP bind hash table entries: 4096 (order: 4, 65536 bytes, linear)
+[    0.000682] TCP: Hash tables configured (established 4096 bind 4096)
+[    0.000682] UDP hash table entries: 256 (order: 1, 8192 bytes, linear)
+[    0.000692] UDP-Lite hash table entries: 256 (order: 1, 8192 bytes, linear)
+[    0.000692] NET: Registered PF_UNIX/PF_LOCAL protocol family
+[    0.000712] Unpacking initramfs...
+[    0.008775] Freeing initrd memory: 8188K
+[    0.008775] workingset: timestamp_bits=30 max_order=17 bucket_order=0
+[    0.010371] Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled
+[    0.010411] printk: console [ttyS0] disabled
+[    0.010411] f4000000.serial: ttyS0 at MMIO 0xf4000000 (irq = 1, base_baud = 312500) is a 16550
+[    0.010421] printk: console [ttyS0] enabled
+[    0.010421] printk: console [ttyS0] enabled
+[    0.010421] printk: bootconsole [ns16550] disabled
+[    0.010421] printk: bootconsole [ns16550] disabled
+[    0.010431] virtio_blk virtio1: 1/0/0 default/read/poll queues
+[    0.010441] virtio_blk virtio1: [vda] 4800 512-byte logical blocks (2.46 MB/2.34 MiB)
+[    0.010471] virtio_net virtio0: Assigned random MAC address 9a:a5:1b:7f:96:3b
+[    0.010510] clk: Disabling unused clocks
+[    0.010540] Freeing unused kernel image (initmem) memory: 180K
+[    0.010540] Kernel memory protection not selected by kernel config.
+[    0.010540] Run /init as init process
+[SEMU LOG]: Use clock_gettime
+[SEMU LOG]: Real boot time: 12.05580 seconds, called 220579106 times semu_timer_clocksource
+[SEMU LOG]: ns_per_call = 2.77813, predict_sec = 5.55625, scale_factor = 1.79977
+[SEMU LOG]: test_total_clocksource_ns = 2523327095, real_total_clocksource_ns = 1297733725, percentage = 0.10764
+[SEMU LOG]: real_ns_per_call = 5.88330, diff_ns_per_call = 3.10518