Fixes for Windows compatibility

automation/script/module.py

@@ -2116,10 +2116,10 @@ def _update_state_from_variations(
         if variation_tags:
             variation_tags_string = ','.join(['_' + t for t in variation_tags])
 
-            logger.debug(
-                f"{self.recursion_spaces}Prepared variations: {variation_tags_string}")
+            logger.debug(self.recursion_spaces +
+                         f" - Prepared variations: {variation_tags_string}")
 
-            # 2️⃣ Apply individual variations
+            # Apply individual variations
             for variation_tag in variation_tags:
                 r = self._apply_single_variation(
                     variation_tag, variations,
@@ -2128,7 +2128,7 @@ def _update_state_from_variations(
                 if r['return'] > 0:
                     return r
 
-            # 3️⃣ Apply combined variations
+            # Apply combined variations
             r = self._apply_combined_variations(
                 variations, variation_tags,
                 run_state, i, meta, required_disk_space, warnings

automation/script/remote_run.py

@@ -111,7 +111,6 @@ def remote_run(self_module, i):
     r = regenerate_script_cmd(i_copy)
     if r['return'] > 0:
         return r
-
     # " ".join(mlc_run_cmd.split(" ")[1:])
     script_run_cmd = r['run_cmd_string']
 
@@ -262,8 +261,8 @@ def rebuild_flags(
         keys = sorted(command_dict.keys(), key=lambda x: x != "tags")
 
         for key in keys:
-            if key in ["input", "output", "outdirname"]:
-                continue  # We have the corresponding env keys in container env string
+            # if key in ["input", "output", "outdirname"]:
+            #    continue  # We have the corresponding env keys in container env string
             # Construct the full key with the prefix.
             full_key = f"{prefix}.{key}" if prefix else key
 

script/detect-cpu/customize.py

@@ -40,7 +40,14 @@ def postprocess(i):
             else:
                 keys = {}
                 j = 0
-                with open(f, 'r') as csvf:
+                # Try UTF-8 first (PowerShell), fallback to system encoding
+                # (WMIC)
+                try:
+                    csvfile = open(f, 'r', encoding='utf-8-sig')
+                except BaseException:
+                    csvfile = open(f, 'r')
+
+                with csvfile as csvf:
                     for s in csv.reader(csvf):
                         if j == 0:
                             keys = s
@@ -71,7 +78,13 @@ def postprocess(i):
                 keys = {}
                 j = 0
 
-                with open(f, 'r', encoding='utf16') as csvf:
+                # Try UTF-8 first (PowerShell), fallback to UTF-16 (WMIC)
+                try:
+                    csvfile = open(f, 'r', encoding='utf-8-sig')
+                except BaseException:
+                    csvfile = open(f, 'r', encoding='utf16')
+
+                with csvfile as csvf:
                     for s in csv.reader(csvf):
                         if j == 1:
                             keys = s
@@ -96,13 +109,8 @@ def postprocess(i):
         state['host_device_raw_info'] = {
             'sys': sys, 'sys1': sys1, 'cpu': cpu, 'cpu1': cpu1}
 
-        logger.warning(
-            'WARNING: need to unify system and cpu output on Windows')
-
-        return {'return': 0}
-
     ##########################################################################
-    # Linux
+    # Process lscpu output (both Linux and Windows)
     if not os.path.isfile(lscpu_out):
         logger.warning('lscpu.out file was not generated!')
 
@@ -145,17 +153,23 @@ def postprocess(i):
         'MLC_CPUINFO_hw_l2cachesize': 'MLC_HOST_CPU_L2_CACHE_SIZE'
     }
 
-    if env['MLC_HOST_OS_TYPE'] == 'linux':
+    vkeys = []
+    if env.get('MLC_HOST_OS_TYPE',
+               '') == 'linux' or os_info['platform'] == 'windows':
         vkeys = ['Architecture', 'Model name', 'Vendor ID', 'CPU family', 'NUMA node(s)', 'CPU(s)',
                  'On-line CPU(s) list', 'Socket(s)', 'Core(s) per socket', 'Core(s) per cluster', 'Thread(s) per core', 'L1d cache', 'L1i cache', 'L2 cache',
                  'L3 cache', 'CPU max MHz']
-    elif env['MLC_HOST_OS_FLAVOR'] == 'macos':
+    elif env.get('MLC_HOST_OS_FLAVOR', '') == 'macos':
         vkeys = ['hw.physicalcpu', 'hw.logicalcpu', 'hw.packages', 'hw.ncpu', 'hw.memsize', 'hw.l1icachesize',
                  'hw.l2cachesize']
     if vkeys:
         for s in ss.split('\n'):
-            v = s.split(':')
-            key = v[0]
+            # Split only on first colon to handle values with colons
+            v = s.split(':', 1)
+            if len(v) < 2:
+                continue
+            key = v[0].strip()
+            value = v[1].strip()
             if key in vkeys:
                 env_key = 'MLC_CPUINFO_' + key.replace(
                     " ",
@@ -169,9 +183,9 @@ def postprocess(i):
                     '.',
                     '_')
                 if env_key in unified_env:
-                    env[unified_env[env_key]] = v[1].strip()
+                    env[unified_env[env_key]] = value
                 else:
-                    env[env_key] = v[1].strip()
+                    env[env_key] = value
 
     # get start cores
     matches = re.findall(r"NUMA node\d+ CPU\(s\):\s+([\d,-]+)", ss)

script/detect-cpu/detect-cpu.ps1

@@ -0,0 +1,91 @@
+$ErrorActionPreference = 'SilentlyContinue'
+
+try {
+    # Get system information
+    $allCpus = Get-CimInstance Win32_Processor
+    $cpu = $allCpus | Select-Object -First 1
+    $cs = Get-CimInstance Win32_ComputerSystem
+    $os = Get-CimInstance Win32_OperatingSystem
+    $mem = Get-CimInstance Win32_PhysicalMemory | Measure-Object -Property Capacity -Sum
+    $disk = Get-PSDrive -PSProvider FileSystem | Where-Object {$_.Free -ne $null} | Measure-Object -Property Used,Free -Sum
+
+    # Calculate CPU topology
+    $socketCount = $allCpus.Count
+    $coresPerSocket = $cpu.NumberOfCores
+    $totalCores = ($allCpus | Measure-Object -Property NumberOfCores -Sum).Sum
+    $logicalProcessors = $cs.NumberOfLogicalProcessors
+    $threadsPerCore = if ($totalCores -gt 0) { [math]::Round($logicalProcessors / $totalCores, 0) } else { 1 }
+
+    # Output lscpu-like information
+    $output = @()
+    $output += "Architecture: $($cpu.Architecture)"
+    $output += "CPU op-mode(s): $(if($cpu.AddressWidth -eq 64){'32-bit, 64-bit'}else{'32-bit'})"
+    $output += "Byte Order: Little Endian"
+    $output += "CPU(s): $logicalProcessors"
+    $output += "On-line CPU(s) list: 0-$(($logicalProcessors)-1)"
+    $output += "Thread(s) per core: $threadsPerCore"
+    $output += "Core(s) per socket: $coresPerSocket"
+    $output += "Socket(s): $socketCount"
+    $output += "Vendor ID: $($cpu.Manufacturer)"
+    $output += "CPU family: $($cpu.Family)"
+    $output += "Model: $($cpu.Model)"
+    $output += "Model name: $($cpu.Name)"
+    $output += "Stepping: $($cpu.Stepping)"
+    $output += "CPU MHz: $($cpu.CurrentClockSpeed)"
+    $output += "CPU max MHz: $($cpu.MaxClockSpeed)"
+    $output += "Virtualization: $(if($cpu.VirtualizationFirmwareEnabled){'Enabled'}else{'Disabled'})"
+    if ($cpu.L1CacheSize) { $output += "L1d cache: $($cpu.L1CacheSize) KB" }
+    if ($cpu.L2CacheSize) { $output += "L2 cache: $($cpu.L2CacheSize) KB" }
+    if ($cpu.L3CacheSize) { $output += "L3 cache: $($cpu.L3CacheSize) KB" }
+    $output += "Flags: $($cpu.Description)"
+
+    # Write to tmp-lscpu.out
+    $output | Out-File -FilePath "tmp-lscpu.out" -Encoding ASCII
+
+    # Export CPU details to CSV with UTF-8 encoding (with BOM for Python compatibility)
+    $cpu | Export-Csv -Path "tmp-wmic-cpu.csv" -NoTypeInformation -Encoding UTF8
+
+    # Generate systeminfo CSV for compatibility
+    $systemInfo = [PSCustomObject]@{
+        'Host Name' = $env:COMPUTERNAME
+        'OS Name' = $os.Caption
+        'OS Version' = $os.Version
+        'OS Manufacturer' = $os.Manufacturer
+        'OS Configuration' = $os.OSType
+        'OS Build Type' = $os.BuildType
+        'System Manufacturer' = $cs.Manufacturer
+        'System Model' = $cs.Model
+        'System Type' = $cs.SystemType
+        'Processor(s)' = "$($allCpus.Count) Processor(s) Installed."
+        'Total Physical Memory' = "$memGB GB"
+        'Domain' = $cs.Domain
+    }
+    $systemInfo | Export-Csv -Path "tmp-systeminfo.csv" -NoTypeInformation -Encoding UTF8
+
+    # Calculate memory and disk capacity
+    $memGB = [math]::Round($mem.Sum / 1GB, 2)
+    $diskUsedGB = [math]::Round(($disk | Where-Object {$_.Property -eq 'Used'}).Sum / 1GB, 2)
+    $diskFreeGB = [math]::Round(($disk | Where-Object {$_.Property -eq 'Free'}).Sum / 1GB, 2)
+    $diskTotalGB = $diskUsedGB + $diskFreeGB
+
+    # Write environment variables
+    $envOutput = @()
+    $envOutput += "MLC_HOST_MEMORY_CAPACITY=${memGB}G"
+    $envOutput += "MLC_HOST_DISK_CAPACITY=${diskTotalGB}G"
+    $envOutput += "MLC_HOST_CPU_MICROCODE=$($cpu.Revision)"
+    $envOutput += "MLC_HOST_CPU_WRITE_PROTECT_SUPPORT=Not Available on Windows"
+    $envOutput += "MLC_HOST_CPU_FPU_SUPPORT=$(if($cpu.ProcessorType -ge 3){'yes'}else{'Not Found'})"
+    $envOutput += "MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT=Not Available on Windows"
+    $envOutput += "MLC_HOST_CPU_BUGS=Not Available on Windows"
+    $envOutput += "MLC_HOST_CPU_TLB_SIZE=Not Available on Windows"
+    $envOutput += "MLC_HOST_CPU_CFLUSH_SIZE=Not Available on Windows"
+    $envOutput += "MLC_HOST_CACHE_ALIGNMENT_SIZE=$($cpu.DataWidth) bits"
+    $envOutput += "MLC_HOST_POWER_MANAGEMENT=$(if($cpu.PowerManagementSupported){'Supported'}else{'Not Supported'})"
+
+    $envOutput | Out-File -FilePath "tmp-run-env.out" -Encoding ASCII
+
+    exit 0
+} catch {
+    Write-Error $_.Exception.Message
+    exit 1
+}

script/detect-cpu/run.bat

@@ -1,9 +1,50 @@
-rem systeminfo /fo csv > tmp-systeminfo.csv
 @echo off
-REM Try PowerShell first
-powershell -Command "Get-CimInstance Win32_Processor | Export-Csv -Path tmp-wmic-cpu.csv -NoTypeInformation" 2>nul
+setlocal enabledelayedexpansion
+
+REM Clear output files
+if exist tmp-lscpu.out del tmp-lscpu.out
+if exist tmp-run-env.out del tmp-run-env.out
+if exist tmp-wmic-cpu.csv del tmp-wmic-cpu.csv
+
+REM Get the directory where this script is located
+set SCRIPT_DIR=%~dp0
+
+REM Try PowerShell script first for comprehensive CPU information
+powershell -NoProfile -ExecutionPolicy Bypass -File "%SCRIPT_DIR%detect-cpu.ps1" 2>nul
+
 if %errorlevel% neq 0 (
-    REM Fallback for old Windows with WMIC
-    wmic cpu get /FORMAT:csv > tmp-wmic-cpu.csv
+    echo PowerShell script failed, falling back to WMIC...
+
+    REM Fallback to WMIC for older Windows systems
+    wmic cpu get Name,Manufacturer,NumberOfCores,NumberOfLogicalProcessors,MaxClockSpeed,CurrentClockSpeed,L2CacheSize,L3CacheSize,AddressWidth /format:list > tmp-lscpu.out 2>nul
+    wmic cpu get /FORMAT:csv > tmp-wmic-cpu.csv 2>nul
+
+    REM Get memory capacity using WMIC
+    for /f "skip=1 tokens=2 delims=," %%a in ('wmic ComputerSystem get TotalPhysicalMemory /format:csv') do (
+        set /a memGB=%%a/1000000000
+        echo MLC_HOST_MEMORY_CAPACITY=!memGB!G > tmp-run-env.out
+    )
+
+    REM Get disk capacity using WMIC
+    set totalDisk=0
+    for /f "skip=1 tokens=2 delims=," %%a in ('wmic logicaldisk where "DriveType=3" get Size /format:csv') do (
+        if not "%%a"=="" (
+            set /a totalDisk+=%%a/1000000000
+        )
+    )
+    echo MLC_HOST_DISK_CAPACITY=!totalDisk!G >> tmp-run-env.out
+
+    REM CPU details (limited on Windows)
+    echo MLC_HOST_CPU_WRITE_PROTECT_SUPPORT=Not Available on Windows >> tmp-run-env.out
+    echo MLC_HOST_CPU_MICROCODE=Not Available via WMIC >> tmp-run-env.out
+    echo MLC_HOST_CPU_FPU_SUPPORT=Not Available via WMIC >> tmp-run-env.out
+    echo MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT=Not Available on Windows >> tmp-run-env.out
+    echo MLC_HOST_CPU_BUGS=Not Available on Windows >> tmp-run-env.out
+    echo MLC_HOST_CPU_TLB_SIZE=Not Available on Windows >> tmp-run-env.out
+    echo MLC_HOST_CPU_CFLUSH_SIZE=Not Available on Windows >> tmp-run-env.out
+    echo MLC_HOST_CACHE_ALIGNMENT_SIZE=Not Available via WMIC >> tmp-run-env.out
+    echo MLC_HOST_POWER_MANAGEMENT=Not Available via WMIC >> tmp-run-env.out
 )
 
+endlocal
+

script/detect-os/run.bat

@@ -1 +1,48 @@
-echo {"detect-os-test":"win"} > tmp-run-state.json
+@echo off
+
+REM Get platform architecture
+echo %PROCESSOR_ARCHITECTURE% > tmp-run.out
+
+REM Get system information
+systeminfo | findstr /B /C:"OS Name" /C:"OS Version" /C:"System Type" >> tmp-run.out
+
+REM Detect OS details
+for /f "tokens=4-7 delims=[.] " %%i in ('ver') do (
+    set WIN_VERSION=%%i.%%j
+)
+
+REM Get OS caption and version from systeminfo output
+for /f "tokens=2* delims=:" %%i in ('systeminfo ^| findstr /B /C:"OS Name"') do set "OS_CAPTION=%%j"
+for /f "tokens=2* delims=:" %%i in ('systeminfo ^| findstr /B /C:"OS Version"') do set "OS_VERSION=%%j"
+
+REM Trim leading spaces
+if defined OS_CAPTION for /f "tokens=* delims= " %%i in ("%OS_CAPTION%") do set "OS_CAPTION=%%i"
+if defined OS_VERSION for /f "tokens=* delims= " %%i in ("%OS_VERSION%") do set "OS_VERSION=%%i"
+
+REM Set default flavor
+set "OS_FLAVOR=windows"
+
+REM Determine OS flavor
+if defined OS_CAPTION (
+    echo %OS_CAPTION% | findstr /i "Windows 11" >nul
+    if not errorlevel 1 set "OS_FLAVOR=windows11"
+
+    echo %OS_CAPTION% | findstr /i "Windows 10" >nul
+    if not errorlevel 1 set "OS_FLAVOR=windows10"
+
+    echo %OS_CAPTION% | findstr /i "Server 2022" >nul
+    if not errorlevel 1 set "OS_FLAVOR=windowsserver2022"
+
+    echo %OS_CAPTION% | findstr /i "Server 2019" >nul
+    if not errorlevel 1 set "OS_FLAVOR=windowsserver2019"
+
+    echo %OS_CAPTION% | findstr /i "Server 2016" >nul
+    if not errorlevel 1 set "OS_FLAVOR=windowsserver2016"
+)
+
+REM Write environment variables to tmp-run-env.out
+echo MLC_HOST_OS_FLAVOR=%OS_FLAVOR% > tmp-run-env.out
+echo MLC_HOST_OS_FLAVOR_LIKE=windows >> tmp-run-env.out
+echo MLC_HOST_OS_VERSION=%OS_VERSION% >> tmp-run-env.out
+echo MLC_HOST_OS_KERNEL_VERSION=%OS_VERSION% >> tmp-run-env.out
+echo MLC_HOST_PLATFORM_FLAVOR=%PROCESSOR_ARCHITECTURE% >> tmp-run-env.out

script/get-aocc/customize.py

@@ -46,12 +46,12 @@ def preprocess(i):
     if 'MLC_AOCC_BIN_WITH_PATH' not in env:
         if env.get('MLC_AOCC_DIR_PATH', '') != '':
             aocc_path = env['MLC_AOCC_DIR_PATH']
-            if os.path.exists(os.path.join(aocc_path, 'bin', 'clang')):
+            if os.path.exists(os.path.join(aocc_path, 'bin', exe_c)):
                 env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin')
             else:
                 for l in os.listdir(aocc_path):
                     if os.path.exists(os.path.join(
-                            aocc_path, l, 'bin', 'clang')):
+                            aocc_path, l, 'bin', exe_c)):
                         aocc_path = os.path.join(aocc_path, l)
                         env['MLC_AOCC_DIR_PATH'] = aocc_path
                         env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin')