diff --git a/automation/script/module.py b/automation/script/module.py index c0eaa1cb3..417aead7a 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2116,10 +2116,10 @@ def _update_state_from_variations( if variation_tags: variation_tags_string = ','.join(['_' + t for t in variation_tags]) - logger.debug( - f"{self.recursion_spaces}Prepared variations: {variation_tags_string}") + logger.debug(self.recursion_spaces + + f" - Prepared variations: {variation_tags_string}") - # 2️⃣ Apply individual variations + # Apply individual variations for variation_tag in variation_tags: r = self._apply_single_variation( variation_tag, variations, @@ -2128,7 +2128,7 @@ def _update_state_from_variations( if r['return'] > 0: return r - # 3️⃣ Apply combined variations + # Apply combined variations r = self._apply_combined_variations( variations, variation_tags, run_state, i, meta, required_disk_space, warnings diff --git a/automation/script/remote_run.py b/automation/script/remote_run.py index a7f0459d1..c4cb735e0 100644 --- a/automation/script/remote_run.py +++ b/automation/script/remote_run.py @@ -111,7 +111,6 @@ def remote_run(self_module, i): r = regenerate_script_cmd(i_copy) if r['return'] > 0: return r - # " ".join(mlc_run_cmd.split(" ")[1:]) script_run_cmd = r['run_cmd_string'] @@ -262,8 +261,8 @@ def rebuild_flags( keys = sorted(command_dict.keys(), key=lambda x: x != "tags") for key in keys: - if key in ["input", "output", "outdirname"]: - continue # We have the corresponding env keys in container env string + # if key in ["input", "output", "outdirname"]: + # continue # We have the corresponding env keys in container env string # Construct the full key with the prefix. full_key = f"{prefix}.{key}" if prefix else key diff --git a/script/detect-cpu/customize.py b/script/detect-cpu/customize.py index 3931ccab5..2e9c5e327 100644 --- a/script/detect-cpu/customize.py +++ b/script/detect-cpu/customize.py @@ -40,7 +40,14 @@ def postprocess(i): else: keys = {} j = 0 - with open(f, 'r') as csvf: + # Try UTF-8 first (PowerShell), fallback to system encoding + # (WMIC) + try: + csvfile = open(f, 'r', encoding='utf-8-sig') + except BaseException: + csvfile = open(f, 'r') + + with csvfile as csvf: for s in csv.reader(csvf): if j == 0: keys = s @@ -71,7 +78,13 @@ def postprocess(i): keys = {} j = 0 - with open(f, 'r', encoding='utf16') as csvf: + # Try UTF-8 first (PowerShell), fallback to UTF-16 (WMIC) + try: + csvfile = open(f, 'r', encoding='utf-8-sig') + except BaseException: + csvfile = open(f, 'r', encoding='utf16') + + with csvfile as csvf: for s in csv.reader(csvf): if j == 1: keys = s @@ -96,13 +109,8 @@ def postprocess(i): state['host_device_raw_info'] = { 'sys': sys, 'sys1': sys1, 'cpu': cpu, 'cpu1': cpu1} - logger.warning( - 'WARNING: need to unify system and cpu output on Windows') - - return {'return': 0} - ########################################################################## - # Linux + # Process lscpu output (both Linux and Windows) if not os.path.isfile(lscpu_out): logger.warning('lscpu.out file was not generated!') @@ -145,17 +153,23 @@ def postprocess(i): 'MLC_CPUINFO_hw_l2cachesize': 'MLC_HOST_CPU_L2_CACHE_SIZE' } - if env['MLC_HOST_OS_TYPE'] == 'linux': + vkeys = [] + if env.get('MLC_HOST_OS_TYPE', + '') == 'linux' or os_info['platform'] == 'windows': vkeys = ['Architecture', 'Model name', 'Vendor ID', 'CPU family', 'NUMA node(s)', 'CPU(s)', 'On-line CPU(s) list', 'Socket(s)', 'Core(s) per socket', 'Core(s) per cluster', 'Thread(s) per core', 'L1d cache', 'L1i cache', 'L2 cache', 'L3 cache', 'CPU max MHz'] - elif env['MLC_HOST_OS_FLAVOR'] == 'macos': + elif env.get('MLC_HOST_OS_FLAVOR', '') == 'macos': vkeys = ['hw.physicalcpu', 'hw.logicalcpu', 'hw.packages', 'hw.ncpu', 'hw.memsize', 'hw.l1icachesize', 'hw.l2cachesize'] if vkeys: for s in ss.split('\n'): - v = s.split(':') - key = v[0] + # Split only on first colon to handle values with colons + v = s.split(':', 1) + if len(v) < 2: + continue + key = v[0].strip() + value = v[1].strip() if key in vkeys: env_key = 'MLC_CPUINFO_' + key.replace( " ", @@ -169,9 +183,9 @@ def postprocess(i): '.', '_') if env_key in unified_env: - env[unified_env[env_key]] = v[1].strip() + env[unified_env[env_key]] = value else: - env[env_key] = v[1].strip() + env[env_key] = value # get start cores matches = re.findall(r"NUMA node\d+ CPU\(s\):\s+([\d,-]+)", ss) diff --git a/script/detect-cpu/detect-cpu.ps1 b/script/detect-cpu/detect-cpu.ps1 new file mode 100644 index 000000000..1744319b9 --- /dev/null +++ b/script/detect-cpu/detect-cpu.ps1 @@ -0,0 +1,91 @@ +$ErrorActionPreference = 'SilentlyContinue' + +try { + # Get system information + $allCpus = Get-CimInstance Win32_Processor + $cpu = $allCpus | Select-Object -First 1 + $cs = Get-CimInstance Win32_ComputerSystem + $os = Get-CimInstance Win32_OperatingSystem + $mem = Get-CimInstance Win32_PhysicalMemory | Measure-Object -Property Capacity -Sum + $disk = Get-PSDrive -PSProvider FileSystem | Where-Object {$_.Free -ne $null} | Measure-Object -Property Used,Free -Sum + + # Calculate CPU topology + $socketCount = $allCpus.Count + $coresPerSocket = $cpu.NumberOfCores + $totalCores = ($allCpus | Measure-Object -Property NumberOfCores -Sum).Sum + $logicalProcessors = $cs.NumberOfLogicalProcessors + $threadsPerCore = if ($totalCores -gt 0) { [math]::Round($logicalProcessors / $totalCores, 0) } else { 1 } + + # Output lscpu-like information + $output = @() + $output += "Architecture: $($cpu.Architecture)" + $output += "CPU op-mode(s): $(if($cpu.AddressWidth -eq 64){'32-bit, 64-bit'}else{'32-bit'})" + $output += "Byte Order: Little Endian" + $output += "CPU(s): $logicalProcessors" + $output += "On-line CPU(s) list: 0-$(($logicalProcessors)-1)" + $output += "Thread(s) per core: $threadsPerCore" + $output += "Core(s) per socket: $coresPerSocket" + $output += "Socket(s): $socketCount" + $output += "Vendor ID: $($cpu.Manufacturer)" + $output += "CPU family: $($cpu.Family)" + $output += "Model: $($cpu.Model)" + $output += "Model name: $($cpu.Name)" + $output += "Stepping: $($cpu.Stepping)" + $output += "CPU MHz: $($cpu.CurrentClockSpeed)" + $output += "CPU max MHz: $($cpu.MaxClockSpeed)" + $output += "Virtualization: $(if($cpu.VirtualizationFirmwareEnabled){'Enabled'}else{'Disabled'})" + if ($cpu.L1CacheSize) { $output += "L1d cache: $($cpu.L1CacheSize) KB" } + if ($cpu.L2CacheSize) { $output += "L2 cache: $($cpu.L2CacheSize) KB" } + if ($cpu.L3CacheSize) { $output += "L3 cache: $($cpu.L3CacheSize) KB" } + $output += "Flags: $($cpu.Description)" + + # Write to tmp-lscpu.out + $output | Out-File -FilePath "tmp-lscpu.out" -Encoding ASCII + + # Export CPU details to CSV with UTF-8 encoding (with BOM for Python compatibility) + $cpu | Export-Csv -Path "tmp-wmic-cpu.csv" -NoTypeInformation -Encoding UTF8 + + # Generate systeminfo CSV for compatibility + $systemInfo = [PSCustomObject]@{ + 'Host Name' = $env:COMPUTERNAME + 'OS Name' = $os.Caption + 'OS Version' = $os.Version + 'OS Manufacturer' = $os.Manufacturer + 'OS Configuration' = $os.OSType + 'OS Build Type' = $os.BuildType + 'System Manufacturer' = $cs.Manufacturer + 'System Model' = $cs.Model + 'System Type' = $cs.SystemType + 'Processor(s)' = "$($allCpus.Count) Processor(s) Installed." + 'Total Physical Memory' = "$memGB GB" + 'Domain' = $cs.Domain + } + $systemInfo | Export-Csv -Path "tmp-systeminfo.csv" -NoTypeInformation -Encoding UTF8 + + # Calculate memory and disk capacity + $memGB = [math]::Round($mem.Sum / 1GB, 2) + $diskUsedGB = [math]::Round(($disk | Where-Object {$_.Property -eq 'Used'}).Sum / 1GB, 2) + $diskFreeGB = [math]::Round(($disk | Where-Object {$_.Property -eq 'Free'}).Sum / 1GB, 2) + $diskTotalGB = $diskUsedGB + $diskFreeGB + + # Write environment variables + $envOutput = @() + $envOutput += "MLC_HOST_MEMORY_CAPACITY=${memGB}G" + $envOutput += "MLC_HOST_DISK_CAPACITY=${diskTotalGB}G" + $envOutput += "MLC_HOST_CPU_MICROCODE=$($cpu.Revision)" + $envOutput += "MLC_HOST_CPU_WRITE_PROTECT_SUPPORT=Not Available on Windows" + $envOutput += "MLC_HOST_CPU_FPU_SUPPORT=$(if($cpu.ProcessorType -ge 3){'yes'}else{'Not Found'})" + $envOutput += "MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT=Not Available on Windows" + $envOutput += "MLC_HOST_CPU_BUGS=Not Available on Windows" + $envOutput += "MLC_HOST_CPU_TLB_SIZE=Not Available on Windows" + $envOutput += "MLC_HOST_CPU_CFLUSH_SIZE=Not Available on Windows" + $envOutput += "MLC_HOST_CACHE_ALIGNMENT_SIZE=$($cpu.DataWidth) bits" + $envOutput += "MLC_HOST_POWER_MANAGEMENT=$(if($cpu.PowerManagementSupported){'Supported'}else{'Not Supported'})" + + $envOutput | Out-File -FilePath "tmp-run-env.out" -Encoding ASCII + + exit 0 +} catch { + Write-Error $_.Exception.Message + exit 1 +} diff --git a/script/detect-cpu/run.bat b/script/detect-cpu/run.bat index 74167fbbe..ec25e42e8 100644 --- a/script/detect-cpu/run.bat +++ b/script/detect-cpu/run.bat @@ -1,9 +1,50 @@ -rem systeminfo /fo csv > tmp-systeminfo.csv @echo off -REM Try PowerShell first -powershell -Command "Get-CimInstance Win32_Processor | Export-Csv -Path tmp-wmic-cpu.csv -NoTypeInformation" 2>nul +setlocal enabledelayedexpansion + +REM Clear output files +if exist tmp-lscpu.out del tmp-lscpu.out +if exist tmp-run-env.out del tmp-run-env.out +if exist tmp-wmic-cpu.csv del tmp-wmic-cpu.csv + +REM Get the directory where this script is located +set SCRIPT_DIR=%~dp0 + +REM Try PowerShell script first for comprehensive CPU information +powershell -NoProfile -ExecutionPolicy Bypass -File "%SCRIPT_DIR%detect-cpu.ps1" 2>nul + if %errorlevel% neq 0 ( - REM Fallback for old Windows with WMIC - wmic cpu get /FORMAT:csv > tmp-wmic-cpu.csv + echo PowerShell script failed, falling back to WMIC... + + REM Fallback to WMIC for older Windows systems + wmic cpu get Name,Manufacturer,NumberOfCores,NumberOfLogicalProcessors,MaxClockSpeed,CurrentClockSpeed,L2CacheSize,L3CacheSize,AddressWidth /format:list > tmp-lscpu.out 2>nul + wmic cpu get /FORMAT:csv > tmp-wmic-cpu.csv 2>nul + + REM Get memory capacity using WMIC + for /f "skip=1 tokens=2 delims=," %%a in ('wmic ComputerSystem get TotalPhysicalMemory /format:csv') do ( + set /a memGB=%%a/1000000000 + echo MLC_HOST_MEMORY_CAPACITY=!memGB!G > tmp-run-env.out + ) + + REM Get disk capacity using WMIC + set totalDisk=0 + for /f "skip=1 tokens=2 delims=," %%a in ('wmic logicaldisk where "DriveType=3" get Size /format:csv') do ( + if not "%%a"=="" ( + set /a totalDisk+=%%a/1000000000 + ) + ) + echo MLC_HOST_DISK_CAPACITY=!totalDisk!G >> tmp-run-env.out + + REM CPU details (limited on Windows) + echo MLC_HOST_CPU_WRITE_PROTECT_SUPPORT=Not Available on Windows >> tmp-run-env.out + echo MLC_HOST_CPU_MICROCODE=Not Available via WMIC >> tmp-run-env.out + echo MLC_HOST_CPU_FPU_SUPPORT=Not Available via WMIC >> tmp-run-env.out + echo MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT=Not Available on Windows >> tmp-run-env.out + echo MLC_HOST_CPU_BUGS=Not Available on Windows >> tmp-run-env.out + echo MLC_HOST_CPU_TLB_SIZE=Not Available on Windows >> tmp-run-env.out + echo MLC_HOST_CPU_CFLUSH_SIZE=Not Available on Windows >> tmp-run-env.out + echo MLC_HOST_CACHE_ALIGNMENT_SIZE=Not Available via WMIC >> tmp-run-env.out + echo MLC_HOST_POWER_MANAGEMENT=Not Available via WMIC >> tmp-run-env.out ) +endlocal + diff --git a/script/detect-os/run.bat b/script/detect-os/run.bat index 89b468ecc..3cb175ee2 100644 --- a/script/detect-os/run.bat +++ b/script/detect-os/run.bat @@ -1 +1,48 @@ -echo {"detect-os-test":"win"} > tmp-run-state.json +@echo off + +REM Get platform architecture +echo %PROCESSOR_ARCHITECTURE% > tmp-run.out + +REM Get system information +systeminfo | findstr /B /C:"OS Name" /C:"OS Version" /C:"System Type" >> tmp-run.out + +REM Detect OS details +for /f "tokens=4-7 delims=[.] " %%i in ('ver') do ( + set WIN_VERSION=%%i.%%j +) + +REM Get OS caption and version from systeminfo output +for /f "tokens=2* delims=:" %%i in ('systeminfo ^| findstr /B /C:"OS Name"') do set "OS_CAPTION=%%j" +for /f "tokens=2* delims=:" %%i in ('systeminfo ^| findstr /B /C:"OS Version"') do set "OS_VERSION=%%j" + +REM Trim leading spaces +if defined OS_CAPTION for /f "tokens=* delims= " %%i in ("%OS_CAPTION%") do set "OS_CAPTION=%%i" +if defined OS_VERSION for /f "tokens=* delims= " %%i in ("%OS_VERSION%") do set "OS_VERSION=%%i" + +REM Set default flavor +set "OS_FLAVOR=windows" + +REM Determine OS flavor +if defined OS_CAPTION ( + echo %OS_CAPTION% | findstr /i "Windows 11" >nul + if not errorlevel 1 set "OS_FLAVOR=windows11" + + echo %OS_CAPTION% | findstr /i "Windows 10" >nul + if not errorlevel 1 set "OS_FLAVOR=windows10" + + echo %OS_CAPTION% | findstr /i "Server 2022" >nul + if not errorlevel 1 set "OS_FLAVOR=windowsserver2022" + + echo %OS_CAPTION% | findstr /i "Server 2019" >nul + if not errorlevel 1 set "OS_FLAVOR=windowsserver2019" + + echo %OS_CAPTION% | findstr /i "Server 2016" >nul + if not errorlevel 1 set "OS_FLAVOR=windowsserver2016" +) + +REM Write environment variables to tmp-run-env.out +echo MLC_HOST_OS_FLAVOR=%OS_FLAVOR% > tmp-run-env.out +echo MLC_HOST_OS_FLAVOR_LIKE=windows >> tmp-run-env.out +echo MLC_HOST_OS_VERSION=%OS_VERSION% >> tmp-run-env.out +echo MLC_HOST_OS_KERNEL_VERSION=%OS_VERSION% >> tmp-run-env.out +echo MLC_HOST_PLATFORM_FLAVOR=%PROCESSOR_ARCHITECTURE% >> tmp-run-env.out diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index c9e4bd2e6..a1d7e1c80 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -46,12 +46,12 @@ def preprocess(i): if 'MLC_AOCC_BIN_WITH_PATH' not in env: if env.get('MLC_AOCC_DIR_PATH', '') != '': aocc_path = env['MLC_AOCC_DIR_PATH'] - if os.path.exists(os.path.join(aocc_path, 'bin', 'clang')): + if os.path.exists(os.path.join(aocc_path, 'bin', exe_c)): env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') else: for l in os.listdir(aocc_path): if os.path.exists(os.path.join( - aocc_path, l, 'bin', 'clang')): + aocc_path, l, 'bin', exe_c)): aocc_path = os.path.join(aocc_path, l) env['MLC_AOCC_DIR_PATH'] = aocc_path env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin')