Skip to content

Commit 61c070f

Browse files
authored
Fixes for Windows compatibility (#801)
* Fix --outdirname for mlcrr * Fix get-aocc for windows * Handle detect-cpu better on Windows * Improve run.bat for detect-os * Fix a bug in debug output
1 parent 73499b3 commit 61c070f

File tree

7 files changed

+221
-29
lines changed

7 files changed

+221
-29
lines changed

automation/script/module.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2116,10 +2116,10 @@ def _update_state_from_variations(
21162116
if variation_tags:
21172117
variation_tags_string = ','.join(['_' + t for t in variation_tags])
21182118

2119-
logger.debug(
2120-
f"{self.recursion_spaces}Prepared variations: {variation_tags_string}")
2119+
logger.debug(self.recursion_spaces +
2120+
f" - Prepared variations: {variation_tags_string}")
21212121

2122-
# 2️⃣ Apply individual variations
2122+
# Apply individual variations
21232123
for variation_tag in variation_tags:
21242124
r = self._apply_single_variation(
21252125
variation_tag, variations,
@@ -2128,7 +2128,7 @@ def _update_state_from_variations(
21282128
if r['return'] > 0:
21292129
return r
21302130

2131-
# 3️⃣ Apply combined variations
2131+
# Apply combined variations
21322132
r = self._apply_combined_variations(
21332133
variations, variation_tags,
21342134
run_state, i, meta, required_disk_space, warnings

automation/script/remote_run.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,6 @@ def remote_run(self_module, i):
111111
r = regenerate_script_cmd(i_copy)
112112
if r['return'] > 0:
113113
return r
114-
115114
# " ".join(mlc_run_cmd.split(" ")[1:])
116115
script_run_cmd = r['run_cmd_string']
117116

@@ -262,8 +261,8 @@ def rebuild_flags(
262261
keys = sorted(command_dict.keys(), key=lambda x: x != "tags")
263262

264263
for key in keys:
265-
if key in ["input", "output", "outdirname"]:
266-
continue # We have the corresponding env keys in container env string
264+
# if key in ["input", "output", "outdirname"]:
265+
# continue # We have the corresponding env keys in container env string
267266
# Construct the full key with the prefix.
268267
full_key = f"{prefix}.{key}" if prefix else key
269268

script/detect-cpu/customize.py

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,14 @@ def postprocess(i):
4040
else:
4141
keys = {}
4242
j = 0
43-
with open(f, 'r') as csvf:
43+
# Try UTF-8 first (PowerShell), fallback to system encoding
44+
# (WMIC)
45+
try:
46+
csvfile = open(f, 'r', encoding='utf-8-sig')
47+
except BaseException:
48+
csvfile = open(f, 'r')
49+
50+
with csvfile as csvf:
4451
for s in csv.reader(csvf):
4552
if j == 0:
4653
keys = s
@@ -71,7 +78,13 @@ def postprocess(i):
7178
keys = {}
7279
j = 0
7380

74-
with open(f, 'r', encoding='utf16') as csvf:
81+
# Try UTF-8 first (PowerShell), fallback to UTF-16 (WMIC)
82+
try:
83+
csvfile = open(f, 'r', encoding='utf-8-sig')
84+
except BaseException:
85+
csvfile = open(f, 'r', encoding='utf16')
86+
87+
with csvfile as csvf:
7588
for s in csv.reader(csvf):
7689
if j == 1:
7790
keys = s
@@ -96,13 +109,8 @@ def postprocess(i):
96109
state['host_device_raw_info'] = {
97110
'sys': sys, 'sys1': sys1, 'cpu': cpu, 'cpu1': cpu1}
98111

99-
logger.warning(
100-
'WARNING: need to unify system and cpu output on Windows')
101-
102-
return {'return': 0}
103-
104112
##########################################################################
105-
# Linux
113+
# Process lscpu output (both Linux and Windows)
106114
if not os.path.isfile(lscpu_out):
107115
logger.warning('lscpu.out file was not generated!')
108116

@@ -145,17 +153,23 @@ def postprocess(i):
145153
'MLC_CPUINFO_hw_l2cachesize': 'MLC_HOST_CPU_L2_CACHE_SIZE'
146154
}
147155

148-
if env['MLC_HOST_OS_TYPE'] == 'linux':
156+
vkeys = []
157+
if env.get('MLC_HOST_OS_TYPE',
158+
'') == 'linux' or os_info['platform'] == 'windows':
149159
vkeys = ['Architecture', 'Model name', 'Vendor ID', 'CPU family', 'NUMA node(s)', 'CPU(s)',
150160
'On-line CPU(s) list', 'Socket(s)', 'Core(s) per socket', 'Core(s) per cluster', 'Thread(s) per core', 'L1d cache', 'L1i cache', 'L2 cache',
151161
'L3 cache', 'CPU max MHz']
152-
elif env['MLC_HOST_OS_FLAVOR'] == 'macos':
162+
elif env.get('MLC_HOST_OS_FLAVOR', '') == 'macos':
153163
vkeys = ['hw.physicalcpu', 'hw.logicalcpu', 'hw.packages', 'hw.ncpu', 'hw.memsize', 'hw.l1icachesize',
154164
'hw.l2cachesize']
155165
if vkeys:
156166
for s in ss.split('\n'):
157-
v = s.split(':')
158-
key = v[0]
167+
# Split only on first colon to handle values with colons
168+
v = s.split(':', 1)
169+
if len(v) < 2:
170+
continue
171+
key = v[0].strip()
172+
value = v[1].strip()
159173
if key in vkeys:
160174
env_key = 'MLC_CPUINFO_' + key.replace(
161175
" ",
@@ -169,9 +183,9 @@ def postprocess(i):
169183
'.',
170184
'_')
171185
if env_key in unified_env:
172-
env[unified_env[env_key]] = v[1].strip()
186+
env[unified_env[env_key]] = value
173187
else:
174-
env[env_key] = v[1].strip()
188+
env[env_key] = value
175189

176190
# get start cores
177191
matches = re.findall(r"NUMA node\d+ CPU\(s\):\s+([\d,-]+)", ss)

script/detect-cpu/detect-cpu.ps1

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
$ErrorActionPreference = 'SilentlyContinue'
2+
3+
try {
4+
# Get system information
5+
$allCpus = Get-CimInstance Win32_Processor
6+
$cpu = $allCpus | Select-Object -First 1
7+
$cs = Get-CimInstance Win32_ComputerSystem
8+
$os = Get-CimInstance Win32_OperatingSystem
9+
$mem = Get-CimInstance Win32_PhysicalMemory | Measure-Object -Property Capacity -Sum
10+
$disk = Get-PSDrive -PSProvider FileSystem | Where-Object {$_.Free -ne $null} | Measure-Object -Property Used,Free -Sum
11+
12+
# Calculate CPU topology
13+
$socketCount = $allCpus.Count
14+
$coresPerSocket = $cpu.NumberOfCores
15+
$totalCores = ($allCpus | Measure-Object -Property NumberOfCores -Sum).Sum
16+
$logicalProcessors = $cs.NumberOfLogicalProcessors
17+
$threadsPerCore = if ($totalCores -gt 0) { [math]::Round($logicalProcessors / $totalCores, 0) } else { 1 }
18+
19+
# Output lscpu-like information
20+
$output = @()
21+
$output += "Architecture: $($cpu.Architecture)"
22+
$output += "CPU op-mode(s): $(if($cpu.AddressWidth -eq 64){'32-bit, 64-bit'}else{'32-bit'})"
23+
$output += "Byte Order: Little Endian"
24+
$output += "CPU(s): $logicalProcessors"
25+
$output += "On-line CPU(s) list: 0-$(($logicalProcessors)-1)"
26+
$output += "Thread(s) per core: $threadsPerCore"
27+
$output += "Core(s) per socket: $coresPerSocket"
28+
$output += "Socket(s): $socketCount"
29+
$output += "Vendor ID: $($cpu.Manufacturer)"
30+
$output += "CPU family: $($cpu.Family)"
31+
$output += "Model: $($cpu.Model)"
32+
$output += "Model name: $($cpu.Name)"
33+
$output += "Stepping: $($cpu.Stepping)"
34+
$output += "CPU MHz: $($cpu.CurrentClockSpeed)"
35+
$output += "CPU max MHz: $($cpu.MaxClockSpeed)"
36+
$output += "Virtualization: $(if($cpu.VirtualizationFirmwareEnabled){'Enabled'}else{'Disabled'})"
37+
if ($cpu.L1CacheSize) { $output += "L1d cache: $($cpu.L1CacheSize) KB" }
38+
if ($cpu.L2CacheSize) { $output += "L2 cache: $($cpu.L2CacheSize) KB" }
39+
if ($cpu.L3CacheSize) { $output += "L3 cache: $($cpu.L3CacheSize) KB" }
40+
$output += "Flags: $($cpu.Description)"
41+
42+
# Write to tmp-lscpu.out
43+
$output | Out-File -FilePath "tmp-lscpu.out" -Encoding ASCII
44+
45+
# Export CPU details to CSV with UTF-8 encoding (with BOM for Python compatibility)
46+
$cpu | Export-Csv -Path "tmp-wmic-cpu.csv" -NoTypeInformation -Encoding UTF8
47+
48+
# Generate systeminfo CSV for compatibility
49+
$systemInfo = [PSCustomObject]@{
50+
'Host Name' = $env:COMPUTERNAME
51+
'OS Name' = $os.Caption
52+
'OS Version' = $os.Version
53+
'OS Manufacturer' = $os.Manufacturer
54+
'OS Configuration' = $os.OSType
55+
'OS Build Type' = $os.BuildType
56+
'System Manufacturer' = $cs.Manufacturer
57+
'System Model' = $cs.Model
58+
'System Type' = $cs.SystemType
59+
'Processor(s)' = "$($allCpus.Count) Processor(s) Installed."
60+
'Total Physical Memory' = "$memGB GB"
61+
'Domain' = $cs.Domain
62+
}
63+
$systemInfo | Export-Csv -Path "tmp-systeminfo.csv" -NoTypeInformation -Encoding UTF8
64+
65+
# Calculate memory and disk capacity
66+
$memGB = [math]::Round($mem.Sum / 1GB, 2)
67+
$diskUsedGB = [math]::Round(($disk | Where-Object {$_.Property -eq 'Used'}).Sum / 1GB, 2)
68+
$diskFreeGB = [math]::Round(($disk | Where-Object {$_.Property -eq 'Free'}).Sum / 1GB, 2)
69+
$diskTotalGB = $diskUsedGB + $diskFreeGB
70+
71+
# Write environment variables
72+
$envOutput = @()
73+
$envOutput += "MLC_HOST_MEMORY_CAPACITY=${memGB}G"
74+
$envOutput += "MLC_HOST_DISK_CAPACITY=${diskTotalGB}G"
75+
$envOutput += "MLC_HOST_CPU_MICROCODE=$($cpu.Revision)"
76+
$envOutput += "MLC_HOST_CPU_WRITE_PROTECT_SUPPORT=Not Available on Windows"
77+
$envOutput += "MLC_HOST_CPU_FPU_SUPPORT=$(if($cpu.ProcessorType -ge 3){'yes'}else{'Not Found'})"
78+
$envOutput += "MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT=Not Available on Windows"
79+
$envOutput += "MLC_HOST_CPU_BUGS=Not Available on Windows"
80+
$envOutput += "MLC_HOST_CPU_TLB_SIZE=Not Available on Windows"
81+
$envOutput += "MLC_HOST_CPU_CFLUSH_SIZE=Not Available on Windows"
82+
$envOutput += "MLC_HOST_CACHE_ALIGNMENT_SIZE=$($cpu.DataWidth) bits"
83+
$envOutput += "MLC_HOST_POWER_MANAGEMENT=$(if($cpu.PowerManagementSupported){'Supported'}else{'Not Supported'})"
84+
85+
$envOutput | Out-File -FilePath "tmp-run-env.out" -Encoding ASCII
86+
87+
exit 0
88+
} catch {
89+
Write-Error $_.Exception.Message
90+
exit 1
91+
}

script/detect-cpu/run.bat

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,50 @@
1-
rem systeminfo /fo csv > tmp-systeminfo.csv
21
@echo off
3-
REM Try PowerShell first
4-
powershell -Command "Get-CimInstance Win32_Processor | Export-Csv -Path tmp-wmic-cpu.csv -NoTypeInformation" 2>nul
2+
setlocal enabledelayedexpansion
3+
4+
REM Clear output files
5+
if exist tmp-lscpu.out del tmp-lscpu.out
6+
if exist tmp-run-env.out del tmp-run-env.out
7+
if exist tmp-wmic-cpu.csv del tmp-wmic-cpu.csv
8+
9+
REM Get the directory where this script is located
10+
set SCRIPT_DIR=%~dp0
11+
12+
REM Try PowerShell script first for comprehensive CPU information
13+
powershell -NoProfile -ExecutionPolicy Bypass -File "%SCRIPT_DIR%detect-cpu.ps1" 2>nul
14+
515
if %errorlevel% neq 0 (
6-
REM Fallback for old Windows with WMIC
7-
wmic cpu get /FORMAT:csv > tmp-wmic-cpu.csv
16+
echo PowerShell script failed, falling back to WMIC...
17+
18+
REM Fallback to WMIC for older Windows systems
19+
wmic cpu get Name,Manufacturer,NumberOfCores,NumberOfLogicalProcessors,MaxClockSpeed,CurrentClockSpeed,L2CacheSize,L3CacheSize,AddressWidth /format:list > tmp-lscpu.out 2>nul
20+
wmic cpu get /FORMAT:csv > tmp-wmic-cpu.csv 2>nul
21+
22+
REM Get memory capacity using WMIC
23+
for /f "skip=1 tokens=2 delims=," %%a in ('wmic ComputerSystem get TotalPhysicalMemory /format:csv') do (
24+
set /a memGB=%%a/1000000000
25+
echo MLC_HOST_MEMORY_CAPACITY=!memGB!G > tmp-run-env.out
26+
)
27+
28+
REM Get disk capacity using WMIC
29+
set totalDisk=0
30+
for /f "skip=1 tokens=2 delims=," %%a in ('wmic logicaldisk where "DriveType=3" get Size /format:csv') do (
31+
if not "%%a"=="" (
32+
set /a totalDisk+=%%a/1000000000
33+
)
34+
)
35+
echo MLC_HOST_DISK_CAPACITY=!totalDisk!G >> tmp-run-env.out
36+
37+
REM CPU details (limited on Windows)
38+
echo MLC_HOST_CPU_WRITE_PROTECT_SUPPORT=Not Available on Windows >> tmp-run-env.out
39+
echo MLC_HOST_CPU_MICROCODE=Not Available via WMIC >> tmp-run-env.out
40+
echo MLC_HOST_CPU_FPU_SUPPORT=Not Available via WMIC >> tmp-run-env.out
41+
echo MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT=Not Available on Windows >> tmp-run-env.out
42+
echo MLC_HOST_CPU_BUGS=Not Available on Windows >> tmp-run-env.out
43+
echo MLC_HOST_CPU_TLB_SIZE=Not Available on Windows >> tmp-run-env.out
44+
echo MLC_HOST_CPU_CFLUSH_SIZE=Not Available on Windows >> tmp-run-env.out
45+
echo MLC_HOST_CACHE_ALIGNMENT_SIZE=Not Available via WMIC >> tmp-run-env.out
46+
echo MLC_HOST_POWER_MANAGEMENT=Not Available via WMIC >> tmp-run-env.out
847
)
948

49+
endlocal
50+

script/detect-os/run.bat

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,48 @@
1-
echo {"detect-os-test":"win"} > tmp-run-state.json
1+
@echo off
2+
3+
REM Get platform architecture
4+
echo %PROCESSOR_ARCHITECTURE% > tmp-run.out
5+
6+
REM Get system information
7+
systeminfo | findstr /B /C:"OS Name" /C:"OS Version" /C:"System Type" >> tmp-run.out
8+
9+
REM Detect OS details
10+
for /f "tokens=4-7 delims=[.] " %%i in ('ver') do (
11+
set WIN_VERSION=%%i.%%j
12+
)
13+
14+
REM Get OS caption and version from systeminfo output
15+
for /f "tokens=2* delims=:" %%i in ('systeminfo ^| findstr /B /C:"OS Name"') do set "OS_CAPTION=%%j"
16+
for /f "tokens=2* delims=:" %%i in ('systeminfo ^| findstr /B /C:"OS Version"') do set "OS_VERSION=%%j"
17+
18+
REM Trim leading spaces
19+
if defined OS_CAPTION for /f "tokens=* delims= " %%i in ("%OS_CAPTION%") do set "OS_CAPTION=%%i"
20+
if defined OS_VERSION for /f "tokens=* delims= " %%i in ("%OS_VERSION%") do set "OS_VERSION=%%i"
21+
22+
REM Set default flavor
23+
set "OS_FLAVOR=windows"
24+
25+
REM Determine OS flavor
26+
if defined OS_CAPTION (
27+
echo %OS_CAPTION% | findstr /i "Windows 11" >nul
28+
if not errorlevel 1 set "OS_FLAVOR=windows11"
29+
30+
echo %OS_CAPTION% | findstr /i "Windows 10" >nul
31+
if not errorlevel 1 set "OS_FLAVOR=windows10"
32+
33+
echo %OS_CAPTION% | findstr /i "Server 2022" >nul
34+
if not errorlevel 1 set "OS_FLAVOR=windowsserver2022"
35+
36+
echo %OS_CAPTION% | findstr /i "Server 2019" >nul
37+
if not errorlevel 1 set "OS_FLAVOR=windowsserver2019"
38+
39+
echo %OS_CAPTION% | findstr /i "Server 2016" >nul
40+
if not errorlevel 1 set "OS_FLAVOR=windowsserver2016"
41+
)
42+
43+
REM Write environment variables to tmp-run-env.out
44+
echo MLC_HOST_OS_FLAVOR=%OS_FLAVOR% > tmp-run-env.out
45+
echo MLC_HOST_OS_FLAVOR_LIKE=windows >> tmp-run-env.out
46+
echo MLC_HOST_OS_VERSION=%OS_VERSION% >> tmp-run-env.out
47+
echo MLC_HOST_OS_KERNEL_VERSION=%OS_VERSION% >> tmp-run-env.out
48+
echo MLC_HOST_PLATFORM_FLAVOR=%PROCESSOR_ARCHITECTURE% >> tmp-run-env.out

script/get-aocc/customize.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,12 @@ def preprocess(i):
4646
if 'MLC_AOCC_BIN_WITH_PATH' not in env:
4747
if env.get('MLC_AOCC_DIR_PATH', '') != '':
4848
aocc_path = env['MLC_AOCC_DIR_PATH']
49-
if os.path.exists(os.path.join(aocc_path, 'bin', 'clang')):
49+
if os.path.exists(os.path.join(aocc_path, 'bin', exe_c)):
5050
env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin')
5151
else:
5252
for l in os.listdir(aocc_path):
5353
if os.path.exists(os.path.join(
54-
aocc_path, l, 'bin', 'clang')):
54+
aocc_path, l, 'bin', exe_c)):
5555
aocc_path = os.path.join(aocc_path, l)
5656
env['MLC_AOCC_DIR_PATH'] = aocc_path
5757
env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin')

0 commit comments

Comments
 (0)