Skip to content

Commit a207b45

Browse files
CapostrophicCapostrophic
committed
Merge branch 'splitting-headache' into 'master'
Split the Windows build into two, like it was when the time limit was an hour before mid 2022 See merge request OpenMW/openmw!5070
2 parents 9efef81 + 2554309 commit a207b45

File tree

3 files changed

+515
-381
lines changed

3 files changed

+515
-381
lines changed

‎.gitlab-ci.yml‎

Lines changed: 131 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -625,7 +625,7 @@ macOS15_Xcode16_arm64:
625625
- ./aws/install
626626
- popd
627627
- aws --version
628-
- unzip -d sym_store *sym_store.zip
628+
- unzip -n -d sym_store '*sym_store.zip'
629629
- shopt -s globstar
630630
- |
631631
for file in sym_store/**/*.exe; do
@@ -650,13 +650,68 @@ macOS15_Xcode16_arm64:
650650
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp --recursive --exclude '*' --include '*.ex_' --include '*.dl_' --include '*.pd_' sym_store s3://openmw-sym
651651
fi
652652
653+
.Merge_Artifacts_Base:
654+
extends: .Ubuntu_Image
655+
stage: build
656+
variables:
657+
GIT_STRATEGY: none
658+
script:
659+
- apt-get update
660+
- apt-get install -y curl dos2unix unzip zip
661+
- curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o awscli-exe-linux-x86_64.zip
662+
- unzip -d awscli-exe-linux-x86_64 awscli-exe-linux-x86_64.zip
663+
- pushd awscli-exe-linux-x86_64
664+
- ./aws/install
665+
- popd
666+
- aws --version
667+
- mkdir -p incoming_artifacts
668+
- mv *.zip incoming_artifacts/
669+
- declare -A destinations
670+
- |
671+
for merge_list in *_to-be-merged.txt; do
672+
while IFS=: read -r partial destination; do
673+
destinations["$destination"]=""
674+
echo "Unzipping '$partial' to '$destination'"
675+
unzip -n -d "$destination" "incoming_artifacts/$partial"
676+
done < <(dos2unix < $merge_list)
677+
done
678+
- |
679+
for destination in ${!destinations[@]}; do
680+
pushd "$destination"
681+
for ci_id in CI-ID_*.txt; do
682+
cat "$ci_id" >> CI-ID.txt
683+
rm "$ci_id"
684+
done
685+
echo "Creating $destination.zip"
686+
zip -r "../$destination.zip" .
687+
popd
688+
done
689+
- |
690+
if [[ -v AWS_ACCESS_KEY_ID ]]; then
691+
artifactDirectory="$CI_PROJECT_NAMESPACE/$CI_COMMIT_REF_NAME/$CI_COMMIT_SHORT_SHA-$CI_JOB_ID/"
692+
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp *.zip s3://openmw-artifacts/$artifactDirectory
693+
fi
694+
artifacts:
695+
when: always
696+
paths:
697+
- "*.zip"
698+
699+
.variables-for-split-jobs: &target-group-one
700+
targets: "openmw openmw-tests"
701+
group_name: "group-one"
702+
703+
.variables-for-split-jobs: &target-group-two
704+
targets: "bsatool components-tests esmtool niftest openmw-cs openmw-cs-tests openmw_detournavigator_navmeshtilescache_benchmark openmw_esm_refid_benchmark openmw_settings_access_benchmark openmw-bulletobjecttool openmw-essimporter openmw-iniimporter openmw-launcher openmw-navmeshtool openmw-wizard"
705+
group_name: "group-two"
706+
653707
.Windows_Ninja_Base:
654708
tags:
655709
- saas-windows-medium-amd64
656710
rules:
657711
- if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event"
658712
before_script:
659713
- Get-Volume
714+
- Stop-Service docker
660715
- Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
661716
- choco source add -n=openmw-proxy -s="https://repo.openmw.org/repository/Chocolatey/" --priority=1
662717
- choco source disable -n=chocolatey
@@ -691,38 +746,40 @@ macOS15_Xcode16_arm64:
691746
- New-Item -Type Directory -Force -Path $env:CCACHE_DIR
692747
- New-Item -Type File -Force -Path MSVC2022_64_Ninja\.cmake\api\v1\query\codemodel-v2
693748
- sh CI/before_script.msvc.sh -c $config -p Win64 -v 2022 -k -V -N -b -t -C $multiview -E
694-
- Get-Volume
695749
- cd MSVC2022_64_Ninja
750+
- Get-Volume
696751
- .\ActivateMSVC.ps1
697-
- cmake --build . --config $config --target $targets
752+
- cmake --build . --config $config --target ($targets.Split(' '))
698753
- ccache --show-stats -v
699754
- cd $config
700-
- echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt
755+
- echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`r`nCI_JOB_ID ${CI_JOB_ID}`r`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt
701756
- $artifactDirectory = "$(Make-SafeFileName("${CI_PROJECT_NAMESPACE}"))/$(Make-SafeFileName("${CI_COMMIT_REF_NAME}"))/$(Make-SafeFileName("${CI_COMMIT_SHORT_SHA}-${CI_JOB_ID}"))/"
702757
- Get-ChildItem -Recurse *.ilk | Remove-Item
703758
- aws --version
704759
- |
705760
if (Get-ChildItem -Recurse *.pdb) {
706-
7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt
761+
7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt
707762
if(!$?) { Exit $LASTEXITCODE }
708763
if (Test-Path env:AWS_ACCESS_KEY_ID) {
709-
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory}
764+
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory}
710765
if(!$?) { Exit $LASTEXITCODE }
711766
}
712767
Push-Location ..
713768
..\CI\Store-Symbols.ps1 -SkipCompress
714769
if(!$?) { Exit $LASTEXITCODE }
715-
7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt
770+
7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt
716771
if(!$?) { Exit $LASTEXITCODE }
717772
Pop-Location
718773
Get-ChildItem -Recurse *.pdb | Remove-Item
719774
}
720-
- 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}.zip"))" '*'
775+
- Rename-Item CI-ID.txt CI-ID_${group_name}.txt
776+
- 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip"))" '*'
721777
- |
722778
if (Test-Path env:AWS_ACCESS_KEY_ID) {
723-
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}.zip"))" s3://openmw-artifacts/${artifactDirectory}
779+
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip"))" s3://openmw-artifacts/${artifactDirectory}
724780
if(!$?) { Exit $LASTEXITCODE }
725781
}
782+
- echo "$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip")):$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}"))" | Out-File -Encoding UTF8 "..\..\${group_name}_to-be-merged.txt"
726783
- |
727784
if ($executables) {
728785
foreach ($exe in $executables.Split(',')) {
@@ -746,6 +803,7 @@ macOS15_Xcode16_arm64:
746803
- "*.log"
747804
- MSVC2022_64_Ninja/*.log
748805
- MSVC2022_64_Ninja/**/*.log
806+
- "*_to-be-merged.txt"
749807
variables:
750808
targets: all
751809
# When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
@@ -800,11 +858,34 @@ macOS15_Xcode16_arm64:
800858
# Gitlab can't successfully execute following binaries due to unknown reason
801859
# executables: "components-tests.exe,openmw-tests.exe,openmw-cs-tests.exe,openmw_detournavigator_navmeshtilescache_benchmark.exe"
802860

861+
.Windows_Ninja_RelWithDebInfo_GroupOne:
862+
extends:
863+
- .Windows_Ninja_RelWithDebInfo
864+
variables:
865+
<<: *target-group-one
866+
867+
.Windows_Ninja_RelWithDebInfo_GroupTwo:
868+
extends:
869+
- .Windows_Ninja_RelWithDebInfo
870+
variables:
871+
<<: *target-group-two
872+
803873
.Windows_Compress_And_Upload_Symbols_Ninja_RelWithDebInfo:
804874
extends:
805875
- .Compress_And_Upload_Symbols_Base
806876
needs:
807-
- job: "Windows_Ninja_RelWithDebInfo"
877+
- job: "Windows_Ninja_RelWithDebInfo_GroupOne"
878+
artifacts: true
879+
- job: "Windows_Ninja_RelWithDebInfo_GroupTwo"
880+
artifacts: true
881+
882+
.Windows_Merge_Artifacts_Ninja_RelWithDebInfo:
883+
extends:
884+
- .Merge_Artifacts_Base
885+
needs:
886+
- job: "Windows_Ninja_RelWithDebInfo_GroupOne"
887+
artifacts: true
888+
- job: "Windows_Ninja_RelWithDebInfo_GroupTwo"
808889
artifacts: true
809890

810891
.Windows_Ninja_CacheInit:
@@ -823,6 +904,7 @@ macOS15_Xcode16_arm64:
823904
- if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event"
824905
before_script:
825906
- Get-Volume
907+
- Stop-Service docker
826908
- Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
827909
- choco source add -n=openmw-proxy -s="https://repo.openmw.org/repository/Chocolatey/" --priority=1
828910
- choco source disable -n=chocolatey
@@ -854,34 +936,36 @@ macOS15_Xcode16_arm64:
854936
- sh CI/before_script.msvc.sh -c $config -p Win64 -v 2022 -k -V -b -t -C $multiview -E
855937
- cd MSVC2022_64
856938
- Get-Volume
857-
- cmake --build . --config $config --target $targets
939+
- cmake --build . --config $config --target ($targets.Split(' '))
858940
- cd $config
859-
- echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt
941+
- echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`r`nCI_JOB_ID ${CI_JOB_ID}`r`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt
860942
- $artifactDirectory = "$(Make-SafeFileName("${CI_PROJECT_NAMESPACE}"))/$(Make-SafeFileName("${CI_COMMIT_REF_NAME}"))/$(Make-SafeFileName("${CI_COMMIT_SHORT_SHA}-${CI_JOB_ID}"))/"
861943
- Get-ChildItem -Recurse *.ilk | Remove-Item
862944
- aws --version
863945
- |
864946
if (Get-ChildItem -Recurse *.pdb) {
865-
7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt
947+
7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt
866948
if(!$?) { Exit $LASTEXITCODE }
867949
if (Test-Path env:AWS_ACCESS_KEY_ID) {
868-
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory}
950+
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory}
869951
if(!$?) { Exit $LASTEXITCODE }
870952
}
871953
Push-Location ..
872954
..\CI\Store-Symbols.ps1 -SkipCompress
873955
if(!$?) { Exit $LASTEXITCODE }
874-
7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt
956+
7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt
875957
if(!$?) { Exit $LASTEXITCODE }
876958
Pop-Location
877959
Get-ChildItem -Recurse *.pdb | Remove-Item
878960
}
879-
- 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}.zip"))" '*'
961+
- Rename-Item CI-ID.txt CI-ID_${group_name}.txt
962+
- 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip"))" '*'
880963
- |
881964
if (Test-Path env:AWS_ACCESS_KEY_ID) {
882-
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}.zip"))" s3://openmw-artifacts/${artifactDirectory}
965+
aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip"))" s3://openmw-artifacts/${artifactDirectory}
883966
if(!$?) { Exit $LASTEXITCODE }
884967
}
968+
- echo "$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip")):$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}"))" | Out-File -Encoding UTF8 "..\..\${group_name}_to-be-merged.txt"
885969
- |
886970
if ($executables) {
887971
foreach ($exe in $executables.Split(',')) {
@@ -904,6 +988,7 @@ macOS15_Xcode16_arm64:
904988
- "*.log"
905989
- MSVC2022_64/*.log
906990
- MSVC2022_64/**/*.log
991+
- "*_to-be-merged.txt"
907992
variables:
908993
targets: ALL_BUILD
909994
# When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
@@ -936,7 +1021,7 @@ macOS15_Xcode16_arm64:
9361021
- job: "Windows_MSBuild_Debug"
9371022
artifacts: true
9381023

939-
Windows_MSBuild_RelWithDebInfo:
1024+
.Windows_MSBuild_RelWithDebInfo:
9401025
extends:
9411026
- .Windows_MSBuild_Base
9421027
variables:
@@ -948,11 +1033,38 @@ Windows_MSBuild_RelWithDebInfo:
9481033
# run this for both pushes and schedules so 'latest successful pipeline for branch' always includes it
9491034
- if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "schedule"
9501035

1036+
Windows_MSBuild_RelWithDebInfo_GroupOne:
1037+
extends:
1038+
- .Windows_MSBuild_RelWithDebInfo
1039+
variables:
1040+
<<: *target-group-one
1041+
1042+
Windows_MSBuild_RelWithDebInfo_GroupTwo:
1043+
extends:
1044+
- .Windows_MSBuild_RelWithDebInfo
1045+
variables:
1046+
<<: *target-group-two
1047+
9511048
Windows_Compress_And_Upload_Symbols_MSBuild_RelWithDebInfo:
9521049
extends:
9531050
- .Compress_And_Upload_Symbols_Base
9541051
needs:
955-
- job: "Windows_MSBuild_RelWithDebInfo"
1052+
- job: "Windows_MSBuild_RelWithDebInfo_GroupOne"
1053+
artifacts: true
1054+
- job: "Windows_MSBuild_RelWithDebInfo_GroupTwo"
1055+
artifacts: true
1056+
# temporarily enabled while we're linking the above on the downloads page
1057+
rules:
1058+
# run this for both pushes and schedules so 'latest successful pipeline for branch' always includes it
1059+
- if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "schedule"
1060+
1061+
Windows_Merge_Artifacts_MSBuild_RelWithDebInfo:
1062+
extends:
1063+
- .Merge_Artifacts_Base
1064+
needs:
1065+
- job: "Windows_MSBuild_RelWithDebInfo_GroupOne"
1066+
artifacts: true
1067+
- job: "Windows_MSBuild_RelWithDebInfo_GroupTwo"
9561068
artifacts: true
9571069
# temporarily enabled while we're linking the above on the downloads page
9581070
rules:

‎apps/openmw/CMakeLists.txt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,19 @@ add_library(openmw-lib STATIC
123123
${OPENMW_SOURCES}
124124
)
125125

126+
if (MSVC)
127+
# these TUs take a lot of memory for codegen
128+
# in CI, they often go over the 8GB memory limit if another cl.exe is running at the same time
129+
# this option (make sure you're not looking at the docs for the identically-named linker option) stops four codegen threads being spun off
130+
# it hurts build time ~33% to enable this unnecessarily, though, so target it to the hungriest boys
131+
set_source_files_properties(
132+
options.cpp
133+
mwlua/stats.cpp
134+
PROPERTIES
135+
COMPILE_OPTIONS /cgthreads1
136+
)
137+
endif()
138+
126139
if(BUILD_OPENMW)
127140
if (ANDROID)
128141
add_library(openmw SHARED

0 commit comments

Comments
 (0)