Skip to content

Commit 1978992

Browse files
mmzakharmmzakhar
authored
Merge pull request #28 from oneapi-src/dev/azure_pipelines_caching
Added caching in Azure Pipelines
2 parents 994d516 + cb97181 commit 1978992

File tree

1 file changed

+100
-0
lines changed

1 file changed

+100
-0
lines changed

‎.azure-pipelines.yml

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,19 @@ jobs:
4343
pool:
4444
vmImage: 'windows-latest'
4545
steps:
46+
- task: Cache@2
47+
inputs:
48+
path: C:\Program Files (x86)\Intel\oneAPI\compiler
49+
key: '"install" | "$(WINDOWS_HPCKIT_URL)" | "$(WINDOWS_CPP_COMPONENTS)" | compiler | scripts/cache_exclude_windows.sh'
50+
cacheHitVar: CACHE_RESTORED
4651
- script: scripts/install_windows.bat $(WINDOWS_HPCKIT_URL) $(WINDOWS_CPP_COMPONENTS)
4752
displayName: install
53+
condition: ne(variables.CACHE_RESTORED, 'true')
4854
- script: scripts/build_windows.bat c++
4955
displayName: build
56+
- bash: scripts/cache_exclude_windows.sh
57+
displayName: exclude unused files from cache
58+
condition: ne(variables.CACHE_RESTORED, 'true')
5059

5160
# Delete the following if you don't want to save install logs
5261
- task: CopyFiles@2
@@ -68,10 +77,19 @@ jobs:
6877
pool:
6978
vmImage: 'windows-latest'
7079
steps:
80+
- task: Cache@2
81+
inputs:
82+
path: C:\Program Files (x86)\Intel\oneAPI\compiler
83+
key: '"install" | "$(WINDOWS_HPCKIT_URL)" | "$(WINDOWS_FORTRAN_COMPONENTS)" | "compiler" | scripts/cache_exclude_windows.sh'
84+
cacheHitVar: CACHE_RESTORED
7185
- script: scripts/install_windows.bat $(WINDOWS_HPCKIT_URL) $(WINDOWS_FORTRAN_COMPONENTS)
7286
displayName: install
87+
condition: ne(variables.CACHE_RESTORED, 'true')
7388
- script: scripts/build_windows.bat fortran
7489
displayName: build
90+
- bash: scripts/cache_exclude_windows.sh
91+
displayName: exclude unused files from cache
92+
condition: ne(variables.CACHE_RESTORED, 'true')
7593

7694
# Delete the following if you don't want to save install logs
7795
- task: CopyFiles@2
@@ -93,10 +111,35 @@ jobs:
93111
pool:
94112
vmImage: 'windows-latest'
95113
steps:
114+
- task: Cache@2 # multiple paths per cache not supported yet. See https://github.com/microsoft/azure-pipelines-agent/pull/2834
115+
inputs:
116+
path: C:\Program Files (x86)\Intel\oneAPI\compiler
117+
key: '"install" | "$(WINDOWS_BASEKIT_URL)" | "$(WINDOWS_DPCPP_COMPONENTS)" | "compiler" | scripts/cache_exclude_windows.sh'
118+
- task: Cache@2
119+
inputs:
120+
path: C:\Program Files (x86)\Intel\oneAPI\tbb
121+
key: '"install" | "$(WINDOWS_BASEKIT_URL)" | "$(WINDOWS_DPCPP_COMPONENTS)" | "tbb" | scripts/cache_exclude_windows.sh'
122+
- task: Cache@2
123+
inputs:
124+
path: opencl # caching of individual files is not supported, caching OpenCL.dll in a folder as a workaround.
125+
key: '"install" | "$(WINDOWS_BASEKIT_URL)" | "$(WINDOWS_DPCPP_COMPONENTS)" | "opencl_folder" | scripts/cache_exclude_windows.sh'
126+
cacheHitVar: CACHE_RESTORED
96127
- script: scripts/install_windows.bat $(WINDOWS_BASEKIT_URL) $(WINDOWS_DPCPP_COMPONENTS)
97128
displayName: install
129+
condition: ne(variables.CACHE_RESTORED, 'true')
130+
- bash: cp opencl/OpenCL.dll C:/Windows/System32/
131+
displayName: restore OpenCL.dll from cache
132+
condition: eq(variables.CACHE_RESTORED, 'true')
98133
- script: scripts/build_windows.bat dpc++
99134
displayName: build
135+
- bash: scripts/cache_exclude_windows.sh
136+
displayName: exclude unused files from cache
137+
condition: ne(variables.CACHE_RESTORED, 'true')
138+
- bash: |
139+
mkdir -p opencl
140+
cp C:/Windows/System32/OpenCL.dll opencl/
141+
displayName: copy OpenCL.dll to a folder for caching
142+
condition: ne(variables.CACHE_RESTORED, 'true')
100143
101144
# Delete the following if you don't want to save install logs
102145
- task: CopyFiles@2
@@ -120,32 +163,69 @@ jobs:
120163
steps:
121164
- script: scripts/setup_apt_repo_linux.sh
122165
displayName: setup apt repo
166+
- script: scripts/apt_depends.sh $LINUX_CPP_COMPONENTS | tee depends.txt
167+
displayName: collect versioned dependencies of apt packages
168+
- task: Cache@2
169+
inputs:
170+
path: /opt/intel/oneapi/compiler
171+
key: '"install" | "$(LINUX_CPP_COMPONENTS)" | depends.txt | "compiler" | scripts/cache_exclude_linux.sh'
172+
cacheHitVar: CACHE_RESTORED
123173
- script: scripts/install_linux_apt.sh $(LINUX_CPP_COMPONENTS)
124174
displayName: install
175+
condition: ne(variables.CACHE_RESTORED, 'true')
125176
- script: scripts/build_linux.sh c++
126177
displayName: build
178+
- bash: scripts/cache_exclude_linux.sh
179+
displayName: exclude unused files from cache
180+
condition: ne(variables.CACHE_RESTORED, 'true')
127181

128182
- job: build_linux_apt_fortran
129183
pool:
130184
vmImage: 'ubuntu-latest'
131185
steps:
132186
- script: scripts/setup_apt_repo_linux.sh
133187
displayName: setup apt repo
188+
- script: scripts/apt_depends.sh $LINUX_CPP_COMPONENTS | tee depends.txt
189+
displayName: collect versioned dependencies of apt packages
190+
- task: Cache@2
191+
inputs:
192+
path: /opt/intel/oneapi/compiler
193+
key: '"install" | "$(LINUX_FORTRAN_COMPONENTS)" | depends.txt | "compiler" | scripts/cache_exclude_linux.sh'
194+
cacheHitVar: CACHE_RESTORED
134195
- script: scripts/install_linux_apt.sh $(LINUX_FORTRAN_COMPONENTS)
135196
displayName: install
197+
condition: ne(variables.CACHE_RESTORED, 'true')
136198
- script: scripts/build_linux.sh fortran
137199
displayName: build
200+
- bash: scripts/cache_exclude_linux.sh
201+
displayName: exclude unused files from cache
202+
condition: ne(variables.CACHE_RESTORED, 'true')
138203

139204
- job: build_linux_apt_dpcpp
140205
pool:
141206
vmImage: 'ubuntu-latest'
142207
steps:
143208
- script: scripts/setup_apt_repo_linux.sh
144209
displayName: setup apt repo
210+
- script: scripts/apt_depends.sh $LINUX_CPP_COMPONENTS | tee depends.txt
211+
displayName: collect versioned dependencies of apt packages
212+
- task: Cache@2
213+
inputs:
214+
path: /opt/intel/oneapi/compiler
215+
key: '"install" | "$(LINUX_DPCPP_COMPONENTS)" | depends.txt | "compiler" | scripts/cache_exclude_linux.sh'
216+
- task: Cache@2
217+
inputs:
218+
path: /opt/intel/oneapi/tbb
219+
key: '"install" | "$(LINUX_DPCPP_COMPONENTS)" | depends.txt | "tbb" | scripts/cache_exclude_linux.sh'
220+
cacheHitVar: CACHE_RESTORED
145221
- script: scripts/install_linux_apt.sh $(LINUX_DPCPP_COMPONENTS)
146222
displayName: install
223+
condition: ne(variables.CACHE_RESTORED, 'true')
147224
- script: scripts/build_linux.sh dpc++
148225
displayName: build
226+
- bash: scripts/cache_exclude_linux.sh
227+
displayName: exclude unused files from cache
228+
condition: ne(variables.CACHE_RESTORED, 'true')
149229

150230
- job: build_linux_container_cpp
151231
pool:
@@ -181,8 +261,18 @@ jobs:
181261
pool:
182262
vmImage: 'macOS-latest'
183263
steps:
264+
- script: |
265+
sudo mkdir -p /opt/intel
266+
sudo chown $USER /opt/intel
267+
displayName: prepare for cache restore
268+
- task: Cache@2
269+
inputs:
270+
path: /opt/intel/oneapi
271+
key: '"install" | "$(MACOS_HPCKIT_URL)" | "$(MACOS_CPP_COMPONENTS)"'
272+
cacheHitVar: CACHE_RESTORED
184273
- script: scripts/install_macos.sh $(MACOS_HPCKIT_URL) $(MACOS_CPP_COMPONENTS)
185274
displayName: install
275+
condition: ne(variables.CACHE_RESTORED, 'true')
186276
- script: scripts/build_macos.sh c++
187277
displayName: build
188278

@@ -205,8 +295,18 @@ jobs:
205295
pool:
206296
vmImage: 'macOS-latest'
207297
steps:
298+
- script: |
299+
sudo mkdir -p /opt/intel
300+
sudo chown $USER /opt/intel
301+
displayName: prepare for cache restore
302+
- task: Cache@2
303+
inputs:
304+
path: /opt/intel/oneapi
305+
key: '"install" | "$(MACOS_HPCKIT_URL)" | "$(MACOS_FORTRAN_COMPONENTS)"'
306+
cacheHitVar: CACHE_RESTORED
208307
- script: scripts/install_macos.sh $(MACOS_HPCKIT_URL) $(MACOS_FORTRAN_COMPONENTS)
209308
displayName: install
309+
condition: ne(variables.CACHE_RESTORED, 'true')
210310
- script: scripts/build_macos.sh fortran
211311
displayName: build
212312

0 commit comments

Comments
 (0)