Skip to content

Commit 0f46d2a

Browse files
mmzakharmmzakhar
committed
GitHib caching on linux, optimizations, removed redundant scripts
1 parent ef71808 commit 0f46d2a

19 files changed

+173
-95
lines changed

‎.appveyor.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,23 +95,29 @@ for:
9595
only:
9696
- job_name: build_linux_apt_cpp
9797
install:
98-
sh: scripts/install_linux_apt.sh $LINUX_CPP_COMPONENTS
98+
sh: |
99+
scripts/setup_apt_repo_linux.sh
100+
scripts/install_linux_apt.sh $LINUX_CPP_COMPONENTS
99101
build_script:
100102
sh: scripts/build_linux.sh c++
101103
-
102104
matrix:
103105
only:
104106
- job_name: build_linux_apt_fortran
105107
install:
106-
sh: scripts/install_linux_apt.sh $LINUX_FORTRAN_COMPONENTS
108+
sh: |
109+
scripts/setup_apt_repo_linux.sh
110+
scripts/install_linux_apt.sh $LINUX_FORTRAN_COMPONENTS
107111
build_script:
108112
sh: scripts/build_linux.sh fortran
109113
-
110114
matrix:
111115
only:
112116
- job_name: build_linux_apt_dpcpp
113117
install:
114-
sh: scripts/install_linux_apt.sh $LINUX_DPCPP_COMPONENTS
118+
sh: |
119+
scripts/setup_apt_repo_linux.sh
120+
scripts/install_linux_apt.sh $LINUX_DPCPP_COMPONENTS
115121
build_script:
116122
sh: scripts/build_linux.sh dpc++
117123

‎.azure-pipelines.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ jobs:
118118
pool:
119119
vmImage: 'ubuntu-latest'
120120
steps:
121+
- script: scripts/setup_apt_repo_linux.sh
122+
displayName: setup apt repo
121123
- script: scripts/install_linux_apt.sh $(LINUX_CPP_COMPONENTS)
122124
displayName: install
123125
- script: scripts/build_linux.sh c++
@@ -127,6 +129,8 @@ jobs:
127129
pool:
128130
vmImage: 'ubuntu-latest'
129131
steps:
132+
- script: scripts/setup_apt_repo_linux.sh
133+
displayName: setup apt repo
130134
- script: scripts/install_linux_apt.sh $(LINUX_FORTRAN_COMPONENTS)
131135
displayName: install
132136
- script: scripts/build_linux.sh fortran
@@ -136,6 +140,8 @@ jobs:
136140
pool:
137141
vmImage: 'ubuntu-latest'
138142
steps:
143+
- script: scripts/setup_apt_repo_linux.sh
144+
displayName: setup apt repo
139145
- script: scripts/install_linux_apt.sh $(LINUX_DPCPP_COMPONENTS)
140146
displayName: install
141147
- script: scripts/build_linux.sh dpc++

‎.circleci/config.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,12 @@ jobs:
7979
- image: ubuntu:latest
8080
steps:
8181
- checkout
82+
- run:
83+
name: install prerequisites
84+
command: . scripts/install_prerequisites_linux_apt_no_sudo.sh
85+
- run:
86+
name: setup apt repo
87+
command: . scripts/setup_apt_repo_linux_no_sudo.sh
8288
- run:
8389
name: install
8490
command: . scripts/install_linux_apt_no_sudo.sh intel-oneapi-dpcpp-cpp-compiler-pro
@@ -91,6 +97,12 @@ jobs:
9197
- image: ubuntu:latest
9298
steps:
9399
- checkout
100+
- run:
101+
name: install prerequisites
102+
command: . scripts/install_prerequisites_linux_apt_no_sudo.sh
103+
- run:
104+
name: setup apt repo
105+
command: . scripts/setup_apt_repo_linux_no_sudo.sh
94106
- run:
95107
name: install
96108
command: . scripts/install_linux_apt_no_sudo.sh intel-oneapi-ifort
@@ -103,6 +115,12 @@ jobs:
103115
- image: ubuntu:latest
104116
steps:
105117
- checkout
118+
- run:
119+
name: install prerequisites
120+
command: . scripts/install_prerequisites_linux_apt_no_sudo.sh
121+
- run:
122+
name: setup apt repo
123+
command: . scripts/setup_apt_repo_linux_no_sudo.sh
106124
- run:
107125
name: install
108126
command: . scripts/install_linux_apt_no_sudo.sh intel-oneapi-dpcpp-cpp-compiler

‎.github/workflows/build_all.yml

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,16 @@ jobs:
3333
uses: actions/cache@v2
3434
with:
3535
path: C:\Program Files (x86)\Intel\oneAPI\compiler
36-
key: install-${{ env.WINDOWS_HPCKIT_URL }}-${{ env.WINDOWS_CPP_COMPONENTS }}-compiler
36+
key: install-${{ env.WINDOWS_HPCKIT_URL }}-${{ env.WINDOWS_CPP_COMPONENTS }}-compiler-${{ hashFiles('**/scripts/cache_exclude_windows.sh') }}
3737
- name: install
3838
if: steps.cache-install.outputs.cache-hit != 'true'
3939
run: scripts/install_windows.bat $WINDOWS_HPCKIT_URL $WINDOWS_CPP_COMPONENTS
4040
- name: build
4141
run: scripts/build_windows.bat c++
42+
- name: exclude unused files from cache
43+
if: steps.cache-install.outputs.cache-hit != 'true'
44+
shell: bash
45+
run: scripts/cache_exclude_windows.sh
4246

4347
# Delete the following if you don't want to save install logs
4448
- name: Saving install logs
@@ -64,12 +68,16 @@ jobs:
6468
uses: actions/cache@v2
6569
with:
6670
path: C:\Program Files (x86)\Intel\oneAPI\compiler
67-
key: install-${{ env.WINDOWS_HPCKIT_URL }}-${{ env.WINDOWS_FORTRAN_COMPONENTS }}-compiler
71+
key: install-${{ env.WINDOWS_HPCKIT_URL }}-${{ env.WINDOWS_FORTRAN_COMPONENTS }}-compiler-${{ hashFiles('**/scripts/cache_exclude_windows.sh') }}
6872
- name: install
6973
if: steps.cache-install.outputs.cache-hit != 'true'
7074
run: scripts/install_windows.bat $WINDOWS_HPCKIT_URL $WINDOWS_FORTRAN_COMPONENTS
7175
- name: build
7276
run: scripts/build_windows.bat fortran
77+
- name: exclude unused files from cache
78+
if: steps.cache-install.outputs.cache-hit != 'true'
79+
shell: bash
80+
run: scripts/cache_exclude_windows.sh
7381

7482
# Delete the following if you don't want to save install logs
7583
- name: Saving install logs
@@ -98,12 +106,16 @@ jobs:
98106
C:\Program Files (x86)\Intel\oneAPI\compiler
99107
C:\Program Files (x86)\Intel\oneAPI\tbb
100108
C:\Windows\System32\OpenCL.dll
101-
key: install-${{ env.WINDOWS_BASEKIT_URL }}-${{ env.WINDOWS_DPCPP_COMPONENTS }}-compiler-tbb-opencl
109+
key: install-${{ env.WINDOWS_BASEKIT_URL }}-${{ env.WINDOWS_DPCPP_COMPONENTS }}-compiler-tbb-opencl-${{ hashFiles('**/scripts/cache_exclude_windows.sh') }}
102110
- name: install
103111
if: steps.cache-install.outputs.cache-hit != 'true'
104112
run: scripts/install_windows.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_COMPONENTS
105113
- name: build
106114
run: scripts/build_windows.bat dpc++
115+
- name: exclude unused files from cache
116+
if: steps.cache-install.outputs.cache-hit != 'true'
117+
shell: bash
118+
run: scripts/cache_exclude_windows.sh
107119

108120
# Delete the following if you don't want to save install logs
109121
- name: Saving install logs
@@ -124,10 +136,25 @@ jobs:
124136
shell: bash
125137
steps:
126138
- uses: actions/checkout@v2
139+
- name: setup apt repo
140+
run: scripts/setup_apt_repo_linux.sh
141+
- name: collect versioned dependencies of apt packages
142+
run: scripts/apt_depends.sh $LINUX_CPP_COMPONENTS | tee depends.txt
143+
- name: cache install
144+
id: cache-install
145+
uses: actions/cache@v2
146+
with:
147+
path: |
148+
/opt/intel/oneapi/compiler
149+
key: install-${{ env.LINUX_CPP_COMPONENTS }}-${{ hashFiles('**/depends.txt') }}-compiler-${{ hashFiles('**/scripts/cache_exclude_linux.sh') }}
127150
- name: install
151+
if: steps.cache-install.outputs.cache-hit != 'true'
128152
run: scripts/install_linux_apt.sh $LINUX_CPP_COMPONENTS
129153
- name: build
130154
run: scripts/build_linux.sh c++
155+
- name: exclude unused files from cache
156+
if: steps.cache-install.outputs.cache-hit != 'true'
157+
run: scripts/cache_exclude_linux.sh
131158

132159
build_linux_apt_fortran:
133160
runs-on: ubuntu-latest
@@ -136,10 +163,25 @@ jobs:
136163
shell: bash
137164
steps:
138165
- uses: actions/checkout@v2
166+
- name: setup apt repo
167+
run: scripts/setup_apt_repo_linux.sh
168+
- name: collect versioned dependencies of apt packages
169+
run: scripts/apt_depends.sh $LINUX_FORTRAN_COMPONENTS | tee depends.txt
170+
- name: cache install
171+
id: cache-install
172+
uses: actions/cache@v2
173+
with:
174+
path: |
175+
/opt/intel/oneapi/compiler
176+
key: install-${{ env.LINUX_FORTRAN_COMPONENTS }}-${{ hashFiles('**/depends.txt') }}-compiler-${{ hashFiles('**/scripts/cache_exclude_linux.sh') }}
139177
- name: install
178+
if: steps.cache-install.outputs.cache-hit != 'true'
140179
run: scripts/install_linux_apt.sh $LINUX_FORTRAN_COMPONENTS
141180
- name: build
142181
run: scripts/build_linux.sh fortran
182+
- name: exclude unused files from cache
183+
if: steps.cache-install.outputs.cache-hit != 'true'
184+
run: scripts/cache_exclude_linux.sh
143185

144186
build_linux_apt_dpcpp:
145187
runs-on: ubuntu-latest
@@ -148,10 +190,26 @@ jobs:
148190
shell: bash
149191
steps:
150192
- uses: actions/checkout@v2
193+
- name: setup apt repo
194+
run: scripts/setup_apt_repo_linux.sh
195+
- name: collect versioned dependencies of apt packages
196+
run: scripts/apt_depends.sh $LINUX_DPCPP_COMPONENTS | tee depends.txt
197+
- name: cache install
198+
id: cache-install
199+
uses: actions/cache@v2
200+
with:
201+
path: |
202+
/opt/intel/oneapi/compiler
203+
/opt/intel/oneapi/tbb
204+
key: install-${{ env.LINUX_DPCPP_COMPONENTS }}-${{ hashFiles('**/depends.txt') }}-compiler-tbb-${{ hashFiles('**/scripts/cache_exclude_linux.sh') }}
151205
- name: install
206+
if: steps.cache-install.outputs.cache-hit != 'true'
152207
run: scripts/install_linux_apt.sh $LINUX_DPCPP_COMPONENTS
153208
- name: build
154209
run: scripts/build_linux.sh dpc++
210+
- name: exclude unused files from cache
211+
if: steps.cache-install.outputs.cache-hit != 'true'
212+
run: scripts/cache_exclude_linux.sh
155213

156214
build_linux_container_cpp:
157215
runs-on: ubuntu-20.04
@@ -224,7 +282,7 @@ jobs:
224282
shell: bash
225283
steps:
226284
- uses: actions/checkout@v2
227-
- name: Prepare for cache restore
285+
- name: prepare for cache restore
228286
run: |
229287
sudo mkdir -p /opt/intel
230288
sudo chown $USER /opt/intel

‎.gitlab-ci.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ build_windows_cpp:
3434

3535
# Delete the following if you don't want to save install logs
3636
artifacts:
37+
when: always
3738
paths:
3839
- extract.log
3940
- bootstrapper*
@@ -50,6 +51,7 @@ build_windows_fortran:
5051

5152
# Delete the following if you don't want to save install logs
5253
artifacts:
54+
when: always
5355
paths:
5456
- extract.log
5557
- bootstrapper*
@@ -66,6 +68,7 @@ build_windows_dpcpp:
6668

6769
# Delete the following if you don't want to save install logs
6870
artifacts:
71+
when: always
6972
paths:
7073
- extract.log
7174
- bootstrapper*
@@ -76,19 +79,25 @@ build_linux_apt_cpp:
7679
image: ubuntu:latest
7780
stage: build
7881
script:
82+
- . scripts/install_prerequisites_linux_apt_no_sudo.sh
83+
- . scripts/setup_apt_repo_linux_no_sudo.sh
7984
- . scripts/install_linux_apt_no_sudo.sh $LINUX_CPP_COMPONENTS
8085
- scripts/build_linux.sh c++
8186

8287
build_linux_apt_fortran:
8388
image: ubuntu:latest
8489
stage: build
8590
script:
91+
- . scripts/install_prerequisites_linux_apt_no_sudo.sh
92+
- . scripts/setup_apt_repo_linux_no_sudo.sh
8693
- . scripts/install_linux_apt_no_sudo.sh $LINUX_FORTRAN_COMPONENTS
8794
- scripts/build_linux.sh fortran
8895

8996
build_linux_apt_dpcpp:
9097
image: ubuntu:latest
9198
stage: build
9299
script:
100+
- . scripts/install_prerequisites_linux_apt_no_sudo.sh
101+
- . scripts/setup_apt_repo_linux_no_sudo.sh
93102
- . scripts/install_linux_apt_no_sudo.sh $LINUX_DPCPP_COMPONENTS
94103
- scripts/build_linux.sh dpc++

‎.travis.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,19 +36,25 @@ jobs:
3636
- name: build_linux_apt_cpp
3737
os: linux
3838
dist: bionic
39-
install: scripts/install_linux_apt.sh $LINUX_CPP_COMPONENTS
39+
install: |
40+
scripts/setup_apt_repo_linux.sh
41+
scripts/install_linux_apt.sh $LINUX_CPP_COMPONENTS
4042
script: scripts/build_linux.sh c++
4143

4244
- name: build_linux_apt_fortran
4345
os: linux
4446
dist: bionic
45-
install: scripts/install_linux_apt.sh $LINUX_FORTRAN_COMPONENTS
47+
install: |
48+
scripts/setup_apt_repo_linux.sh
49+
scripts/install_linux_apt.sh $LINUX_FORTRAN_COMPONENTS
4650
script: scripts/build_linux.sh fortran
4751

4852
- name: build_linux_apt_dpcpp
4953
os: linux
5054
dist: bionic
51-
install: scripts/install_linux_apt.sh $LINUX_DPCPP_COMPONENTS
55+
install: |
56+
scripts/setup_apt_repo_linux.sh
57+
scripts/install_linux_apt.sh $LINUX_DPCPP_COMPONENTS
5258
script: scripts/build_linux.sh dpc++
5359

5460
- name: build_linux_container_cpp

‎scripts/apt_depends.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
# SPDX-FileCopyrightText: 2020 Intel Corporation
4+
#
5+
# SPDX-License-Identifier: MIT
6+
7+
echo "$1" | sed "s/,/ /g" | xargs -n 1 apt-cache depends

‎scripts/build_linux.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ LANGUAGE=$1
88

99
git clone --depth 1 https://github.com/oneapi-src/oneAPI-samples.git
1010

11-
source /opt/intel/oneapi/setvars.sh
11+
source /opt/intel/oneapi/compiler/2021.1-beta10/env/vars.sh
1212

1313
case $LANGUAGE in
1414
c++)
@@ -20,6 +20,7 @@ fortran)
2020
make && make run
2121
;;
2222
dpc++)
23+
source /opt/intel/oneapi/tbb/2021.1-beta10/env/vars.sh
2324
cd oneAPI-samples/DirectProgramming/DPC++/DenseLinearAlgebra/vector-add
2425
make all && make run
2526
;;

‎scripts/cache_exclude_linux.sh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
# SPDX-FileCopyrightText: 2020 Intel Corporation
4+
#
5+
# SPDX-License-Identifier: MIT
6+
7+
sudo rm -rf /opt/intel/oneapi/compiler/2021.1-beta10/linux/compiler/lib/ia32_lin
8+
sudo rm -rf /opt/intel/oneapi/compiler/2021.1-beta10/linux/bin/ia32
9+
sudo rm -rf /opt/intel/oneapi/compiler/2021.1-beta10/linux/lib/emu
10+
sudo rm -rf /opt/intel/oneapi/compiler/2021.1-beta10/linux/lib/oclfpga

‎scripts/cache_exclude_windows.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
# SPDX-FileCopyrightText: 2020 Intel Corporation
4+
#
5+
# SPDX-License-Identifier: MIT
6+
7+
rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\compiler\lib\ia32_win"
8+
rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\bin\intel64_ia32"
9+
rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\lib\emu"
10+
rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\lib\oclfpga"
11+
rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\lib\ocloc"
12+
rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\lib\x86"

0 commit comments

Comments
 (0)