Added GitLab CI caching, fixed Circle CI caching on Windows (#29)

Author: mmzakhar

.circleci/config.yml

@@ -71,7 +71,7 @@ jobs:
       - run:
           command: |
             mkdir InstallLogs
-            cp extract* bootstrapper* installer* InstallLogs
+            cp extract* bootstrapper* installer* InstallLogs || :
           when: always
       - store_artifacts:
           path: InstallLogs
@@ -105,7 +105,7 @@ jobs:
       - run:
           command: |
             mkdir InstallLogs
-            cp extract* bootstrapper* installer* InstallLogs
+            cp extract* bootstrapper* installer* InstallLogs || :
           when: always
       - store_artifacts:
           path: InstallLogs
@@ -118,7 +118,7 @@ jobs:
       - checkout
       - restore_cache:
           keys:
-            - install-<< pipeline.parameters.WINDOWS_BASEKIT_URL >>-<< pipeline.parameters.WINDOWS_DPCPP_COMPONENTS >>-compiler-{{ checksum "scripts/cache_exclude_windows.sh" }}
+            - install-<< pipeline.parameters.WINDOWS_BASEKIT_URL >>-<< pipeline.parameters.WINDOWS_DPCPP_COMPONENTS >>-compiler-tbb-opencl-{{ checksum "scripts/cache_exclude_windows.sh" }}
       - run:
           name: install
           command: |
@@ -131,7 +131,7 @@ jobs:
           name: exclude unused files from cache
           command: scripts/cache_exclude_windows.sh
       - save_cache:
-          key: install-<< pipeline.parameters.WINDOWS_BASEKIT_URL >>-<< pipeline.parameters.WINDOWS_DPCPP_COMPONENTS >>-compiler-{{ checksum "scripts/cache_exclude_windows.sh" }}
+          key: install-<< pipeline.parameters.WINDOWS_BASEKIT_URL >>-<< pipeline.parameters.WINDOWS_DPCPP_COMPONENTS >>-compiler-tbb-opencl-{{ checksum "scripts/cache_exclude_windows.sh" }}
           paths:
             - C:\Program Files (x86)\Intel\oneAPI\compiler
             - C:\Program Files (x86)\Intel\oneAPI\tbb
@@ -141,7 +141,7 @@ jobs:
       - run:
           command: |
             mkdir InstallLogs
-            cp extract* bootstrapper* installer* InstallLogs
+            cp extract* bootstrapper* installer* InstallLogs || :
           when: always
       - store_artifacts:
           path: InstallLogs

.gitlab-ci.yml

@@ -17,6 +17,7 @@ variables:
   LINUX_DPCPP_COMPONENTS: intel-oneapi-dpcpp-cpp-compiler
   MACOS_CPP_COMPONENTS: intel.oneapi.mac.cpp-compiler
   MACOS_FORTRAN_COMPONENTS: intel.oneapi.mac.ifort-compiler
+  RELEASE: 2021.1-beta10
 
 .shared_windows_runners:
   tags:
@@ -28,9 +29,32 @@ build_windows_cpp:
   extends:
   - .shared_windows_runners
   stage: build
+  cache:
+    key:
+      prefix: install-${RELEASE}-${WINDOWS_CPP_COMPONENTS}-compiler
+      files:
+        - scripts/cache_exclude_windows.ps1
+    paths:
+      - cache # GitLab CI cannot cache files outside build directory
+  before_script: # Restoring install directory from 'cache' folder
+    - |
+        if(Test-Path cache) {
+          New-Item -ItemType "directory" -Path "C:\Program Files (x86)\Intel\oneAPI" -Force
+          Move-Item -Path cache\compiler -Destination "C:\Program Files (x86)\Intel\oneAPI"
+        } else {
+          scripts/install_windows.bat $WINDOWS_HPCKIT_URL $WINDOWS_CPP_COMPONENTS
+        }
   script:
-    - scripts/install_windows.bat $WINDOWS_HPCKIT_URL $WINDOWS_CPP_COMPONENTS
     - scripts/build_windows.bat c++ 2019_build_tools
+  after_script:  # Moving install directory into 'cache' folder inside build directory
+    - |
+        if(!(Test-Path cache)) {
+          scripts/cache_exclude_windows.ps1
+          New-Item -ItemType "directory" -Path "cache"
+        }
+        # "push-pull" is the default cache policy, so cache folder needs to be setup in every run.
+        # consider setting "policy: pull" once the cache is warm.
+        Move-Item -Path "C:\Program Files (x86)\Intel\oneAPI\compiler" -Destination "cache"
 
   # Delete the following if you don't want to save install logs
   artifacts:
@@ -45,9 +69,30 @@ build_windows_fortran:
   extends:
   - .shared_windows_runners
   stage: build
+  cache:
+    key:
+      prefix: install-${RELEASE}-${WINDOWS_FORTRAN_COMPONENTS}-compiler
+      files:
+        - scripts/cache_exclude_windows.ps1
+    paths:
+      - cache
+  before_script:
+    - |
+        if(Test-Path cache) {
+          New-Item -ItemType "directory" -Path "C:\Program Files (x86)\Intel\oneAPI" -Force
+          Move-Item -Path cache\compiler -Destination "C:\Program Files (x86)\Intel\oneAPI"
+        } else {
+          scripts/install_windows.bat $WINDOWS_HPCKIT_URL $WINDOWS_FORTRAN_COMPONENTS
+        }
   script:
-    - scripts/install_windows.bat $WINDOWS_HPCKIT_URL $WINDOWS_FORTRAN_COMPONENTS
     - scripts/build_windows.bat fortran 2019_build_tools
+  after_script:
+    - |
+        if(!(Test-Path cache)) {
+          scripts/cache_exclude_windows.ps1
+          New-Item -ItemType "directory" -Path "cache"
+        }
+        Move-Item -Path "C:\Program Files (x86)\Intel\oneAPI\compiler" -Destination "cache"
 
   # Delete the following if you don't want to save install logs
   artifacts:
@@ -62,10 +107,34 @@ build_windows_dpcpp:
   extends:
   - .shared_windows_runners
   stage: build
+  cache:
+    key:
+      prefix: install-${RELEASE}-${WINDOWS_DPCPP_COMPONENTS}-compiler-tbb-opencl
+      files:
+        - scripts/cache_exclude_windows.ps1
+    paths:
+      - cache
+  before_script:
+    - |
+        if(Test-Path cache) {
+          New-Item -ItemType "directory" -Path "C:\Program Files (x86)\Intel\oneAPI" -Force
+          Move-Item -Path cache\compiler -Destination "C:\Program Files (x86)\Intel\oneAPI"
+          Move-Item -Path cache\tbb -Destination "C:\Program Files (x86)\Intel\oneAPI"
+          Move-Item -Path cache\OpenCL.dll -Destination "C:\Windows\System32"
+        } else {
+          scripts/install_windows.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_COMPONENTS
+        }
   script:
-    - scripts/install_windows.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_COMPONENTS
     - scripts/build_windows.bat dpc++ 2019_build_tools
-
+  after_script:
+    - |
+        if(!(Test-Path cache)) {
+          scripts/cache_exclude_windows.ps1
+          New-Item -ItemType "directory" -Path "cache"
+        }
+        Move-Item -Path "C:\Program Files (x86)\Intel\oneAPI\compiler" -Destination "cache"
+        Move-Item -Path "C:\Program Files (x86)\Intel\oneAPI\tbb" -Destination "cache"
+        Move-Item -Path "C:\Windows\System32\OpenCL.dll" -Destination "cache"
   # Delete the following if you don't want to save install logs
   artifacts:
     when: always
@@ -78,26 +147,97 @@ build_windows_dpcpp:
 build_linux_apt_cpp:
   image: ubuntu:latest
   stage: build
-  script:
+  cache:
+    key: # GitLab CI doesn't support steps before cache is restored, i.e. only static cache keys are supported.
+      prefix: install-${RELEASE}-${LINUX_CPP_COMPONENTS}-compiler
+      files:
+        - scripts/cache_exclude_linux_no_sudo.sh
+    paths:
+      - cache
+  before_script:
     - . scripts/install_prerequisites_linux_apt_no_sudo.sh
-    - . scripts/setup_apt_repo_linux_no_sudo.sh
-    - . scripts/install_linux_apt_no_sudo.sh $LINUX_CPP_COMPONENTS
+    - |
+        if [[ -d cache ]]
+        then
+          mkdir -p /opt/intel/oneapi
+          mv cache/compiler /opt/intel/oneapi/
+        else
+          . scripts/setup_apt_repo_linux_no_sudo.sh
+          . scripts/install_linux_apt_no_sudo.sh $LINUX_CPP_COMPONENTS
+        fi
+  script:
     - scripts/build_linux.sh c++
+  after_script:
+    - |
+        if [[ ! -d cache ]]
+        then
+          . scripts/cache_exclude_linux_no_sudo.sh
+          mkdir cache
+        fi
+        mv /opt/intel/oneapi/compiler cache
 
 build_linux_apt_fortran:
   image: ubuntu:latest
   stage: build
-  script:
+  cache:
+    key:
+      prefix: install-${RELEASE}-${LINUX_FORTRAN_COMPONENTS}-compiler
+      files:
+        - scripts/cache_exclude_linux_no_sudo.sh
+    paths:
+      - cache
+  before_script:
     - . scripts/install_prerequisites_linux_apt_no_sudo.sh
-    - . scripts/setup_apt_repo_linux_no_sudo.sh
-    - . scripts/install_linux_apt_no_sudo.sh $LINUX_FORTRAN_COMPONENTS
+    - |
+        if [[ -d cache ]]
+        then
+          mkdir -p /opt/intel/oneapi
+          mv cache/compiler /opt/intel/oneapi/
+        else
+          . scripts/setup_apt_repo_linux_no_sudo.sh
+          . scripts/install_linux_apt_no_sudo.sh $LINUX_FORTRAN_COMPONENTS
+        fi
+  script:
     - scripts/build_linux.sh fortran
+  after_script:
+    - |
+        if [[ ! -d cache ]]
+        then
+          . scripts/cache_exclude_linux_no_sudo.sh
+          mkdir cache
+        fi
+        mv /opt/intel/oneapi/compiler cache
 
 build_linux_apt_dpcpp:
   image: ubuntu:latest
   stage: build
-  script:
+  cache:
+    key:
+      prefix: install-${RELEASE}-${LINUX_DPCPP_COMPONENTS}-compiler-tbb
+      files:
+        - scripts/cache_exclude_linux_no_sudo.sh
+    paths:
+      - cache
+  before_script:
     - . scripts/install_prerequisites_linux_apt_no_sudo.sh
-    - . scripts/setup_apt_repo_linux_no_sudo.sh
-    - . scripts/install_linux_apt_no_sudo.sh $LINUX_DPCPP_COMPONENTS
+    - |
+        if [[ -d cache ]]
+        then
+          mkdir -p /opt/intel/oneapi
+          mv cache/compiler /opt/intel/oneapi/
+          mv cache/tbb /opt/intel/oneapi/
+        else
+          . scripts/setup_apt_repo_linux_no_sudo.sh
+          . scripts/install_linux_apt_no_sudo.sh $LINUX_DPCPP_COMPONENTS
+        fi
+  script:
     - scripts/build_linux.sh dpc++
+  after_script:
+    - |
+        if [[ ! -d cache ]]
+        then
+          . scripts/cache_exclude_linux_no_sudo.sh
+          mkdir cache
+        fi
+        mv /opt/intel/oneapi/compiler cache
+        mv /opt/intel/oneapi/tbb cache

scripts/cache_exclude_windows.ps1

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+Remove-Item "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\compiler\lib\ia32_win" -Force  -Recurse -ErrorAction SilentlyContinue
+Remove-Item "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\bin\intel64_ia32" -Force  -Recurse -ErrorAction SilentlyContinue
+Remove-Item "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\lib\emu" -Force  -Recurse -ErrorAction SilentlyContinue
+Remove-Item "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\lib\oclfpga" -Force  -Recurse -ErrorAction SilentlyContinue
+Remove-Item "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\lib\ocloc" -Force  -Recurse -ErrorAction SilentlyContinue
+Remove-Item "C:\Program Files (x86)\Intel\oneAPI\compiler\2021.1-beta10\windows\lib\x86" -Force  -Recurse -ErrorAction SilentlyContinue
+
+exit 0