Build Linux wrappers on docker (#2603)

Author: nirinchev

.github/templates/codeql.yml

@@ -1,17 +1,17 @@
 #@ load("@ytt:template", "template")
-#@ load("common.lib.yml", "configuration", "nugetPackages", "checkoutCode", "setupVcpkg")
+#@ load("common.lib.yml", "configuration", "nugetPackages", "checkoutCode", "setupVcpkg", "actionCodeQLInit", "actionCodeQLAnalyze", "actionSetupMSBuild")
 
 #@ def runCodeQL(language, buildSteps = []):
   - name: Initialize CodeQL
-    uses: github/codeql-action/init@v1
+    uses: #@ actionCodeQLInit
     with:
       languages: #@ language
       queries: security-and-quality
   #@ for step in buildSteps:
   - #@ template.replace(step)
   #@ end
   - name: Perform CodeQL Analysis
-    uses: github/codeql-action/analyze@v1
+    uses: #@ actionCodeQLAnalyze
 #@ end
 
 #@ def buildWrappers():
@@ -21,7 +21,7 @@
 
 #@ def buildPackages():
   - name: Add msbuild to PATH
-    uses: microsoft/setup-msbuild@v1.0.2
+    uses: #@ actionSetupMSBuild
 #@ for pkgName in nugetPackages:
   - name: #@ "Build " + pkgName
     run: #@ "msbuild Realm/" + pkgName + " -p:UseSharedCompilation=false -restore -p:Configuration=" + configuration

.github/templates/common.lib.yml

@@ -1,9 +1,28 @@
 #@ configuration = "Release"
 #@ nugetPackages = [ 'Realm.Fody', 'Realm', 'Realm.UnityUtils', 'Realm.UnityWeaver' ]
+#@ actionCheckout = "actions/checkout@v2"
+#@ actionCache = "actions/cache@v2"
+#@ actionUploadArtifact = "actions/upload-artifact@v2"
+#@ actionDownloadArtifact = "actions/download-artifact@v2"
+#@ actionSetupMSBuild = "microsoft/setup-msbuild@v1.0.2"
+#@ actionSetupDotnet = "actions/setup-dotnet@v1"
+#@ actionRuniOSSimulator = "realm/ci-actions/run-ios-simulator@v1"
+#@ actionCodeQLInit = "github/codeql-action/init@v1"
+#@ actionCodeQLAnalyze = "github/codeql-action/analyze@v1"
+#@ actionSetupNode = "actions/setup-node@v2"
+#@ actionAWSConfigureCredentials = "aws-actions/configure-aws-credentials@v1"
+
+#@ actionDockerLayerCaching = "satackey/action-docker-layer-caching@cc3f3828e75cbb45f0cf5139b95329c88480aa97" #! 0.0.11
+#@ actionDockerBuild = "docker/build-push-action@375f72aff65b645c51f122f04a776df8416b222b" #! 2.7.0
+#@ actionDockerRun = "addnab/docker-run-action@3e77f186b7a929ef010f183a9e24c0f9955ea609" #! v3
+#@ actionCoveralls = "coverallsapp/github-action@9ba913c152ae4be1327bfb9085dc806cedb44057" #! v1.1.3
+#@ actionDeleteArtifact = "geekyeggo/delete-artifact@56e063d7d8bf9972ac54aca4454d3a6675917f44" #! v1
+#@ actionGithubRelease = "ncipollo/release-action@a4f828a4e5777f12a55454900a18d3e60d8803fd" #! v1
+#@ actionDownloadAllArtifacts = "dawidd6/action-download-artifact@d0f291cf39bd21965ea9c4c6e210fc355c3844ed" #! v2.14.1
 
 #@ def checkoutCode(submodules=False, registerProblemMatchers=True):
   - name: Checkout code
-    uses: actions/checkout@v2
+    uses: #@ actionCheckout
     with:
       submodules: #@ submodules
       ref: ${{ github.event.pull_request.head.sha }}
@@ -18,7 +37,7 @@
 #@ def setupVcpkg(ifCondition = "true"):
   - name: Check Vcpkg cache
     id: check-vcpkg-cache
-    uses: actions/cache@v2
+    uses: #@ actionCache
     with:
       path: 'C:\src'
       key: vcpkg

.github/templates/main.yml

@@ -1,47 +1,53 @@
 #@ load("@ytt:template", "template")
 #@ load("@ytt:overlay", "overlay")
-#@ load("common.lib.yml", "configuration", "nugetPackages", "checkoutCode", "setupVcpkg")
+#@ load("common.lib.yml", "configuration", "nugetPackages", "checkoutCode", "setupVcpkg", "actionCache", "actionUploadArtifact", "actionDownloadArtifact", "actionSetupMSBuild", "actionSetupDotnet", "actionRuniOSSimulator", "actionDockerLayerCaching", "actionDockerBuild", "actionDockerRun", "actionCoveralls", "actionDeleteArtifact")
 
 #@ androidABIs = [ 'armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64' ]
 #@ windowsArchs = [ 'Win32', 'x64' ]
 #@ windowsUWPArchs = [ 'Win32', 'x64', 'ARM' ]
 #@ wrappersCacheCondition = "steps.check-cache.outputs.cache-hit != 'true'"
 #@ dotnetRuntime = "${{ (runner.os == 'macOS' && 'osx-x64') || (runner.os == 'Windows' && 'win-x64') || (runner.os == 'Linux' && 'linux-x64') || '???' }}"
 
-#@ def checkCache(key):
+#@ def getWrapperBuildCommand(cmd):
+#@ configurationParam =  " --configuration=" + configuration
+#@ ltoParam =  " -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${{ github.event_name != 'pull_request' && 'ON' || 'OFF' }}"
+#@ if cmd.startswith("powershell"):
+#@   configurationParam = " -Configuration " + configuration
+#@   ltoParam =  "${{ github.event_name != 'pull_request' && ' -EnableLTO' || '' }}"
+#@ end
+#@ return cmd + configurationParam + ltoParam
+#@ end
+
+#@ def checkCache(outputVar):
+#@ key = outputVar + "-" + configuration + "-${{ github.event_name != 'pull_request' && 'ON' || 'OFF' }}-${{hashFiles('./wrappers/**')}}"
 name: Check cache
 id: check-cache
-uses: actions/cache@v2
+uses: #@ actionCache
 with:
   path: ./wrappers/build/**
   key: #@ key
 #@ end
 
-#@ def buildWrappers(cmd, outputVar, intermediateSteps = []):
-#@ configurationParam =  " --configuration=" + configuration
-#@ ltoParam =  " -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${{ github.event_name != 'pull_request' && 'ON' || 'OFF' }}"
-#@ if cmd.startswith("powershell"):
-#@   configurationParam = " -Configuration " + configuration
-#@   ltoParam =  "${{ github.event_name != 'pull_request' && ' -EnableLTO' || '' }}"
+#@ def storeWrapperBinaries(outputVar):
+name: Store artifacts
+uses: #@ actionUploadArtifact
+with:
+  name: #@ outputVar
+  path: wrappers/build/**
+  retention-days: 1
 #@ end
-#@ cacheKey = outputVar + "-" + configuration + "-${{ github.event_name != 'pull_request' && 'ON' || 'OFF' }}-${{hashFiles('./wrappers/**')}}"
-#@ actualCommand = cmd + configurationParam + ltoParam
 
+#@ def buildWrappers(cmd, outputVar, intermediateSteps = []):
 steps:
   - #@ template.replace(checkoutCode("recursive"))
-  - #@ checkCache(cacheKey)
+  - #@ checkCache(outputVar)
   #@ for step in intermediateSteps:
   - #@ template.replace(step)
   #@ end
   - name: Build wrappers
-    run: #@ actualCommand
+    run: #@ getWrapperBuildCommand(cmd)
     if: #@ wrappersCacheCondition
-  - name: Store artifacts
-    uses: actions/upload-artifact@v2
-    with:
-      name: #@ outputVar
-      path: wrappers/build/**
-      retention-days: 1
+  - #@ storeWrapperBinaries(outputVar)
 #@ end
 
 #@ def setupWin81SDK():
@@ -62,7 +68,7 @@ steps:
 
 #@ def msbuildOnWin(projectPath, **properties):
   - name: Add msbuild to PATH
-    uses: microsoft/setup-msbuild@v1.0.2
+    uses: #@ actionSetupMSBuild
   - #@ msbuild(projectPath, RestoreConfigFile="Tests/Test.NuGet.Config", UseRealmNupkgsWithVersion="${{ needs.build-packages.outputs.package_version }}", **properties)
 #@ end
 
@@ -71,7 +77,7 @@ steps:
   - name: #@ "Build Unity"
     run: #@ "dotnet run --project Tools/SetupUnityPackage/ -- realm --packages-path Realm/packages --pack"
   - name: Store Unity artifacts
-    uses: actions/upload-artifact@v2
+    uses: #@ actionUploadArtifact
     with:
       name: #@ finalPkgName
       path: #@ "${{ github.workspace }}/Realm/Realm.Unity/" + finalPkgName
@@ -84,7 +90,7 @@ steps:
   - name: Check Docfx cache
     id: check-docfx-cache
     if: #@ docsCondition
-    uses: actions/cache@v2
+    uses: #@ actionCache
     with:
       path: 'C:\docfx'
       key: docfx
@@ -101,7 +107,7 @@ steps:
       Compress-Archive -Path Docs/_site -DestinationPath "Realm/packages/Docs.zip"
   - name: Store docs artifacts
     if: #@ docsCondition
-    uses: actions/upload-artifact@v2
+    uses: #@ actionUploadArtifact
     with:
       name: Docs.zip
       path: ${{ github.workspace }}/Realm/packages/Docs.zip
@@ -112,7 +118,7 @@ steps:
 #@ for pkgName in [ 'Realm.Fody', 'Realm' ]:
 #@ finalPkgName = pkgName + ".${{ steps.find-nupkg-version.outputs.package_version }}"
   - name: #@ "Store artifacts for " + pkgName
-    uses: actions/upload-artifact@v2
+    uses: #@ actionUploadArtifact
     with:
       name: #@ finalPkgName
       path: #@ "${{ github.workspace }}/Realm/packages/" + finalPkgName + ".*nupkg"
@@ -123,7 +129,7 @@ steps:
 #@ def fetchPackageArtifacts():
 #@ for pkg in [ "Realm", "Realm.Fody" ]:
   - name: #@ "Fetch " + pkg
-    uses: actions/download-artifact@v2
+    uses: #@ actionDownloadArtifact
     with:
       name: #@ pkg + ".${{ needs.build-packages.outputs.package_version }}"
       path: ${{ github.workspace }}/Realm/packages/
@@ -148,7 +154,7 @@ steps:
 #@ def fetchWrapperBinaries():
 #@ for platform in getWrapperBinaryNames():
   - name: #@ "Fetch artifacts for " + platform
-    uses: actions/download-artifact@v2
+    uses: #@ actionDownloadArtifact
     with:
       name: #@ "wrappers-" + platform
       path: wrappers/build
@@ -240,9 +246,30 @@ jobs:
     name: Wrappers iOS
     _: #@ template.replace(buildWrappers("./wrappers/build-ios.sh", "wrappers-ios"))
   build-wrappers-linux:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: Wrappers Linux
-    _: #@ template.replace(buildWrappers("./wrappers/build.sh", "wrappers-linux"))
+    steps:
+      - #@ template.replace(checkoutCode("recursive"))
+      - #@ checkCache("wrappers-linux")
+      - uses: #@ actionDockerLayerCaching
+        continue-on-error: true
+        if: #@ wrappersCacheCondition
+      - name: Build CentOS image
+        uses: #@ actionDockerBuild
+        with:
+          tags: wrappers-centos:latest
+          file: ./wrappers/centos.Dockerfile
+          push: false
+        if: #@ wrappersCacheCondition
+      - name: Build wrappers
+        uses: #@ actionDockerRun
+        with:
+          image: wrappers-centos:latest
+          shell: bash
+          options: -v ${{ github.workspace }}:/work
+          run: #@ getWrapperBuildCommand("/work/wrappers/build.sh")
+        if: #@ wrappersCacheCondition
+      - #@ storeWrapperBinaries("wrappers-linux")
   build-wrappers-android:
     runs-on: ubuntu-20.04
     name: Wrappers Android
@@ -278,7 +305,7 @@ jobs:
       package_version: ${{ steps.find-nupkg-version.outputs.package_version }}
     steps:
       - name: Add msbuild to PATH
-        uses: microsoft/setup-msbuild@v1.0.2
+        uses: #@ actionSetupMSBuild
       - #@ template.replace(checkoutCode())
       - name: Set version suffix
         id: set-version-suffix
@@ -367,7 +394,7 @@ jobs:
     steps:
       - #@ template.replace(checkoutCode())
       - #@ template.replace(fetchPackageArtifacts())
-      - uses: actions/setup-dotnet@v1
+      - uses: #@ actionSetupDotnet
         if: matrix.targetFramework == 'net6.0'
         with:
           dotnet-version: '6.0.x'
@@ -394,7 +421,7 @@ jobs:
       - #@ template.replace(fetchPackageArtifacts())
       - #@ msbuild("Tests/Tests.iOS", TargetFrameworkVersion="v1.0", Platform="iPhoneSimulator", RestoreConfigFile="Tests/Test.NuGet.Config", UseRealmNupkgsWithVersion="${{ needs.build-packages.outputs.package_version }}")
       - name: Run the tests
-        uses: realm/ci-actions/run-ios-simulator@v1
+        uses: #@ actionRuniOSSimulator
         with:
          appPath: 'Tests/Tests.iOS/bin/iPhoneSimulator/Release/Tests.iOS.app'
          bundleId: 'io.realm.dotnettests'
@@ -417,7 +444,7 @@ jobs:
     steps:
       - #@ template.replace(checkoutCode())
       - #@ template.replace(fetchWrapperBinaries())
-      - uses: actions/setup-dotnet@v1
+      - uses: #@ actionSetupDotnet
         with:
           dotnet-version: '5.0.x'
       - name: Setup Coverlet & Report Generator
@@ -428,7 +455,7 @@ jobs:
       - #@ template.replace(dotnetPublishAndRunCoverage("net5.0"))
       - name: Publish Coverage
         id: publish-coveralls
-        uses: coverallsapp/github-action@9ba913c152ae4be1327bfb9085dc806cedb44057 #! v1.1.3
+        uses: #@ actionCoveralls
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           path-to-lcov: ./report.lcov
@@ -443,7 +470,7 @@ jobs:
     steps:
 #@ for platform in getWrapperBinaryNames():
       - name: #@ "Delete artifacts for " + platform
-        uses: geekyeggo/delete-artifact@56e063d7d8bf9972ac54aca4454d3a6675917f44 #! v1
+        uses: #@ actionDeleteArtifact
         with:
           name: #@ "wrappers-" + platform
 #@ end
@@ -470,7 +497,7 @@ jobs:
           dashboard-path: dashboard.charts
           nuget-package: ${{ github.workspace }}/Realm/packages/Realm.${{ needs.build-packages.outputs.package_version }}.nupkg
       - name: Upload Charts Dashboard File
-        uses: actions/upload-artifact@v2
+        uses: #@ actionUploadArtifact
         with:
           name: dashboard.charts
           path: ${{ github.workspace }}/dashboard.charts

.github/templates/publish-release.yml

@@ -1,4 +1,5 @@
 #@ load("@ytt:template", "template")
+#@ load("common.lib.yml", "checkoutCode", "actionDownloadAllArtifacts", "actionSetupNode", "actionAWSConfigureCredentials", "actionGithubRelease")
 
 #@ def uploadToNuget(packageName, includeSymbols = False):
   - name: #@ "NuGet Publish " + packageName + ".${{ steps.get-version.outputs.version }}"
@@ -11,7 +12,7 @@
 #@ end
 
 #@ def uploadToNpm(packageName):
-  - uses: actions/setup-node@v2
+  - uses: #@ actionSetupNode
     with:
       node-version: '12.x'
       registry-url: 'https://registry.npmjs.org'
@@ -23,7 +24,7 @@
 
 #@ def publishGithubRelease():
   - name: Create Github Release
-    uses: ncipollo/release-action@v1
+    uses: #@ actionGithubRelease
     with:
       artifacts: ${{ github.workspace }}/Realm/packages/io.realm.unity-${{ steps.get-version.outputs.version }}.tgz/io.realm.unity-${{ steps.get-version.outputs.version }}.tgz
       bodyFile: ${{ steps.extract-release-notes.outputs.release-notes-path }}
@@ -36,7 +37,7 @@
 
 #@ def uploadDocsToS3():
   - name: Configure AWS Credentials
-    uses: aws-actions/configure-aws-credentials@v1
+    uses: #@ actionAWSConfigureCredentials
     with:
       aws-access-key-id: ${{ secrets.DOCS_S3_ACCESS_KEY }}
       aws-secret-access-key: ${{ secrets.DOCS_S3_SECRET_KEY }}
@@ -60,12 +61,9 @@ jobs:
     runs-on: windows-latest
     environment: Production
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
-      with:
-        submodules: false
+    - #@ template.replace(checkoutCode(False, False))
     - name: Download all artifacts
-      uses: dawidd6/action-download-artifact@v2
+      uses: #@ actionDownloadAllArtifacts
       with:
         workflow: main.yml
         commit: ${{ github.sha }}

.github/workflows/main.yml

@@ -65,7 +65,7 @@ jobs:
         path: wrappers/build/**
         retention-days: 1
   build-wrappers-linux:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: Wrappers Linux
     steps:
     - name: Checkout code
@@ -83,8 +83,23 @@ jobs:
       with:
         path: ./wrappers/build/**
         key: wrappers-linux-Release-${{ github.event_name != 'pull_request' && 'ON' || 'OFF' }}-${{hashFiles('./wrappers/**')}}
+    - uses: satackey/action-docker-layer-caching@cc3f3828e75cbb45f0cf5139b95329c88480aa97
+      continue-on-error: true
+      if: steps.check-cache.outputs.cache-hit != 'true'
+    - name: Build CentOS image
+      uses: docker/build-push-action@375f72aff65b645c51f122f04a776df8416b222b
+      with:
+        tags: wrappers-centos:latest
+        file: ./wrappers/centos.Dockerfile
+        push: false
+      if: steps.check-cache.outputs.cache-hit != 'true'
     - name: Build wrappers
-      run: ./wrappers/build.sh --configuration=Release -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${{ github.event_name != 'pull_request' && 'ON' || 'OFF' }}
+      uses: addnab/docker-run-action@3e77f186b7a929ef010f183a9e24c0f9955ea609
+      with:
+        image: wrappers-centos:latest
+        shell: bash
+        options: -v ${{ github.workspace }}:/work
+        run: /work/wrappers/build.sh --configuration=Release -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${{ github.event_name != 'pull_request' && 'ON' || 'OFF' }}
       if: steps.check-cache.outputs.cache-hit != 'true'
     - name: Store artifacts
       uses: actions/upload-artifact@v2