[CI] Add AWS EC2 dynamic runner support (#6471)

This adds infrastructure to spawn AWS EC2 runners dynamically for lts suite testing. This will be only functional if you will add "aws-type" keys as well as other keys into devops/test_configs.json configuration file like this:

 {
      "config": "hip_amdgpu",
      "name": "HIP AMDGPU LLVM Test Suite",
      "runs-on": "aws-amdgpu_${{ inputs.uniq }}",
      "aws-ami": "ami-0ccda708841dde988",
      "aws-type": [ "g4ad.2xlarge", "g4ad.4xlarge" ],
      "aws-spot": false,
      "aws-disk": "/dev/xvda:64",
      "image": "${{ inputs.amdgpu_image }}",
      "container_options": "--device=/dev/dri --device=/dev/kfd",
      "check_sycl_all": "hip:gpu,host",
      "cmake_args": "-DHIP_PLATFORM=\"AMD\" -DAMD_ARCH=\"gfx1031\""
    },
    {
      "config": "cuda",
      "name": "CUDA LLVM Test Suite",
      "runs-on": "aws-cuda_${{ inputs.uniq }}",
      "aws-ami": "ami-02ec0f344128253f9",
      "aws-type": [ "g4dn.2xlarge", "g4dn.4xlarge" ],
      "aws-disk": "/dev/xvda:64",
      "image": "${{ inputs.cuda_image }}",
      "container_options": "--gpus all",
      "check_sycl_all": "cuda:gpu,host",
      "cmake_args": ""
    }

Also please make sure that other non AMD/nVidia GPU jobs do not have too generic self-hosted runner labels like "Linux", "x64" since otherwise they can go to these AWS hosts and we do not want to use them for generic workloads.

Intel provided AWS account is supposed to be used. To configure it for this repo please do the following (I will keep this BKM schematic to avoid disclosing any sensitive info):

1 Login to AWS Intel account as admin
2 To go IAM users (https://us-east-1.console.aws.amazon.com/iamv2/home?region=us-east-1#/users)
3 Click "Add users"
4 Select "Access key - Programmatic access"
5 Copy permissions from existing user (sycl-ci)
6 Get new user AWS key and secret key strings (keep them private until step 11).
7 Delete original user sycl-ci (so I can no longer use this AWS account for apstasen/llvm repo for test purposes)
8 Got to https://github.com/intel/llvm/settings/secrets/actions
9 Create "aws" environment and make sure you select required reviewers for extra security (they need to pay special attention that PRs do not expose secrets by making changes workflow .yml and devops .js files)
10 Create AWS_ACCESS_KEY and AWS_SECRET_KEY secrets using obtained new AWS AMI user key strings.
11 Destroy all copies of AWS key and secret key strings (except ones stored as github "aws" environment secrets)
12 Create repository (or even better put them into "aws" environment too for better security) secret GH_PERSONAL_ACCESS_TOKEN (with Github api key with "repo" permissions)

Author: apstasen

.github/workflows/sycl_linux_build_and_test.yml

@@ -40,6 +40,10 @@ on:
         type: string
         required: false
         default: ""
+      lts_aws_matrix:
+        type: string
+        required: false
+        default: ""
       lts_cmake_extra_args:
         type: string
         required: false
@@ -155,9 +159,31 @@ jobs:
         name: sycl_lit_${{ inputs.build_artifact_suffix }}
         path: lit.tar.xz
 
-  llvm_test_suite:
+  aws-start:
+    name: Start AWS
     needs: build
-    if: ${{ inputs.lts_matrix != '' }}
+    if: ${{ inputs.lts_aws_matrix != '' }}
+    runs-on: ubuntu-latest
+    environment: aws
+    steps:
+      - name: Setup script
+        run: |
+          mkdir -p ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/action.yml   -P ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/aws-ec2.js   -P ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/package.json -P ./aws-ec2
+          npm install ./aws-ec2
+      - name: Start AWS EC2 runners
+        uses: ./aws-ec2
+        with:
+          runs-on-list: ${{ inputs.lts_aws_matrix }}
+          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
+
+  llvm_test_suite:
+    needs: [build, aws-start]
+    if: ${{ !failure() && inputs.lts_matrix != '' }}
     strategy:
       fail-fast: false
       max-parallel: ${{ inputs.max_parallel }}
@@ -203,3 +229,26 @@ jobs:
         check_sycl_all: ${{ matrix.check_sycl_all }}
         results_name_suffix: ${{ matrix.config }}_${{ inputs.build_artifact_suffix }}
         cmake_args: '${{ matrix.cmake_args }} ${{ inputs.lts_cmake_extra_args }}'
+
+  aws-stop:
+    name: Stop AWS
+    needs: [ aws-start, llvm_test_suite ]
+    if: ${{ always() && inputs.lts_ats_matrix != '' }}
+    runs-on: ubuntu-latest
+    environment: aws
+    steps:
+      - name: Setup script
+        run: |
+          mkdir -p ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/action.yml   -P ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/aws-ec2.js   -P ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/package.json -P ./aws-ec2
+          npm install ./aws-ec2
+      - name: Stop AWS EC2 runners
+        uses: ./aws-ec2
+        with:
+          runs-on-list: ${{ inputs.lts_aws_matrix }}
+          mode: stop
+          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}

.github/workflows/sycl_nightly.yml

@@ -20,6 +20,7 @@ jobs:
     if: github.repository == 'intel/llvm'
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
     needs: resolve_matrix
+    secrets: inherit
     with:
       build_cache_root: "/__w/"
       build_artifact_suffix: default
@@ -29,6 +30,7 @@ jobs:
     if: github.repository == 'intel/llvm'
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
     needs: resolve_matrix
+    secrets: inherit
     with:
       build_cache_root: "/__w/"
       build_cache_suffix: opaque_pointers

.github/workflows/sycl_post_commit.yml

@@ -21,13 +21,16 @@ jobs:
     name: Linux Default
     needs: resolve_matrix
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
+    secrets: inherit
     with:
       build_cache_root: "/__w/llvm"
       build_artifact_suffix: "post_commit"
       lts_matrix: ${{ needs.resolve_matrix.outputs.lts_matrix }}
+      lts_aws_matrix: ${{ needs.resolve_matrix.outputs.lts_aws_matrix }}
   linux_no_assert:
     name: Linux (no assert)
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
+    secrets: inherit
     with:
       build_cache_root: "/__w/llvm"
       build_cache_suffix: gcc_no_assertions

.github/workflows/sycl_precommit.yml

@@ -1,7 +1,7 @@
 name: SYCL
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
     - sycl
     # Do not run builds if changes are only in the following locations
@@ -25,6 +25,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
       with:
+        persist-credentials: false
         fetch-depth: 2
     - name: Run clang-format
       uses: ./devops/actions/clang-format
@@ -43,9 +44,11 @@ jobs:
     needs: [lint, resolve_matrix]
     if: always() && (success() || contains(github.event.pull_request.labels.*.name, 'ignore-lint'))
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
+    secrets: inherit
     with:
       build_cache_root: "/__w/"
       build_cache_size: "8G"
       build_artifact_suffix: "default"
       build_cache_suffix: "default"
       lts_matrix: ${{ needs.resolve_matrix.outputs.lts_matrix }}
+      lts_aws_matrix: ${{ needs.resolve_matrix.outputs.lts_aws_matrix }}

.github/workflows/sycl_resolve_test_matrix.yml

@@ -19,10 +19,18 @@ on:
         type: string
         required: true
         default: ""
+      uniq:
+        description: Unique string to name dynamic runners in AWS
+        type: string
+        required: false
+        default: ${{ github.run_id }}-${{ github.run_attempt }}
     outputs:
       lts_matrix:
         description: "Generated Matrix"
         value: ${{ jobs.resolve_matrix.outputs.lts_matrix }}
+      lts_aws_matrix:
+        description: "Generated Matrix AWS subset"
+        value: ${{ jobs.resolve_matrix.outputs.lts_aws_matrix }}
 jobs:
   resolve_matrix:
     name: Resolve Test Matrix

devops/actions/aws-ec2/action.yml

@@ -0,0 +1,66 @@
+name: aws-ec2
+description: Start AWS EC2 instances with Github actions runner agent in it
+
+inputs:
+  runs-on-list:
+    description: "JSON string with array of objects with aws-type, runs-on, aws-ami, aws-spot, aws-disk, aws-timebomb, one-job properties"
+    required: true
+    # aws-type:     AWS EC2 instance type. This property must be present if you want to trigger AWS EC2 instance start/stop.
+    # runs-on:      Name of the unique label assigned to the runner used as 'runs-on' property for the following jobs. Mandatory presence required.
+    # aws-ami:      AWS AMI id. Makes sense only for start mode. Default "ami-0966bccbb521ccb24".
+
+    #   ami-0966bccbb521ccb24: Ubuntu 22.04 (ami-02f3416038bdb17fb with /dev/sda1 disk) with docker installed and gh_runner (1001) like this:
+    #     sudo -s
+    #     apt-get update
+    #     curl -fsSL https://get.docker.com -o /tmp/get-docker.sh
+    #     sh /tmp/get-docker.sh
+    #     groupadd -g 1001 gh_runner; useradd gh_runner -u 1001 -g 1001 -m -s /bin/bash; usermod -aG docker gh_runner; usermod -aG video gh_runner
+    #     sync; shutdown -h now
+
+    #   ami-02ec0f344128253f9: Amazon Linux 2 AMI with NVIDIA TESLA GPU Driver (ami-06bf0a3f89fe08f0a with /dev/xvda disk) with docker installed and gh_runner (1001) like this:
+    #     sudo -s
+    #     yum update -y
+    #     amazon-linux-extras install docker
+    #     sudo systemctl --now enable docker
+    #     distribution=$(. /etc/os-release;echo $ID$VERSION_ID) && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.repo | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo
+    #     yum-config-manager --disable amzn2-graphics; yum clean expire-cache; yum install -y nvidia-docker2; systemctl restart docker
+    #     groupadd -g 1001 gh_runner; useradd gh_runner -u 1001 -g 1001 -m -s /bin/bash; usermod -aG docker gh_runner; usermod -aG video gh_runner
+    #     sync; shutdown -h now
+
+    #   ami-0ccda708841dde988: Amazon Linux 2 AMI with AMD Radeon Pro Driver (ami-0bb1072e787242eb6 with /dev/xvda disk) with docker installed and gh_runner (1001) like this:
+    #     sudo -s
+    #     amazon-linux-extras install docker
+    #     sudo systemctl --now enable docker
+    #     groupadd -g 1001 gh_runner; useradd gh_runner -u 1001 -g 1001 -m -s /bin/bash; usermod -aG docker gh_runner; usermod -aG video gh_runner
+    #     sync; shutdown -h now
+
+    # aws-spot:     Enable usage of spot instances to save money (less reliable). Makes sense only for start mode. Default true.
+    # aws-disk:     AWS EC2 instance AMI specific disk device path and size in GB (8 by default). Makes sense only for start mode. Default "/dev/sda1:16".
+    # aws-timebomp: AWS EC2 instance maximum live time. Makes sense only for start mode. Default "1h".
+    # one-job:      Will terminate AWS EC2 instance after one job (not waiting for stop job) saving money. Makes sense only for start mode. Default true.
+
+  mode:
+    description: "Mode of operation: start or stop"
+    required: false
+    default: start
+
+  GH_PERSONAL_ACCESS_TOKEN:
+    description: "Github personal access token with repo permission"
+    required: true
+
+  AWS_ACCESS_KEY:
+    description: "AWS access id"
+    required: true
+
+  AWS_SECRET_KEY:
+    description: "AWS access secret key"
+    required: true
+
+  aws-region:
+    description: "AWS EC2 region"
+    required: false
+    default: "us-east-2" # Ohio
+
+runs:
+  using: node12
+  main: ./aws-ec2.js