ci: ACN PR Pipeline Security Feature Branch (#2985)

* ci: Add Main Pipeline Template

* chore: azure-cni-overlay work

* chore: cilium overlay work

* test: cilium overlay E2E

* chore: cni overlay ds work

* chore: cilium ds work

* fixes

* ci: Add ACN Trigger

* ci: Use Git Ref Under Review for Testing

* Use Duplicate Files for Feature Test

* Disable Pipeline Trigger

---------

Co-authored-by: Sheyla Trudo <[email protected]>
Co-authored-by: jpayne3506 <[email protected]>

Author: 3 people

.config/.gdnsuppress

@@ -0,0 +1,26 @@
+{
+  "version": "latest",
+  "suppressionSets": {
+    "default": {
+      "name": "default",
+      "createdDate": "2022-11-28 20:04:38Z",
+      "lastUpdatedDate": "2022-11-28 20:04:38Z"
+    }
+  },
+  "results": {
+    "d7e55b5f3e54f9253a2fec595f97520ab0ffece607981d2db0fcfe4dae4cd490": {
+      "signature": "d7e55b5f3e54f9253a2fec595f97520ab0ffece607981d2db0fcfe4dae4cd490",
+      "alternativeSignatures": [],
+      "target": "**/testdata/dummy.pem",
+      "memberOf": [
+        "default"
+      ],
+      "tool": "credscan",
+      "ruleId": "CSCAN-GENERAL0020",
+      "justification": null,
+      "createdDate": "2022-11-28 20:04:38Z",
+      "expirationDate": null,
+      "type": null
+    }
+  }
+}

.config/credScanSuppressions.json

@@ -0,0 +1,4 @@
+{
+ "tool": "Credential Scanner",
+ "suppressions": []
+}

.pipelines/cni/k8s-e2e/k8s-e2e.jobs.yaml

@@ -0,0 +1,165 @@
+parameters:
+  clusterName: ""
+  os: ""
+  dependsOn: ""
+  sub: ""
+  cni: cni
+
+
+jobs:
+  - job: CNI_${{ parameters.os }}
+    condition: and( not(canceled()), not(failed()) )
+    displayName: CNI k8s E2E ${{ parameters.os }}
+    dependsOn: ${{ parameters.dependsOn }}
+    pool:
+      isCustom: true
+      type: linux
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    variables:
+      ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output
+      ob_git_checkout: true
+    steps:
+      - checkout: ACNReviewChanges
+        clean: true
+
+      - task: AzureCLI@2
+        inputs:
+          azureSubscription: ${{ parameters.sub }}
+          scriptLocation: "inlineScript"
+          scriptType: "bash"
+          workingDirectory: $(ACN_DIR)
+          addSpnToEnvironment: true
+          inlineScript: |
+            set -e
+            make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+
+            # sig-release provides test suite tarball(s) per k8s release. Just need to provide k8s version "v1.xx.xx"
+            # pulling k8s version from AKS.
+            eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }} -n ${{ parameters.clusterName }} --query "currentKubernetesVersion")
+            echo $k8sVersion
+            curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz
+
+            # https://github.com/kubernetes/sig-release/blob/master/release-engineering/artifacts.md#content-of-kubernetes-test-system-archtargz-on-example-of-kubernetes-test-linux-amd64targz-directories-removed-from-list
+            # explictly unzip and strip directories from ginkgo and e2e.test
+            tar -xvzf kubernetes-test-linux-amd64.tar.gz --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
+        displayName: "Setup Environment"
+        retryCountOnTaskFailure: 5
+
+      - ${{ if contains(parameters.os, 'windows') }}:
+        - script: |
+            set -e
+            kubectl apply -f test/integration/manifests/load/privileged-daemonset-windows.yaml
+            kubectl rollout status -n kube-system ds privileged-daemonset
+
+            kubectl get pod -n kube-system -l app=privileged-daemonset,os=windows -owide
+            pods=`kubectl get pod -n kube-system -l app=privileged-daemonset,os=windows --no-headers | awk '{print $1}'`
+            for pod in $pods; do
+              kubectl exec -i -n kube-system $pod -- powershell "Restart-Service kubeproxy"
+              kubectl exec -i -n kube-system $pod -- powershell "Get-Service kubeproxy"
+            done
+          workingDirectory: $(ACN_DIR)
+          name: kubeproxy
+          displayName: Restart Kubeproxy on Windows nodes
+          retryCountOnTaskFailure: 3
+
+      - ${{ if eq(parameters.datapath, true) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: Datapath
+            name: datapath
+            ginkgoFocus: '(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api'
+            ginkgoSkip: 'SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6'
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 10
+
+      - ${{ if eq(parameters.dns, true) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: DNS
+            name: dns
+            ginkgoFocus: '\[sig-network\].DNS.should'
+            ginkgoSkip: 'resolv|256 search'
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+
+      - ${{ if eq(parameters.portforward, true) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: Kubectl Portforward
+            name: portforward
+            ginkgoFocus: '\[sig-cli\].Kubectl.Port'
+            ginkgoSkip: ''
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+
+      - ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cni') ) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: Service Conformance
+            name: service
+            ginkgoFocus: 'Services.*\[Conformance\].*'
+            ginkgoSkip: ''
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+
+      - ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cilium') ) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: Service Conformance|Cilium
+            name: service
+            ginkgoFocus: 'Services.*\[Conformance\].*'
+            ginkgoSkip: 'should serve endpoints on same port and different protocols' # Cilium does not support this feature. For more info on test: https://github.com/kubernetes/kubernetes/blame/e602e9e03cd744c23dde9fee09396812dd7bdd93/test/conformance/testdata/conformance.yaml#L1780-L1788
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+
+      - ${{ if eq(parameters.hostport, true) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: Host Port
+            name: hostport
+            ginkgoFocus: '\[sig-network\](.*)HostPort|\[sig-scheduling\](.*)hostPort'
+            ginkgoSkip: 'SCTP|exists conflict' # Skip slow 5 minute test
+            os: ${{ parameters.os }}
+            processes: 1 # Has a short serial test
+            attempts: 3
+
+      - ${{ if and(eq(parameters.hybridWin, true), eq(parameters.os, 'windows')) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: Hybrid Network
+            name: hybrid
+            ginkgoFocus: '\[sig-windows\].Hybrid'
+            ginkgoSkip: ''
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+
+      - ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: DualStack Test
+            name: DualStack
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[Feature:IPv6DualStack\]'
+            ginkgoSkip: 'SCTP|session affinity'
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+
+      - ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}:
+        - template: k8s-e2e.steps.yaml@ACNTools
+          parameters:
+            testName: DualStack Test|Cilium
+            name: DualStack
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[Feature:IPv6DualStack\]'
+            ginkgoSkip: 'SCTP|session affinity|should function for service endpoints using hostNetwork' # Cilium dualstack has a known issue with this test https://github.com/cilium/cilium/issues/25135
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+

.pipelines/cni/k8s-e2e/k8s-e2e.steps.yaml

@@ -0,0 +1,67 @@
+parameters:
+  testName: ""
+  name: ""
+  ginkgoFocus: ""
+  ginkgoSkip: ""
+  os: ""
+  processes: "" # Number of parallel processes
+  attempts: ""
+
+
+steps:
+  - script: |
+      set -ex
+
+      # ginkgoSkip cant handle only |LinuxOnly. Need to have check
+      if ${{ lower(and(ge(length(parameters.ginkgoSkip), 1), eq(parameters.os, 'windows'))) }}
+      then
+        SKIP="|LinuxOnly"
+      elif ${{ lower(eq(parameters.os, 'windows')) }}
+      then
+        SKIP="LinuxOnly"
+      fi
+
+      # Taint Linux nodes so that windows tests do not run on them
+      if ${{ lower(eq(parameters.os, 'windows')) }}
+      then
+        kubectl rollout status -n kube-system deployment/konnectivity-agent --timeout=3m
+        kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule
+      fi
+
+      # Taint Windows nodes so that Linux tests do not run on them
+      if ${{ lower(eq(parameters.os, 'linux')) }}
+      then
+        kubectl taint nodes -l kubernetes.azure.com/mode=user node-role.kubernetes.io/control-plane:NoSchedule
+      fi
+
+      # Depreciating flags. Change once k8s minimum version supported is > 1.24
+      # nodes -> procs
+      # flakeAttempts -> flake-attempts
+      # dryRun -> dry-run
+
+      ./ginkgo --nodes=${{ parameters.processes }} \
+      ./e2e.test -- \
+      --num-nodes=2 \
+      --provider=skeleton \
+      --ginkgo.focus='${{ parameters.ginkgoFocus }}' \
+      --ginkgo.skip="${{ parameters.ginkgoSkip }}$SKIP" \
+      --ginkgo.flakeAttempts=${{ parameters.attempts }} \
+      --ginkgo.v \
+      --node-os-distro=${{ parameters.os }} \
+      --kubeconfig=$HOME/.kube/config
+
+      # Untaint Linux nodes once testing is complete
+      if ${{ lower(eq(parameters.os, 'windows')) }}
+      then
+        kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule-
+      fi
+
+      # Untaint Windows nodes once testing is complete
+      if ${{ lower(eq(parameters.os, 'linux')) }}
+      then
+        kubectl taint nodes -l kubernetes.azure.com/mode=user node-role.kubernetes.io/control-plane:NoSchedule-
+      fi
+    name: ${{ parameters.name }}
+    displayName: k8s E2E - ${{ parameters.testName }}
+    workingDirectory: $(ACN_DIR)
+    retryCountOnTaskFailure: 5

.pipelines/containers/container-template.steps.yaml

@@ -0,0 +1,39 @@
+parameters:
+  arch: ""
+  name: ""
+  os: ""
+  os_version: ""
+
+steps:
+- task: AzureCLI@2
+  displayName: "Login"
+  inputs:
+    azureSubscription: $(ACR_ARM_SERVICE_CONNECTION)
+    scriptLocation: "inlineScript"
+    scriptType: "bash"
+    inlineScript: |
+      az acr login -n $(ACR)
+
+- script: |
+    set -e
+    if [ "$IN_OS" = 'windows' ]; then export BUILDX_ACTION='--push'; fi
+    make "$IMGNAME" OS="$IN_OS" ARCH="$IN_ARCH" OS_VERSION="$IN_OS_VERSION"
+  name: image_build
+  displayName: Image Build
+  workingDirectory: $(ACN_DIR)
+  retryCountOnTaskFailure: 3
+  env:
+    IMGNAME: '${{ parameters.name }}-image'
+    IN_OS: '${{ parameters.os }}'
+    IN_ARCH: '${{ parameters.arch }}'
+    IN_OS_VERSION: '${{ parameters.os_version }}'
+
+- task: AzureCLI@2
+  displayName: "Logout"
+  inputs:
+    azureSubscription: $(ACR_ARM_SERVICE_CONNECTION)
+    scriptLocation: "inlineScript"
+    scriptType: "bash"
+    inlineScript: |
+      docker logout
+

.pipelines/containers/manifest-template.steps.yaml

@@ -0,0 +1,54 @@
+parameters:
+  name: ""
+  platforms: ""
+  os_versions: ""
+
+steps:
+
+  - task: AzureCLI@2
+    displayName: "Login"
+    inputs:
+      azureSubscription: $(ACR_ARM_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      inlineScript: |
+        az acr login -n $(ACR)
+
+  - script: |
+      set -e
+      make ${{ parameters.name }}-manifest-build PLATFORMS="${{ parameters.platforms }}" OS_VERSIONS="${{ parameters.os_versions }}"
+    workingDirectory: $(ACN_DIR)
+    name: manifest_build
+    displayName: Manifest Build
+    retryCountOnTaskFailure: 3
+
+  - script: |
+      set -ex
+      echo "checking XDG_RUNTIME_DIR"
+      echo $XDG_RUNTIME_DIR
+      make ${{ parameters.name }}-manifest-push
+      mkdir -p $(Build.ArtifactStagingDirectory)/images
+
+      echo "setting XDG_RUNTIME_DIR"
+      export XDG_RUNTIME_DIR=/run/user/$(id -u)
+      echo $XDG_RUNTIME_DIR
+
+      make ${{ parameters.name }}-skopeo-archive IMAGE_ARCHIVE_DIR=$(Build.ArtifactStagingDirectory)/images
+    name: manifest_push
+    displayName: Manifest Push
+    workingDirectory: $(ACN_DIR)
+    retryCountOnTaskFailure: 3
+
+  - task: AzureCLI@2
+    displayName: "Logout"
+    inputs:
+      azureSubscription: $(ACR_ARM_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      inlineScript: |
+        docker logout
+
+  - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+    displayName: "Add SBOM Generator tool"
+    inputs:
+      BuildDropPath: "$(Build.ArtifactStagingDirectory)"