Skip to content

Commit 972b034

Browse files
suyadav1suyadav1
authored
Test automation framework: pull testkube images from MCR (#1382)
* Pull testkube from mcr * working helm chart * schedule executors on linux nodes * Added multiple cicd clusters configuration for testkube * skip testkube pods in the test * parameterized azure_pipeline_testframework.yaml * resolved comments * parameterized test automation pipeline
1 parent 64aa62a commit 972b034

File tree

8 files changed

+294
-118
lines changed

8 files changed

+294
-118
lines changed

‎.pipelines/azure_pipeline_testframework.yaml‎

Lines changed: 135 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ trigger: none
88
pr: none
99

1010
jobs:
11-
- deployment: Testkube
11+
- deployment: Testkube_Monitoring_Model_Cluster_WCUS
1212
environment: container-insights
1313
timeoutInMinutes: 120
14-
displayName: "Test: run testkube tests"
14+
displayName: "Monitoring-Model-Cluster-WCUS tests"
1515
pool:
1616
name: Azure-Pipelines-CI-Test-EO
1717
variables:
@@ -23,73 +23,155 @@ jobs:
2323
- checkout: self
2424
persistCredentials: true
2525

26-
- bash: |
27-
wget -qO - https://repo.testkube.io/key.pub | sudo apt-key add -
28-
echo "deb https://repo.testkube.io/linux linux main" | sudo tee -a /etc/apt/sources.list
29-
sudo apt-get update
30-
sudo apt-get install -y testkube
31-
workingDirectory: $(Build.SourcesDirectory)
32-
displayName: "Install testkube CLI"
33-
3426
- task: AzureCLI@1
3527
displayName: Get kubeconfig
3628
inputs:
3729
azureSubscription: 'ContainerInsights_Build_Subscription_CI'
3830
scriptLocation: 'inlineScript'
39-
inlineScript: 'az aks get-credentials -g $(RESOURCE_GROUP) -n $(CLUSTER_NAME)'
31+
inlineScript: 'az aks get-credentials -g monitoring-model-cluster-wcus -n Monitoring-Model-Cluster-WCUS'
4032

4133
- bash: |
42-
# Install TestKube
43-
helm repo add kubeshop https://kubeshop.github.io/helm-charts
44-
helm repo update
45-
helm upgrade --install --create-namespace testkube kubeshop/testkube -n testkube -f ./testkube/helm-testkube-values.yaml
34+
chmod +x ./install-and-execute-testkube-tests.sh
35+
./install-and-execute-testkube-tests.sh AzureClientId=$(MonitoringModelClusterWCUSClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
36+
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
37+
displayName: "Install and execute testkube tests"
4638
47-
# Install TestKube CRs
48-
export WEBHOOK_URI=$(WEBHOOK_URI)
49-
envsubst < ./testkube/testkube-teams-integration.yaml > ./testkube/testkube-teams-integration-$(CLUSTER_NAME).yaml
50-
kubectl apply -f ./testkube/testkube-teams-integration-$(CLUSTER_NAME).yaml
51-
kubectl apply -f ./testkube/api-server-permissions.yaml
52-
kubectl apply -f ./testkube/testkube-test-crs.yaml
53-
exit 0
54-
workingDirectory: $(Build.SourcesDirectory)/test/
55-
displayName: "Apply TestKube CRs and pod/service monitors"
39+
- deployment: Testkube_Monitoring_Model_Cluster_WEU
40+
environment: container-insights
41+
timeoutInMinutes: 120
42+
displayName: "Monitoring-Model-Cluster-WEU tests"
43+
pool:
44+
name: Azure-Pipelines-CI-Test-EO
45+
variables:
46+
skipComponentGovernanceDetection: true
47+
strategy:
48+
runOnce:
49+
deploy:
50+
steps:
51+
- checkout: self
52+
persistCredentials: true
5653

57-
- bash: |
58-
sleep 120
59-
displayName: "Wait for cluster to be ready"
54+
- task: AzureCLI@1
55+
displayName: Get kubeconfig
56+
inputs:
57+
azureSubscription: 'ContainerInsights_Build_Subscription_CI'
58+
scriptLocation: 'inlineScript'
59+
inlineScript: 'az aks get-credentials -g Monitoring-Model-Cluster-WEU -n Monitoring-Model-Cluster-WEU'
6060

6161
- bash: |
62-
# Run the full test suite
63-
kubectl testkube run testsuite e2e-tests-merge --verbose
62+
chmod +x ./install-and-execute-testkube-tests.sh
63+
./install-and-execute-testkube-tests.sh AzureClientId=$(MonitoringModelClusterWEUClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
64+
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
65+
displayName: "Install and execute testkube tests"
6466
65-
# Get the current id of the test suite now running
66-
execution_id=$(kubectl testkube get testsuiteexecutions --test-suite e2e-tests-merge --limit 1 | grep e2e-tests | awk '{print $1}')
67+
- deployment: Testkube_ci_logs_prod_aks_geneva_integration
68+
environment: container-insights
69+
timeoutInMinutes: 120
70+
displayName: "ci-logs-prod-aks-geneva-integration tests"
71+
pool:
72+
name: Azure-Pipelines-CI-Test-EO
73+
variables:
74+
skipComponentGovernanceDetection: true
75+
strategy:
76+
runOnce:
77+
deploy:
78+
steps:
79+
- checkout: self
80+
persistCredentials: true
6781

68-
# Watch until the all the tests in the test suite finish
69-
kubectl testkube watch testsuiteexecution $execution_id
82+
- task: AzureCLI@1
83+
displayName: Get kubeconfig
84+
inputs:
85+
azureSubscription: 'ContainerInsights_Build_Subscription_CI'
86+
scriptLocation: 'inlineScript'
87+
inlineScript: 'az aks get-credentials -g ci-logs-prod-aks -n ci-logs-prod-aks-geneva-integration'
88+
89+
- bash: |
90+
chmod +x ./install-and-execute-testkube-tests.sh
91+
./install-and-execute-testkube-tests.sh AzureClientId=$(AksGenevaIntegrationClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
92+
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
93+
displayName: "Install and execute testkube tests"
7094
71-
# Get the results as a formatted json file
72-
kubectl testkube get testsuiteexecution $execution_id --output json > testkube-results.json
95+
- deployment: Testkube_ci_logs_prod_aks_geneva_integration_multi_tenancy
96+
environment: container-insights
97+
timeoutInMinutes: 120
98+
displayName: "ci-logs-prod-aks-geneva-integration-multi-tenancy tests"
99+
pool:
100+
name: Azure-Pipelines-CI-Test-EO
101+
variables:
102+
skipComponentGovernanceDetection: true
103+
strategy:
104+
runOnce:
105+
deploy:
106+
steps:
107+
- checkout: self
108+
persistCredentials: true
73109

74-
# For any test that has failed, print out the Ginkgo logs
75-
if [[ $(jq -r '.status' testkube-results.json) == "failed" ]]; then
110+
- task: AzureCLI@1
111+
displayName: Get kubeconfig
112+
inputs:
113+
azureSubscription: 'ContainerInsights_Build_Subscription_CI'
114+
scriptLocation: 'inlineScript'
115+
inlineScript: 'az aks get-credentials -g ci-logs-prod-aks -n ci-logs-prod-aks-geneva-integration-multi-tenancy'
116+
117+
- bash: |
118+
chmod +x ./install-and-execute-testkube-tests.sh
119+
./install-and-execute-testkube-tests.sh AzureClientId=$(AksGenevaIntegrationMultiTenancyClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
120+
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
121+
displayName: "Install and execute testkube tests"
76122
77-
# Get each test name and id that failed
78-
jq -r '.executeStepResults[].execute[] | select(.execution.executionResult.status=="failed") | "\(.execution.testName) \(.execution.id)"' testkube-results.json | while read line; do
79-
testName=$(echo $line | cut -d ' ' -f 1)
80-
id=$(echo $line | cut -d ' ' -f 2)
81-
echo "Test $testName failed. Test ID: $id"
123+
- deployment: Testkube_ci_logs_prod_aks_work_load_identity
124+
environment: container-insights
125+
timeoutInMinutes: 120
126+
displayName: "ci-logs-prod-aks-work-load-identity tests"
127+
pool:
128+
name: Azure-Pipelines-CI-Test-EO
129+
variables:
130+
skipComponentGovernanceDetection: true
131+
strategy:
132+
runOnce:
133+
deploy:
134+
steps:
135+
- checkout: self
136+
persistCredentials: true
82137

83-
# Get the Ginkgo logs of the test
84-
kubectl testkube get execution $id > out 2>error.log
138+
- task: AzureCLI@1
139+
displayName: Get kubeconfig
140+
inputs:
141+
azureSubscription: 'ContainerInsights_Build_Subscription_CI'
142+
scriptLocation: 'inlineScript'
143+
inlineScript: 'az aks get-credentials -g ci-logs-prod-aks -n ci-logs-prod-aks-work-load-identity'
144+
145+
- bash: |
146+
chmod +x ./install-and-execute-testkube-tests.sh
147+
./install-and-execute-testkube-tests.sh AzureClientId=$(AksWorkLoadIdentityClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
148+
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
149+
displayName: "Install and execute testkube tests"
85150
86-
# Remove superfluous logs of everything before the last occurence of 'go downloading'.
87-
# The actual errors can be viewed from the ADO run, instead of needing to view the testkube dashboard.
88-
cat error.log | tac | awk '/go: downloading/ {exit} 1' | tac
89-
done
151+
- deployment: Testkube_ci_logs_prod_wcus_fips
152+
environment: container-insights
153+
timeoutInMinutes: 120
154+
displayName: "ci-logs-prod-wcus-fips tests"
155+
pool:
156+
name: Azure-Pipelines-CI-Test-EO
157+
variables:
158+
skipComponentGovernanceDetection: true
159+
strategy:
160+
runOnce:
161+
deploy:
162+
steps:
163+
- checkout: self
164+
persistCredentials: true
90165

91-
# Explicitly fail the ADO task since at least one test failed
92-
exit 1
93-
fi
94-
workingDirectory: $(Build.SourcesDirectory)
95-
displayName: "Run tests"
166+
- task: AzureCLI@1
167+
displayName: Get kubeconfig
168+
inputs:
169+
azureSubscription: 'ContainerInsights_Build_Subscription_CI'
170+
scriptLocation: 'inlineScript'
171+
inlineScript: 'az aks get-credentials -g ci-logs-prod-aks -n ci-logs-prod-wcus-fips'
172+
173+
- bash: |
174+
chmod +x ./install-and-execute-testkube-tests.sh
175+
./install-and-execute-testkube-tests.sh AzureClientId=$(WcusFipsClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
176+
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
177+
displayName: "Install and execute testkube tests"

‎test/ginkgo-e2e/utils/query_logs_api_utils.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func QueryLogsForCount(logsClient *azquery.LogsClient, resourceID string, query
4949
return fmt.Errorf("The query returned 0 tables")
5050
}
5151

52-
fmt.Println("Query result:")
52+
fmt.Println("Query result of query: ", query)
5353

5454
for _, table := range tables {
5555
fmt.Println("Number of rows: ", len(table.Rows))
@@ -138,6 +138,10 @@ func CompareResourcesInLogsAndKubeAPI(K8sClient *kubernetes.Clientset, logsClien
138138
return err
139139
}
140140
for _, pod := range pods {
141+
// skip the testkube namespace as it creates runtime pods for the triggered test which might not be present in the logs
142+
if pod.Namespace == "testkube" {
143+
continue
144+
}
141145
resources = append(resources, pod.Name)
142146
}
143147
query = logsTable + " | where TimeGenerated > ago(15m) | distinct Name"

‎test/testkube/custom-job-template.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
spec:
4+
template:
5+
spec:
6+
nodeSelector:
7+
kubernetes.io/os: linux
8+
kubernetes.io/arch: amd64

‎test/testkube/executors.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[{"name":"init-executor","executor":{"executorType":"init","image":"mcr.microsoft.com/azuremonitor/containerinsights/cidev:testkube-init-executor","types":[]}},{"name":"ginkgo-executor","executor":{"executorType":"job","image":"mcr.microsoft.com/azuremonitor/containerinsights/cidev:testkube-ginkgo-executor","command":["ginkgo"],"args":["-r","-p","--randomize-all","--randomize-suites","--keep-going","--trace","--junit-report","<reportFile>","<envVars>","<runPath>"],"types":["ginkgo/test"],"contentTypes":["string","file-uri","git-file","git-dir","git"],"features":["artifacts","junit-report"],"meta":{"iconURI":"ginkgo","docsURI":"https://kubeshop.github.io/testkube/test-types/executor-ginkgo"}}}]

0 commit comments

Comments
 (0)