Skip to content

Commit ba38124

Browse files
suyadav1suyadav1
authored
Test Automation Framework improvements (#1449)
* Skip column data check for waiting container * Skip fluentd process check if resource optimization is enabled * Fixed windows node issue * change branch for testing * Fixed linux pod check in windows node * Skip Geneva tests * test geneva env var * geneva test * nit * nit * Added GENEVA_INTEGRATION variable * Reverted branch name * Skip column data check for waiting container * Skip fluentd process check if resource optimization is enabled * Fixed windows node issue * change branch for testing * Fixed linux pod check in windows node * Skip Geneva tests * test geneva env var * geneva test * nit * nit * Added GENEVA_INTEGRATION variable * Reverted branch name
1 parent 0dd2adf commit ba38124

File tree

9 files changed

+120
-33
lines changed

9 files changed

+120
-33
lines changed

‎.pipelines/azure_pipeline_testframework.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ jobs:
8888

8989
- bash: |
9090
chmod +x ./install-and-execute-testkube-tests.sh
91-
./install-and-execute-testkube-tests.sh AzureClientId=$(AksGenevaIntegrationClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
91+
./install-and-execute-testkube-tests.sh AzureClientId=$(AksGenevaIntegrationClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri) GenevaIntegration=true
9292
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
9393
displayName: "Install and execute testkube tests"
9494
@@ -116,7 +116,7 @@ jobs:
116116

117117
- bash: |
118118
chmod +x ./install-and-execute-testkube-tests.sh
119-
./install-and-execute-testkube-tests.sh AzureClientId=$(AksGenevaIntegrationMultiTenancyClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
119+
./install-and-execute-testkube-tests.sh AzureClientId=$(AksGenevaIntegrationMultiTenancyClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri) GenevaIntegration=true
120120
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
121121
displayName: "Install and execute testkube tests"
122122
@@ -144,7 +144,7 @@ jobs:
144144

145145
- bash: |
146146
chmod +x ./install-and-execute-testkube-tests.sh
147-
./install-and-execute-testkube-tests.sh AzureClientId=$(AksWorkLoadIdentityClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri)
147+
./install-and-execute-testkube-tests.sh AzureClientId=$(AksWorkLoadIdentityClientId) AzureTenantId=$(AzureTenantId) TeamsWebhookUri=$(TeamsWebhookUri) LinuxTestsOnly=true
148148
workingDirectory: $(Build.SourcesDirectory)/test/testkube/
149149
displayName: "Install and execute testkube tests"
150150

‎test/ginkgo-e2e/containerstatus/containerstatus_suite_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616

1717
var K8sClient *kubernetes.Clientset
1818
var Cfg *rest.Config
19+
var ResourceOptimizationEnabled string
1920

2021
func TestContainerStatus(t *testing.T) {
2122
RegisterFailHandler(Fail)
@@ -27,6 +28,8 @@ var _ = BeforeSuite(func() {
2728
var err error
2829
K8sClient, Cfg, err = utils.SetupKubernetesClient()
2930
Expect(err).NotTo(HaveOccurred())
31+
ResourceOptimizationEnabled, err = utils.IsResourceOptimizationEnabled(K8sClient, "kube-system", "component", "ama-logs-agent", "ama-logs")
32+
Expect(err).NotTo(HaveOccurred())
3033
})
3134

3235
var _ = AfterSuite(func() {

‎test/ginkgo-e2e/containerstatus/containerstatus_test.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,24 +55,27 @@ var _ = DescribeTable("The pods should be scheduled in all Fips and ARM64 nodes"
5555
*/
5656
var _ = DescribeTable("All processes are running",
5757
func(namespace, labelName, labelValue, containerName string, processes []string) {
58+
// Fluentd is not running in linux ds pods if AZMON_RESOURCE_OPTIMIZATION_ENABLED is set to true.
59+
if labelValue == "ama-logs-rs" || ResourceOptimizationEnabled != "true" {
60+
// Add fluentd process to the list of processes to check
61+
processes = append(processes, "fluentd")
62+
}
5863
err := utils.CheckAllProcessesRunning(K8sClient, Cfg, labelName, labelValue, namespace, containerName, processes)
5964
Expect(err).NotTo(HaveOccurred())
6065
},
6166
Entry("when checking the ama-logs-rs replica pod", "kube-system", "rsName", "ama-logs-rs", "ama-logs",
6267
[]string{
6368
"fluent-bit",
64-
"fluentd",
65-
"mdsd -a -A -r",
66-
"inotifywait /etc/config/settings",
69+
"mdsd",
70+
"inotifywait",
6771
"crond",
6872
},
6973
),
7074
Entry("when checking the ama-logs daemonset pods", "kube-system", "component", "ama-logs-agent", "ama-logs",
7175
[]string{
7276
"fluent-bit",
73-
"fluentd",
74-
"mdsd -a -A -r",
75-
"inotifywait /etc/config/settings",
77+
"mdsd",
78+
"inotifywait",
7679
"crond",
7780
"telegraf",
7881
},
@@ -112,7 +115,5 @@ var _ = DescribeTable("The container logs should not contain errors",
112115
},
113116
Entry("when checking the ama-logs-rs pods", "kube-system", "rsName", "ama-logs-rs"),
114117
Entry("when checking the ama-logs daemonset pods", "kube-system", "component", "ama-logs-agent"),
115-
Entry("when checking the ama-logs-rs pods", "kube-system", "rsName", "ama-logs-rs", Label(utils.ARM64Label)),
116-
Entry("when checking the ama-logs daemonset pods", "kube-system", "component", "ama-logs-agent", Label(utils.ARM64Label)),
117118
Entry("when checking the ama-logs-windows daemonset pods", "kube-system", "component", "ama-logs-agent-windows", Label(utils.WindowsLabel)),
118119
)

‎test/ginkgo-e2e/querylogs/querylogs_suite_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package querylogs_test
22

33
import (
4+
"os"
45
"testing"
56

67
"docker-provider/test/utils"
@@ -15,6 +16,8 @@ import (
1516
var K8sClient *kubernetes.Clientset
1617
var LogsClient *azquery.LogsClient
1718
var AKSResourceId string
19+
var RetinaNetworkFlowLogsEnabled string
20+
var GenevaIntegrationEnabled string
1821
var Cfg *rest.Config
1922

2023
func TestQuerylogs(t *testing.T) {
@@ -28,6 +31,9 @@ var _ = BeforeSuite(func() {
2831
Expect(err).NotTo(HaveOccurred())
2932
AKSResourceId, err = utils.GetAKSResourceID(K8sClient, "kube-system", "component", "ama-logs-agent", "ama-logs")
3033
Expect(err).NotTo(HaveOccurred())
34+
RetinaNetworkFlowLogsEnabled, err = utils.IsRetinaNetworkFlowLogsEnabled(K8sClient, "kube-system", "component", "ama-logs-agent", "ama-logs")
35+
Expect(err).NotTo(HaveOccurred())
36+
GenevaIntegrationEnabled = os.Getenv("GENEVA_INTEGRATION")
3137
LogsClient, err = utils.SetupLogsClient()
3238
Expect(err).NotTo(HaveOccurred())
3339
})

‎test/ginkgo-e2e/querylogs/querylogs_test.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package querylogs_test
22

33
import (
4-
"os"
54
"strings"
65

76
. "github.com/onsi/ginkgo/v2"
@@ -14,9 +13,12 @@ var _ = Describe("When querying the logs for the table", func() {
1413
DescribeTable("All tables should have logs",
1514
func(table string) {
1615
// Skip RetinaNetworkFlowLogs test if the feature is not enabled
17-
if table == "RetinaNetworkFlowLogs" && os.Getenv("ENABLE_RETINA_NETWORK_FLOW_LOGS") != "true" {
16+
if table == "RetinaNetworkFlowLogs" && RetinaNetworkFlowLogsEnabled != "true" {
1817
Skip("RetinaNetworkFlowLogs test skipped because ENABLE_RETINA_NETWORK_FLOW_LOGS is not set to 'true'")
1918
}
19+
if table == "ContainerLog" && GenevaIntegrationEnabled == "true" {
20+
Skip("ContainerLog test skipped because GENEVA_INTEGRATION is set to 'true'")
21+
}
2022
var err error
2123
query := table + " | where TimeGenerated > ago(15m) | summarize count()"
2224
err = utils.QueryLogsForCount(LogsClient, AKSResourceId, query, false)
@@ -42,7 +44,9 @@ var _ = Describe("When querying the logs for the table", func() {
4244
var _ = Describe("When querying the logs for the ContainerInventory", func() {
4345
DescribeTable("Column should have zero empty values",
4446
func(column string) {
45-
query := "ContainerInventory | where TimeGenerated > ago(1h) | summarize countif(isempty(" + column + ") or isnull(" + column + "))"
47+
// Skip records with ContainerState 'Waiting' to avoid false positives due to the container being in a waiting state.
48+
// If the pod name contains 'ama-logs', we include it to ensure we capture the ama-logs agent containers.
49+
query := "ContainerInventory | where TimeGenerated > ago(1h) and (ContainerState !~ 'Waiting' or ContainerHostname contains 'ama-logs') | summarize countif(isempty(" + column + ") or isnull(" + column + "))"
4650
err := utils.QueryLogsForCount(LogsClient, AKSResourceId, query, true)
4751
Expect(err).NotTo(HaveOccurred())
4852
},

‎test/ginkgo-e2e/utils/constants.go‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,7 @@ var (
1313
)
1414

1515
const (
16-
OperatorLabel = "operator"
17-
ArcExtensionLabel = "arc-extension"
18-
WindowsLabel = "windows"
19-
ARM64Label = "arm64"
20-
FIPSLabel = "fips"
21-
LinuxDaemonsetCustomConfig = "linux-daemonset-custom-config"
16+
WindowsLabel = "windows"
17+
ARM64Label = "arm64"
18+
FIPSLabel = "fips"
2219
)

‎test/ginkgo-e2e/utils/kubernetes_api_utils.go‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,22 @@ func GetAKSResourceID(clientset *kubernetes.Clientset, namespace string, labelKe
9797
return envVars["AKS_RESOURCE_ID"], nil
9898
}
9999

100+
func IsResourceOptimizationEnabled(clientset *kubernetes.Clientset, namespace string, labelKey string, labelValue string, containerName string) (string, error) {
101+
envVars, error := GetContainerEnvVars(clientset, namespace, labelKey, labelValue, containerName)
102+
if error != nil {
103+
return "", fmt.Errorf("failed to get environment variables for container %s in pod with label %s=%s: %v", containerName, labelKey, labelValue, error)
104+
}
105+
return envVars["AZMON_RESOURCE_OPTIMIZATION_ENABLED"], nil
106+
}
107+
108+
func IsRetinaNetworkFlowLogsEnabled(clientset *kubernetes.Clientset, namespace string, labelKey string, labelValue string, containerName string) (string, error) {
109+
envVars, error := GetContainerEnvVars(clientset, namespace, labelKey, labelValue, containerName)
110+
if error != nil {
111+
return "", fmt.Errorf("failed to get environment variables for container %s in pod with label %s=%s: %v", containerName, labelKey, labelValue, error)
112+
}
113+
return envVars["ENABLE_RETINA_NETWORK_FLOW_LOGS"], nil
114+
}
115+
100116
/*
101117
* Returns all pods in the given namespace with the given label.
102118
*/
@@ -385,7 +401,7 @@ func CheckIfAllPodsScheduleOnNodes(clientset *kubernetes.Clientset, namespace, l
385401
* Check that pods with the specified namespace and label value are scheduled in all the Fips and ARM64 nodes. If a node has no schduled pod on it, return an error.
386402
* Also check that the containers are scheduled and running on those nodes.
387403
*/
388-
func CheckIfAllPodsScheduleOnSpecificNodesLabels(clientset *kubernetes.Clientset, namespace, labelKey string, labelValue string, nodeLabelKey string, nodeLabelValue string) error {
404+
func CheckIfAllPodsScheduleOnSpecificNodesLabels(clientset *kubernetes.Clientset, namespace, contollerLabelKey string, ControllerLabelValue string, nodeLabelKey string, nodeLabelValue string) error {
389405

390406
// Get list of all nodes
391407
nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
@@ -394,13 +410,19 @@ func CheckIfAllPodsScheduleOnSpecificNodesLabels(clientset *kubernetes.Clientset
394410
return errors.New(fmt.Sprintf("Error getting nodes with the specified labels: %v", err))
395411
}
396412

413+
osLabel := "kubernetes.io/os"
414+
osLabelValue := "linux"
415+
if ControllerLabelValue == "ama-logs-agent-windows" {
416+
osLabelValue = "windows"
417+
}
418+
397419
for _, node := range nodes.Items {
398-
if value, ok := node.Labels[nodeLabelKey]; ok && value == nodeLabelValue {
420+
if value, ok := node.Labels[nodeLabelKey]; ok && value == nodeLabelValue && node.Labels[osLabel] == osLabelValue {
399421

400422
// Get list of pods scheduled on this node
401423
pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{
402424
FieldSelector: "spec.nodeName=" + node.Name,
403-
LabelSelector: labelKey + "=" + labelValue,
425+
LabelSelector: contollerLabelKey + "=" + ControllerLabelValue,
404426
})
405427

406428
if err != nil || pods == nil || len(pods.Items) == 0 {

‎test/testkube/install-and-execute-testkube-tests.sh‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ do
99
AzureClientId) AzureClientId=$VALUE ;;
1010
AzureTenantId) AzureTenantId=$VALUE ;;
1111
TeamsWebhookUri) TeamsWebhookUri=$VALUE ;;
12+
LinuxTestsOnly) LinuxTestsOnly=$VALUE ;;
13+
GenevaIntegration) GenevaIntegration=$VALUE ;;
1214
*)
1315
esac
1416
done
@@ -31,6 +33,7 @@ echo "Install testkube CRIs"
3133
export AZURE_CLIENT_ID=$AzureClientId
3234
export AZURE_TENANT_ID=$AzureTenantId
3335
export WEBHOOK_URI=$TeamsWebhookUri
36+
export GENEVA_INTEGRATION=$GenevaIntegration
3437
kubectl apply -f ./api-server-permissions.yaml
3538
envsubst < ./testkube-test-crs.yaml > ./testkube-test-crs-updated.yaml
3639
kubectl apply -f ./testkube-test-crs-updated.yaml
@@ -39,11 +42,16 @@ echo "Wait for cluster to be ready"
3942
sleep 120
4043

4144
echo "Run testkube tests"
42-
# Run the full test suite
43-
kubectl testkube run testsuite e2e-tests-merge --job-template ./custom-job-template.yaml --verbose
44-
45-
# Get the current id of the test suite now running
46-
execution_id=$(kubectl testkube get testsuiteexecutions --test-suite e2e-tests-merge --limit 1 | grep e2e-tests | awk '{print $1}')
45+
execution_id=""
46+
if [[ $LinuxTestsOnly == "true" ]]; then
47+
echo "Running Linux tests only"
48+
kubectl testkube run testsuite e2e-tests-linux --job-template ./custom-job-template.yaml --verbose
49+
execution_id=$(kubectl testkube get testsuiteexecutions --test-suite e2e-tests-linux --limit 1 | grep e2e-tests | awk '{print $1}')
50+
else
51+
echo "Running all tests"
52+
kubectl testkube run testsuite e2e-tests-all --job-template ./custom-job-template.yaml --verbose
53+
execution_id=$(kubectl testkube get testsuiteexecutions --test-suite e2e-tests-all --limit 1 | grep e2e-tests | awk '{print $1}')
54+
fi
4755

4856
# Watch until the all the tests in the test suite finish
4957
kubectl testkube watch testsuiteexecution $execution_id

‎test/testkube/testkube-test-crs.yaml‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: tests.testkube.io/v3
22
kind: Test
33
metadata:
4-
name: containerstatus
4+
name: containerstatus-linux
55
namespace: testkube
66
labels:
77
executor: ginkgo-executor
@@ -18,7 +18,31 @@ spec:
1818
executionRequest:
1919
args:
2020
- "--label-filter"
21-
- "!(arc-extension,linux-daemonset-custom-config)"
21+
- "!(windows)"
22+
- "./containerstatus"
23+
executePostRunScriptBeforeScraping: false
24+
---
25+
apiVersion: tests.testkube.io/v3
26+
kind: Test
27+
metadata:
28+
name: containerstatus-windows
29+
namespace: testkube
30+
labels:
31+
executor: ginkgo-executor
32+
test-type: ginkgo-test
33+
spec:
34+
type: ginkgo/test
35+
content:
36+
type: git
37+
repository:
38+
type: git
39+
uri: https://github.com/microsoft/Docker-Provider/
40+
branch: ci_prod
41+
path: test/ginkgo-e2e
42+
executionRequest:
43+
args:
44+
- "--label-filter"
45+
- "windows"
2246
- "./containerstatus"
2347
executePostRunScriptBeforeScraping: false
2448
---
@@ -42,7 +66,7 @@ spec:
4266
executionRequest:
4367
args:
4468
- "--label-filter"
45-
- "!(arc-extension,linux-daemonset-custom-config)"
69+
- "!(windows)"
4670
- "./livenessprobe"
4771
executePostRunScriptBeforeScraping: false
4872
---
@@ -73,20 +97,42 @@ spec:
7397
name: AZURE_CLIENT_ID
7498
value: "$AZURE_CLIENT_ID"
7599
type: basic
100+
GENEVA_INTEGRATION:
101+
name: GENEVA_INTEGRATION
102+
value: "$GENEVA_INTEGRATION"
103+
type: basic
76104
args:
77105
- "./querylogs"
78106
executePostRunScriptBeforeScraping: false
79107
---
80108
apiVersion: tests.testkube.io/v3
81109
kind: TestSuite
82110
metadata:
83-
name: e2e-tests-merge
111+
name: e2e-tests-all
112+
namespace: testkube
113+
spec:
114+
steps:
115+
- stopOnFailure: false
116+
execute:
117+
- test: containerstatus-linux
118+
- test: containerstatus-windows
119+
- stopOnFailure: false
120+
execute:
121+
- delay: 2m0s
122+
- stopOnFailure: false
123+
execute:
124+
- test: querylogs
125+
---
126+
apiVersion: tests.testkube.io/v3
127+
kind: TestSuite
128+
metadata:
129+
name: e2e-tests-linux
84130
namespace: testkube
85131
spec:
86132
steps:
87133
- stopOnFailure: false
88134
execute:
89-
- test: containerstatus
135+
- test: containerstatus-linux
90136
- stopOnFailure: false
91137
execute:
92138
- delay: 2m0s

0 commit comments

Comments
 (0)