Optimize the Gate duration

It spits ["cnf_installation"], ["observability"] and ["shared_database"]
to optimize the gates. All the jobs end before 13 min now.

It also reduces the retries when searching image digests to
avoid wasting too much time in the negative testing.

It renames shared_database2 to shared_database_flaky to avoid breaking
the numbering.

Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

Author: collivier

.github/workflows/actions.yml

@@ -38,10 +38,10 @@ jobs:
         TEST_ARRAY=("${TEST_ARRAY[@]/pod_network_latency/}")
         TEST_ARRAY=("${TEST_ARRAY[@]/zombie/}")
         TEST_ARRAY=("${TEST_ARRAY[@]/oran/}")
-        # Skip 5g, core, shared_database2 tags because they are flaky
+        # Skip 5g, core, shared_database_flaky tags because they are flaky
         TEST_ARRAY=("${TEST_ARRAY[@]/5g/}")
         TEST_ARRAY=("${TEST_ARRAY[@]/core/}")
-        TEST_ARRAY=("${TEST_ARRAY[@]/shared_database2/}")
+        TEST_ARRAY=("${TEST_ARRAY[@]/shared_database_flaky/}")
         TEST_LIST=$(for i in ${TEST_ARRAY[@]}
         do
                  echo "{\"spec\":\"$i\"}," | tr -d '\n'

spec/setup_spec.cr

@@ -21,13 +21,13 @@ def fetch_nginx_chart_tgz(dest_dir : String, version = "15.10.0") : String
 end
 
 describe "Installation" do
-  it "'setup' should install all cnf-testsuite dependencies before installing cnfs", tags: ["cnf_installation"]  do
+  it "'setup' should install all cnf-testsuite dependencies before installing cnfs", tags: ["cnf_installation1"]  do
     result = ShellCmd.run_testsuite("setup")
     result[:status].success?.should be_true
     (/Dependency installation complete/ =~ result[:output]).should_not be_nil
   end
 
-  it "'uninstall_all' should uninstall CNF and testsuite dependencies", tags: ["cnf_installation"] do
+  it "'uninstall_all' should uninstall CNF and testsuite dependencies", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-minimal-cnf/")
       (/CNF installation complete/ =~ result[:output]).should_not be_nil
@@ -38,23 +38,23 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install' should pass with a minimal cnf-testsuite.yml", tags: ["cnf_installation"] do
+  it "'cnf_install' should pass with a minimal cnf-testsuite.yml", tags: ["cnf_installation1"] do
     result = ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample-minimal-cnf/")
     (/CNF installation complete/ =~ result[:output]).should_not be_nil
   ensure
     result = ShellCmd.cnf_uninstall()
     (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil
   end
 
-  it "'cnf_install/cnf_uninstall' should install/uninstall with cnf-config arg as an alias for cnf-path", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should install/uninstall with cnf-config arg as an alias for cnf-path", tags: ["cnf_installation1"] do
     result = ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-minimal-cnf/")
     (/CNF installation complete/ =~ result[:output]).should_not be_nil
   ensure
     result = ShellCmd.cnf_uninstall()
     (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil
   end
 
-  it "'cnf_install/cnf_uninstall' should install/uninstall with cnf-path arg as an alias for cnf-config (.yml)", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should install/uninstall with cnf-path arg as an alias for cnf-config (.yml)", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=example-cnfs/coredns/cnf-testsuite.yml")
       (/CNF installation complete/ =~ result[:output]).should_not be_nil
@@ -64,7 +64,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should install/uninstall with cnf-path arg as an alias for cnf-config (.yaml)", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should install/uninstall with cnf-path arg as an alias for cnf-config (.yaml)", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=spec/fixtures/cnf-testsuite.yaml")
       (/CNF installation complete/ =~ result[:output]).should_not be_nil
@@ -74,7 +74,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should fail on incorrect config", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should fail on incorrect config", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=spec/fixtures/sample-bad-config.yml", expect_failure: true)
       (/Error during parsing CNF config/ =~ result[:output]).should_not be_nil
@@ -83,7 +83,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should fail on invalid config path", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should fail on invalid config path", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=spec/fixtures/bad-config-path", expect_failure: true)
       (/Invalid CNF configuration file: spec\/fixtures\/bad-config-path\./ =~ result[:output]).should_not be_nil
@@ -92,7 +92,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should install/uninstall a cnf with a cnf-testsuite.yml", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should install/uninstall a cnf with a cnf-testsuite.yml", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-config=example-cnfs/coredns/cnf-testsuite.yml")
       (/CNF installation complete/ =~ result[:output]).should_not be_nil
@@ -102,7 +102,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should work with cnf-testsuite.yml that has no directory associated with it", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should work with cnf-testsuite.yml that has no directory associated with it", tags: ["cnf_installation1"] do
     begin
       #TODO force cnfs/<name> to be deployment name and not the directory name
       result = ShellCmd.cnf_install("cnf-config=spec/fixtures/cnf-testsuite.yml")
@@ -113,7 +113,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should install/uninstall with helm_directory that descends multiple directories", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should install/uninstall with helm_directory that descends multiple directories", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml")
       (/CNF installation complete/ =~ result[:output]).should_not be_nil
@@ -123,7 +123,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should properly install/uninstall old versions of cnf configs", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should properly install/uninstall old versions of cnf configs", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=spec/fixtures/cnf-testsuite-v1-example.yml")
       (/CNF installation complete/ =~ result[:output]).should_not be_nil
@@ -133,7 +133,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install' should fail if another CNF is already installed", tags: ["cnf_installation"] do
+  it "'cnf_install' should fail if another CNF is already installed", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml")
       (/CNF installation complete/ =~ result[:output]).should_not be_nil
@@ -145,7 +145,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should install/uninstall a cnf with multiple deployments", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should install/uninstall a cnf with multiple deployments", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_multiple_deployments/cnf-testsuite.yml")
       (/All "coredns" deployment resources are up/ =~ result[:output]).should_not be_nil
@@ -161,7 +161,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should install/uninstall deployment with mixed installation methods", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should install/uninstall deployment with mixed installation methods", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-nginx-redis/cnf-testsuite.yml")
       (/All "nginx" deployment resources are up/ =~ result[:output]).should_not be_nil
@@ -175,7 +175,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install/cnf_uninstall' should handle partial deployment failures gracefully", tags: ["cnf_installation"] do
+  it "'cnf_install/cnf_uninstall' should handle partial deployment failures gracefully", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-partial-deployment-failure/cnf-testsuite.yml", expect_failure: true)
       (/All "nginx" deployment resources are up/ =~ result[:output]).should_not be_nil
@@ -187,7 +187,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install' should detect and report conflicts between deployments", tags: ["cnf_installation"] do
+  it "'cnf_install' should detect and report conflicts between deployments", tags: ["cnf_installation1"] do
     begin
       result = ShellCmd.cnf_install("cnf-path=spec/fixtures/sample-conflicting-deployments.yml", expect_failure: true)
       (/Deployment names should be unique/ =~ result[:output]).should_not be_nil
@@ -196,7 +196,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install' should correctly handle deployment priority", tags: ["cnf_installation"] do
+  it "'cnf_install' should correctly handle deployment priority", tags: ["cnf_installation_priority"] do
     # (kosstennbl) ELK stack requires to be installed with specific order, otherwise it would give errors
     begin
       result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-elk-stack/cnf-testsuite.yml", timeout: 600)
@@ -249,14 +249,14 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_uninstall' should warn user if no CNF is found", tags: ["cnf_installation"] do
+  it "'cnf_uninstall' should warn user if no CNF is found", tags: ["cnf_installation2"] do
     begin
       result = ShellCmd.cnf_uninstall()
       (/CNF uninstallation skipped/ =~ result[:output]).should_not be_nil
     end
   end
 
-  it "'cnf_uninstall' should fail for a stuck manifest deployment", tags: ["cnf_installation"] do
+  it "'cnf_uninstall' should fail for a stuck manifest deployment", tags: ["cnf_installation2"] do
     result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_stuck_finalizer/cnf-testsuite.yml")
     result[:status].success?.should be_true
 
@@ -276,7 +276,7 @@ describe "Installation" do
     (/CNF uninstallation skipped/ =~ result[:output]).should_not be_nil
   end
 
-  it "'cnf_uninstall' should fail for a stuck helm deployment", tags: ["cnf_installation"] do
+  it "'cnf_uninstall' should fail for a stuck helm deployment", tags: ["cnf_installation2"] do
     result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_stuck_helm_deployment/")
     result[:status].success?.should be_true
 
@@ -308,7 +308,7 @@ describe "Installation" do
     (/CNF uninstallation skipped/ =~ result[:output]).should_not be_nil
   end
 
-  it "'cnf_install' should pass for oci repository", tags: ["cnf_installation"] do
+  it "'cnf_install' should pass for oci repository", tags: ["cnf_installation2"] do
     local_registry_port = 53123
     tgz = fetch_nginx_chart_tgz("sample-cnfs/sample_oci_repo")
 
@@ -355,7 +355,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install' should pass for private helm repository", tags: ["cnf_installation"] do
+  it "'cnf_install' should pass for private helm repository", tags: ["cnf_installation2"] do
     chart_museum_port = 53124
     tgz = fetch_nginx_chart_tgz("sample-cnfs/sample_private_repo")
 
@@ -400,7 +400,7 @@ describe "Installation" do
     end
   end
 
-  it "'cnf_install' should require client cert for OCI registry (mTLS)", tags: ["cnf_installation"] do
+  it "'cnf_install' should require client cert for OCI registry (mTLS)", tags: ["cnf_installation2"] do
     registry_port = 54125
     reg_host      = "127.0.0.1.nip.io"
     reg_host_port = "#{reg_host}:#{registry_port}"

spec/workload/microservice_spec.cr

@@ -13,7 +13,7 @@ describe "Microservice" do
     process_result.should be_true
   end
 
-  it "'shared_database' should be skipped no MariaDB containers are found", tags: ["shared_database"]  do
+  it "'shared_database' should be skipped no MariaDB containers are found", tags: ["shared_database1"]  do
     begin
       ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("shared_database")
@@ -25,7 +25,7 @@ describe "Microservice" do
     end
   end
 
-  it "'shared_database' should pass if no database is used by two microservices", tags: ["shared_database"]  do
+  it "'shared_database' should pass if no database is used by two microservices", tags: ["shared_database2"]  do
     begin
       ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("shared_database")
@@ -37,7 +37,7 @@ describe "Microservice" do
     end
   end
 
-  it "'shared_database' should pass if one service connects to a database but other non-service connections are made to the database", tags: ["shared_database"]  do
+  it "'shared_database' should pass if one service connects to a database but other non-service connections are made to the database", tags: ["shared_database3"]  do
     begin
       ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("shared_database")
@@ -49,7 +49,7 @@ describe "Microservice" do
     end
   end
 
-  it "'shared_database' should fail if two services on the cluster connect to the same database", tags: ["shared_database2"]  do
+  it "'shared_database' should fail if two services on the cluster connect to the same database", tags: ["shared_database_flaky"]  do
     begin
       ShellCmd.cnf_install("cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("shared_database")
@@ -62,7 +62,7 @@ describe "Microservice" do
     end
   end
 
-  it "'shared_database' should pass if two services on the cluster connect to the same database but they are not in the helm chart of the cnf", tags: ["shared_database"]  do
+  it "'shared_database' should pass if two services on the cluster connect to the same database but they are not in the helm chart of the cnf", tags: ["shared_database4"]  do
     begin
       ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns")
       KubectlClient::Apply.namespace(DEFAULT_CNF_NAMESPACE)

spec/workload/observability_spec.cr

@@ -10,7 +10,7 @@ describe "Observability" do
     result[:status].success?.should be_true
   end
 
-  it "'log_output' should pass with a cnf that outputs logs to stdout", tags: ["observability"]  do
+  it "'log_output' should pass with a cnf that outputs logs to stdout", tags: ["observability_log_output"]  do
     begin
       ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("log_output")
@@ -21,7 +21,7 @@ describe "Observability" do
     end
   end
 
-  it "'log_output' should fail with a cnf that does not output logs to stdout", tags: ["observability"]  do
+  it "'log_output' should fail with a cnf that does not output logs to stdout", tags: ["observability_log_output"]  do
     begin
       ShellCmd.cnf_install("cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("log_output")
@@ -32,7 +32,7 @@ describe "Observability" do
     end
   end
 
-  it "'prometheus_traffic' should pass if there is prometheus traffic", tags: ["observability"] do
+  it "'prometheus_traffic' should pass if there is prometheus traffic", tags: ["observability_prometheus_traffic"] do
     ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml")
     helm = Helm::Binary.get
 
@@ -54,7 +54,7 @@ describe "Observability" do
     result[:status].success?.should be_true
   end
 
-  it "'prometheus_traffic' should skip if there is no prometheus installed", tags: ["observability"] do
+  it "'prometheus_traffic' should skip if there is no prometheus installed", tags: ["observability_prometheus_traffic"] do
 
       ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml")
       helm = Helm::Binary.get
@@ -66,7 +66,7 @@ describe "Observability" do
       result = ShellCmd.cnf_uninstall()
   end
 
-  it "'prometheus_traffic' should fail if the cnf is not registered with prometheus", tags: ["observability"] do
+  it "'prometheus_traffic' should fail if the cnf is not registered with prometheus", tags: ["observability_prometheus_traffic"] do
 
       ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml")
       Log.info { "Installing prometheus server" }
@@ -85,7 +85,7 @@ describe "Observability" do
       result[:status].success?.should be_true
   end
 
-  it "'open_metrics' should fail if there is not a valid open metrics response from the cnf", tags: ["observability"] do
+  it "'open_metrics' should fail if there is not a valid open metrics response from the cnf", tags: ["observability_open_metrics"] do
     ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml")
     result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true)
     Log.info { "Installing prometheus server" }
@@ -103,7 +103,7 @@ describe "Observability" do
     result[:status].success?.should be_true
   end
 
-  it "'open_metrics' should pass if there is a valid open metrics response from the cnf", tags: ["observability"] do
+  it "'open_metrics' should pass if there is a valid open metrics response from the cnf", tags: ["observability_open_metrics"] do
     ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml")
     result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true)
     Log.info { "Installing prometheus server" }
@@ -121,7 +121,7 @@ describe "Observability" do
     result[:status].success?.should be_true
   end
 
-  it "'routed_logs' should pass if cnfs logs are captured by fluentd", tags: ["observability"] do
+  it "'routed_logs' should pass if cnfs logs are captured by fluentd", tags: ["observability_routed_logs"] do
     ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml")
     result = ShellCmd.run_testsuite("setup:install_fluentd")
     result = ShellCmd.run_testsuite("routed_logs")
@@ -132,7 +132,7 @@ describe "Observability" do
     result[:status].success?.should be_true
   end
 
-  it "'routed_logs' should pass if cnfs logs are captured by fluentd bitnami", tags: ["observability"] do
+  it "'routed_logs' should pass if cnfs logs are captured by fluentd bitnami", tags: ["observability_routed_logs"] do
     ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml")
     result = ShellCmd.run_testsuite("setup:install_fluentdbitnami")
     result = ShellCmd.run_testsuite("routed_logs")
@@ -143,7 +143,7 @@ describe "Observability" do
     result[:status].success?.should be_true
   end
 
-  it "'routed_logs' should pass if cnfs logs are captured by fluentbit", tags: ["observability"] do
+  it "'routed_logs' should pass if cnfs logs are captured by fluentbit", tags: ["observability_routed_logs"] do
     ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-fluentbit")
     result = ShellCmd.run_testsuite("setup:install_fluentbit")
     result = ShellCmd.run_testsuite("routed_logs")
@@ -154,7 +154,7 @@ describe "Observability" do
     result[:status].success?.should be_true
   end
 
-  it "'routed_logs' should fail if cnfs logs are not captured", tags: ["observability"] do
+  it "'routed_logs' should fail if cnfs logs are not captured", tags: ["observability_routed_logs"] do
     ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml")
     Helm.helm_repo_add("bitnami","https://charts.bitnami.com/bitnami")
     #todo  #helm install --values ./override.yml fluentd ./fluentd