[WIP] Use test operator to run some of the ansible tests

.zuul.yaml

@@ -25,6 +25,7 @@
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-metric-verification-test.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
     required-projects: &required_projects
@@ -187,6 +188,7 @@
               - ci/logging_tests_controller.yml
               - ci/report_result.yml
               - .zuul.yaml
+              - ci/vars-metric-verification-test.yml
         - functional-periodic-telemetry-with-ceph:
             files:
               # Run this job for changes to the volume_pool_metrics test changes as this is

ansible.cfg

@@ -0,0 +1,11 @@
+[defaults]
+callbacks_enabled = custom_junit, custom_logger
+callback_plugins = callback_plugins
+# additional paths to search for roles
+roles_path = roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.ansible/roles:../ci-framework/roles
+
+[custom_logger]
+output_dir = /$HOME/ci-framework-data/tests/feature-verification-tests/
+
+#[custom_junit]
+#output_dir = /var/lib/AnsibleTests/external_files/feature-verification-tests/

ci/ansible-tests-fvt.yaml

@@ -0,0 +1,172 @@
+apiVersion: test.openstack.org/v1beta1
+kind: AnsibleTest
+metadata:
+  name: ansible-fvt
+  namespace: openstack
+spec:
+  #ansibleGitRepo: 'http://github.com/infrawatch/feature-verification-tests'
+  #ansibleGitRepo: 'http://github.com/elfiesmelfie/feature-verification-tests" -b "test_operator_changes'
+  # I pushed test_operator_changes to master
+  ansibleGitRepo: 'http://github.com/elfiesmelfie/feature-verification-tests'
+  debug: true
+  containerImage: quay.io/podified-antelope-centos9/openstack-ansible-tests:current-podified
+  # TODO: try an absolute path for the playbook. 
+  #ansiblePlaybookPath: ci/run_verify_metrics_osp18.yml
+  storageClass: crc-csi-hostpath-provisioner
+  computeSSHKeySecretName: dataplane-ansible-ssh-private-key-secret
+  workloadSSHKeySecretName: dataplane-ansible-ssh-private-key-secret
+    # ansibleExtraVars: "ANSIBLE_ROLES_PATH=roles"
+  # need to check where the collection ends up living...
+  # or indeed, just the ansible playbook.
+  # /var/lib/ansible/ansible/ci/"
+  # install_yamls_dir needs to be updated later, maybe removed... the required dirs should be mounted into the pods
+  ansibleVarFiles: |
+    install_yamls_dir: "{{ ansible_env.HOME }}/install_yamls"
+    cifmw_openshift_kubeconfig: ~/.kube.config
+    cifmw_path: "{{ ansible_env.PATH }}:/var/lib/ansible/bin"
+    cifmw_openshift_api: api.crc.testing:6443
+    cifmw_openshift_password: '12345678'
+    cifmw_openshift_user: kubeadmin
+    openstack_cmd: "oc -n openstack rsh openstackclient openstack" 
+    patch_observabilityclient: true
+    telemetry_verify_metrics_metric_sources_to_test:
+      - ceilometer_compute_agent
+      - ceilometer_central_agent
+      #- ksm
+      - node_exporter
+      #- podman_exporter
+      #- rabbitmq
+      #- kepler
+      #- openstack_network_exporter
+      #- ceilometer_prom_exporter
+      #- mariadb
+  #ansibleCollections: "git+https://github.com/infrawatch/feature-verification-tests.git,master"
+  #ansibleCollections: 'git+https://github.com/elfiesmelfie/feature-verification-tests.git,test_operator_changes" "git+https://github.com/openstack-k8s-operators/ci-framework'
+  #ansibleCollections: 'git+https://github.com/elfiesmelfie/feature-verification-tests.git,test_operator_changes'
+  ansibleCollections: 'git+https://github.com/elfiesmelfie/feature-verification-tests.git,master'
+  # allow privilige escalation so ci_setup/tasks/packages can be run, even if we don't install any packages. 
+  # Since the oc install tasks are in the pre-tasks playbook, and don't use the ci_setup role, this can be removed.
+  # TODO: Consider updating cifmw ci_setup role to have a separate get_oc_client task.
+  privileged: true
+  extraMounts:
+    - extraVol:
+        - mounts:
+            - name: test-operator-logs
+              mountPath: /var/lib/ansible/ci-framework-data/tests/feature-verification-tests
+          volumes: []
+  ansibleInventory: |
+    localhost ansible_connection=local ansible_python_interpreter=python3
+    compute-0 ansible_host=192.168.122.100 ansible_user=root ansible_ssh_private_key_file=/var/lib/ansible/.ssh/compute_id
+    compute-1 ansible_host=192.168.122.101 ansible_user=root ansible_ssh_private_key_file=/var/lib/ansible/.ssh/compute_id
+    [compute]
+    compute-0
+    compute-1
+    [controller]
+    localhost
+    [local]
+    localhost
+  workflow:
+    - stepName: verify-metrics
+      ansiblePlaybookPath: ci/run_verify_metrics_osp18.yml
+    - stepName: logging
+      ansiblePlaybookPath: ci/logging_tests_all.yml
+    - stepName: autoscaling
+      ansiblePlaybookPath: ci/run_autoscaling_tests.yml
+    - stepName: graphing
+      ansblePlaybookPath: ci/run_graphing_test.yml
+    - stepName: report-results
+      ansiblePlaybookPath: ci/report_result.yml
+  #FIELDS:
+  #  SELinuxLevel	<string>
+  #    A SELinuxLevel that should be used for test pods spawned by the test
+  #    operator.
+  #
+  # TODO: check this option in test-operator, since it only seems to support a sigle extra collection at the moment
+  # TODO: Correct this to requirements.yml instead of requirements.yaml (or support both)
+  #  ansibleCollections	<string>
+  #    AnsibleCollections - extra ansible collections to instal in additionn to the
+  #    ones exist in the requirements.yaml
+  #
+  #  ansibleExtraVars	<string>
+  #    AnsibleExtraVars - string to pass parameters to ansible using
+  #
+  #  ansibleGitRepo	<string> -required-
+  #    AnsibleGitRepo - git repo to clone into container
+  #  ansibleInventory	<string>
+  #    AnsibleInventory - string that contains the inventory file content
+  #
+  #  ansiblePlaybookPath	<string> -required-
+  #    AnsiblePlaybookPath - path to ansible playbook
+  #
+  # TODO: If the repos from the hosts are copied into the extraMounts, we can skip using the ansibleVarFiles interface, and use ansibleExtraVars: @<file> instead.
+  #  ansibleVarFiles	<string>
+  #    AnsibleVarFiles - interface to create ansible var files Those get added to
+  #    the
+  #
+  #  backoffLimit	<integer>
+  #    BackoffLimit allows to define the maximum number of retried executions
+  #    (defaults to 0).
+  #
+  #  computeSSHKeySecretName	<string>
+  #    ComputeSSHKeySecretName is the name of the k8s secret that contains an ssh
+  #    key for computes.
+  #    The key is mounted to ~/.ssh/id_ecdsa in the ansible pod
+  #  containerImage	<string>
+  #    A URL of a container image that should be used by the test-operator for
+  #    tests execution.
+  #
+  #  debug	<boolean>
+  #    Run ansible playbook with -vvvv
+  #  extraConfigmapsMounts	<[]Object>
+  #    Extra configmaps for mounting inside the pod
+  # TODO: Investigate preparing a PV and populating it with the git repos from the host.
+  # This would removed the need to install from galaxy and push to elfie/master in order to run a playbook.
+  # The git repos could then be cloned from the local copy of the repo.
+  #
+  #  extraMounts	<[]Object>
+  #    ExtraMounts containing conf files, credentials and storage volumes
+  #
+  #  nodeSelector	<map[string]string>
+  #    This value contains a nodeSelector value that is applied to test pods
+  #    spawned by the test operator.
+  #
+  #  openStackConfigMap	<string>
+  #    OpenStackConfigMap is the name of the ConfigMap containing the clouds.yaml
+  #
+  #  openStackConfigSecret	<string>
+  #    OpenStackConfigSecret is the name of the Secret containing the secure.yaml
+  # TODO: add openstack config secrets, then skip running openstack commands in openstackclient. This also means that the version of promclient, observabilityclient and apdh client can be updated without patching the containers.
+  # clouds.yaml is included by default. Need to check the cloudname.
+  # The OS_CLOUD_NAME can be set in the env to select the cloud.
+  # clouds.yaml is mounted at /etc/openstack/clouds.yaml
+  # The cloud is called default.
+  #  privileged	<boolean>
+  #    Use with caution! This parameter specifies whether test-operator should
+  #    spawn
+  #    test pods with allowedPrivilegedEscalation: true, readOnlyRootFilesystem:
+  #    false,
+  #    runAsNonRoot: false, automountServiceAccountToken: true, and the default
+  #    capabilities on top of capabilities that are usually needed by the test
+  #    pods (NET_ADMIN, NET_RAW). This parameter is deemed insecure but it is
+  #    needed for certain test-operator functionalities to work properly (e.g.:
+  #    extraRPMs in Tempest CR, or certain set of tobiko tests).
+  #
+  #  resources	<Object>
+  #    The desired amount of resources that should be assigned to each test pod
+  #    spawned using the AnsibleTest CR.
+  #    https://pkg.go.dev/k8s.io/api/core/v1#ResourceRequirements
+  #
+  #  storageClass	<string>
+  #    StorageClass used to create any test-operator related PVCs.
+  #
+  #  tolerations	<[]Object>
+  #    This value contains a toleration that is applied to pods spawned by the
+  #    test pods that are spawned by the test-operator.
+  #
+  #  workflow	<[]Object>
+  #    A parameter that contains a workflow definition.
+  #
+  #  workloadSSHKeySecretName	<string>
+  #    WorkloadSSHKeySecretName is the name of the k8s secret that contains an ssh
+  #    key for the ansible workload.
+  #    The key is mounted to ~/test_keypair.key in the ansible pod

ci/logging_tests_all.yml

@@ -1,3 +1,7 @@
+- name: Run pre-test steps
+  ansible.builtin.import_playbook: pre-test-steps.yml
+
+
 - name: Run the compute node tests
   ansible.builtin.import_playbook: logging_tests_computes.yml
 

ci/logging_tests_computes.yml

@@ -12,6 +12,15 @@
       - nova_compute
     common_file_list:
       - /etc/rsyslog.d/10-telemetry.conf
+  pre_tasks:
+    - name: "Skip logging test by run_logging_test false"
+      ansible.builtin.meta: end_play
+      when: not run_logging_test | default(true) | bool
+
+    - name: "Skip logging test by logging disabled"
+      ansible.builtin.meta: end_play
+      when: not logging_enabled | bool
+
   tasks:
     - name: "Run Telemetry Logging tests"
       ansible.builtin.import_role:
@@ -33,6 +42,14 @@
     common_container_list:
       - ceilometer_agent_compute
       - node_exporter
+  pre_tasks:
+    - name: "Skip logging test by run_logging_test false"
+      ansible.builtin.meta: end_play
+      when: not run_logging_test | default(true) | bool
+
+    - name: "Skip logging test by logging disabled"
+      ansible.builtin.meta: end_play
+      when: not logging_enabled | bool
   tasks:
     - name: "Run the tests from the common role"
       ansible.builtin.import_role:

ci/logging_tests_controller.yml

@@ -68,6 +68,15 @@
       - logging-loki-query-frontend-grpc
       - logging-loki-query-frontend-http
       - openstack-logging
+  pre_tasks:
+    - name: "Skip logging test by run_logging_test false"
+      ansible.builtin.meta: end_play
+      when: not run_logging_test | default(true) | bool
+
+    - name: "Skip logging test by logging disabled"
+      ansible.builtin.meta: end_play
+      when: not logging_enabled | bool
+
   tasks:
     - name: "Verify logging infrastructure components"
       ansible.builtin.import_role:
@@ -105,6 +114,15 @@
     common_pod_nspace: openshift-operators-redhat
     common_pod_list:
       - loki-operator-controller-manager
+  pre_tasks:
+    - name: "Skip logging test by run_logging_test false"
+      ansible.builtin.meta: end_play
+      when: not run_logging_test | default(true) | bool
+
+    - name: "Skip logging test by logging disabled"
+      ansible.builtin.meta: end_play
+      when: not logging_enabled | bool
+
   tasks:
     - name: "Verify Pods running"
       ansible.builtin.import_role:
@@ -131,7 +149,15 @@
       - logging-loki-querier
       - logging-loki-query-frontend
       - collector
+  pre_tasks:
+    - name: "Skip logging test by run_logging_test false"
+      ansible.builtin.meta: end_play
+      when: not run_logging_test | default(true) | bool
 
+    - name: "Skip logging test by logging disabled"
+      ansible.builtin.meta: end_play
+      when: not logging_enabled | bool
+
   ### see JIRA LOG-5431 if pods not running
   tasks:
     - name: "Verify Pods running"
@@ -151,6 +177,15 @@
     common_pod_nspace: minio-dev
     common_pod_list:
       - minio
+  pre_tasks:
+    - name: "Skip logging test by run_logging_test false"
+      ansible.builtin.meta: end_play
+      when: not run_logging_test | default(true) | bool
+
+    - name: "Skip logging test by logging disabled"
+      ansible.builtin.meta: end_play
+      when: not logging_enabled | bool
+
   tasks:
     - name: "Run pod running tests"
       ansible.builtin.import_role:
@@ -183,7 +218,17 @@
       - ssh-known-hosts-edpm-deployment
       - telemetry-edpm-deployment-openstack-edpm-ipam
       - validate-network-edpm-deployment-openstack-edpm-ipam
+  pre_tasks:
+    - name: "Skip logging test by run_logging_test false"
+      ansible.builtin.meta: end_play
+      when: not run_logging_test | default(true) | bool
+
+    - name: "Skip logging test by logging disabled"
+      ansible.builtin.meta: end_play
+      when: not logging_enabled | bool
+
   tasks:
+
     - name: "Run pods completed tests"
       ansible.builtin.import_role:
         name: common
@@ -209,6 +254,15 @@
       - lokistacks.loki.grafana.com
       - recordingrules.loki.grafana.com
       - rulerconfigs.loki.grafana.com
+  pre_tasks:
+    - name: "Skip logging test by run_logging_test false"
+      ansible.builtin.meta: end_play
+      when: not run_logging_test | default(true) | bool
+
+    - name: "Skip logging test by logging disabled"
+      ansible.builtin.meta: end_play
+      when: not logging_enabled | bool
+
   tasks:
     - name: "Run Services and CRD tests"
       ansible.builtin.import_role: