From 41674eb461ec6593a37f669b4c61e38e08586d4a Mon Sep 17 00:00:00 2001 From: Alex Yefimov <126113326+ayefimov-1@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:44:43 -0400 Subject: [PATCH 01/18] [zuul] Add logging test job Add a job that runs the telemetry_logging and common roles against the openstack cloud. Separate plays target compute and control nodes --- .zuul.yaml | 15 ++ ci/logging_tests_computes.yml | 21 +++ ci/logging_tests_local.yml | 248 ++++++++++++++++++++++++++++++++++ ci/run_playbooks_logging.yml | 57 ++++++++ 4 files changed, 341 insertions(+) create mode 100644 ci/logging_tests_local.yml create mode 100644 ci/run_playbooks_logging.yml diff --git a/.zuul.yaml b/.zuul.yaml index b44ab4740..eac54504b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -75,6 +75,21 @@ required-projects: *required_projects irrelevant-files: *irrelevant_files +- job: + name: functional-logging-tests-osp18 + parent: telemetry-operator-multinode-logging + description: | + Run the logging functional test on osp18+patched version + vars: + cifmw_extras: + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml" + roles: + - zuul: github.com/openstack-k8s-operators/ci-framework + required-projects: *required_projects + irrelevant-files: *irrelevant_files + - job: name: feature-verification-tests-noop parent: noop diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index d520a9c42..b208ce0cc 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -15,3 +15,24 @@ - name: "Run Telemetry Logging tests" ansible.builtin.import_role: name: telemetry_logging + +- name: "Verify ctlplane logging containers and files" + hosts: computes + gather_facts: no + vars: + container_test_id: "RHOSO-12753" + container_list: + - ceilometer_agent_compute + - ceilometer_agent_ipmi + - node_exporter + + file_test_id: "RHOSO-12754" + file_list: + - /etc/rsyslog.d/10-telemetry.conf + + tasks: + - name: "Run file and container tests" + ansible.builtin.import_role: + name: common + + diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml new file mode 100644 index 000000000..a2dee02ec --- /dev/null +++ b/ci/logging_tests_local.yml @@ -0,0 +1,248 @@ +- name: Prepare Logging Tests + hosts: controller + gather_facts: false + vars: + id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + + tasks: + - name: "Log into OCP" + ansible.builtin.shell: + cmd: | + oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443 + failed_when: false + changed_when: false + + - name: "Create id rsa file for ctlplane access" + ansible.builtin.shell: + cmd: | + oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}" + register: rsa_results + failed_when: rsa_results.rc != 0 + changed_when: false + + - name: "Change files permissions" + ansible.builtin.shell: + cmd: | + chmod 600 "{{ id_rsa_path }}" + changed_when: false + +- name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" + hosts: controller + gather_facts: no + vars: + proj_test_id: "RHOSO-12668" + proj_list: + - openshift-openstack-infra + - openshift + - openstack-operators + - openshift-logging + + + endpoint_test_id: "RHOSO-12682" + endpoint_list: + - [nova,compute,public] + - [nova,compute,internal] + - [placement,placement,public] + - [placement,placement,internal] + - [swift,object-store,public] + - [swift,object-store,internal] + - [cinderv3,volumev3,public] + - [cinderv3,volumev3,internal] + - [barbican,key-manager,public] + - [barbican,key-manager,internal] + - [keystone,identity,public] + - [keystone,identity,internal] + - [glance,image,public] + - [glance,image,internal] + - [neutron,network,public] + - [neutron,network,internal] + + + cred_test_id: "RHOSO-12670" + cred_list: + - alertingrules.loki.grafana.com + - lokistacks.loki.grafana.com + - recordingrules.loki.grafana.com + - rulerconfigs.loki.grafana.com + + + node_test_id: "RHOSO-12671" + node_list: + - edpm-compute-0 + - edpm-compute-1 + - crc + + + pod_test_id: "RHOS0-12672" + pod_status_str: "Running" + pod_nspace: openstack-operators + pod_list: + - telemetry-operator-controller-manager + - dataplane-operator-controller-manager + + + service_test_id: "RHOSO-12675" + service_nspace: openshift-logging + service_list: + - cluster-logging-operator-metrics + - logging-loki-compactor-grpc + - logging-loki-compactor-http + - logging-loki-distributor-grpc + - logging-loki-distributor-http + - logging-loki-gateway-http + - logging-loki-gossip-ring + - logging-loki-index-gateway-grpc + - logging-loki-index-gateway-http + - logging-loki-ingester-grpc + - logging-loki-ingester-http + - logging-loki-querier-grpc + - logging-loki-querier-http + - logging-loki-query-frontend-grpc + - logging-loki-query-frontend-http + - logging-view-plugin + - openstack-logging + + + manifest_test_id: "RHOSO-12677" + manifest_list: + - "loki-operator 2" + - "loki-helm-operator 1" + + + subscription_test_id: "RHOSO-12678" + subscription_nspace: openshift-operators-redhat + subscription_list: + - loki-operator + + tasks: + - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests" + ansible.builtin.import_role: + name: common + + + +- name: "Verify logging pods are running in openstack" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12752" + pod_status_str: "Running" + pod_nspace: openstack + pod_list: + - openstackclient + + tasks: + - name: "Verify Running Pods" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in openshift-operators-redhat" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12673" + pod_status_str: "Running" + pod_nspace: openshift-operators-redhat + pod_list: + - loki-operator-controller-manager + + tasks: + - name: "Verify Pods running" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in openshift-logging" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12676" + pod_status_str: "Running" + pod_nspace: openshift-logging + pod_list: + - cluster-logging-operator + - collector + - logging-loki-compactor + - logging-loki-distributor + - logging-loki-gateway + - logging-loki-index-gateway + - logging-loki-ingester + - logging-loki-querier + - logging-loki-query-frontend + - logging-view-plugin + + ### see JIRA LOG-5431 if pods not running + tasks: + - name: "Verify Pods running" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in minio-dev" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12674" + pod_status_str: "Running" + pod_nspace: minio-dev + pod_list: + - minio + + tasks: + - name: "Run pod running tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods have complete status in openstack" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12679" + pod_nspace: openstack + pod_status_str: "Completed" + pod_list: + - bootstrap-edpm-deployment-logging + - configure-network-edpm-deployment-logging + - configure-os-edpm-deployment-logging + - download-cache-edpm-deployment-logging + - install-certs-edpm-deployment-logging + - install-os-edpm-deployment-logging + - libvirt-edpm-deployment-logging + - logging-edpm-deployment-logging-openstack-edpm + - neutron-metadata-edpm-deployment-logging + - nova-custom-edpm-deployment + - ovn-edpm-deployment-logging + - reboot-os-edpm-deployment-logging + - repo-setup-edpm-deployment-logging + - run-os-edpm-deployment-logging + - ssh-known-hosts-edpm-deployment-logging + - telemetry-edpm-deployment-logging + - validate-network-edpm-deployment-logging + + tasks: + - name: "Run pods completed tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging services are running in openstack" + hosts: controller + gather_facts: no + vars: + service_test_id: "RHOSO-12749" + service_nspace: openstack + service_list: + - nova-internal + - nova-metadata-internal + - nova-novncproxy-cell1-public + - nova-public + + tasks: + - name: "Run Services tests" + ansible.builtin.import_role: + name: common diff --git a/ci/run_playbooks_logging.yml b/ci/run_playbooks_logging.yml new file mode 100644 index 000000000..1e144f4f1 --- /dev/null +++ b/ci/run_playbooks_logging.yml @@ -0,0 +1,57 @@ +--- +- name: "Run logging functional test playbooks" + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars_files: + - vars/common.yml + tasks: + - name: "Create log dir" + ansible.builtin.file: + path: "{{ logs_dir }}" + state: directory + mode: "0755" + + - name: "Set dynamic KUBECONFIG and PATH vars" + copy: + content: | + cifmw_openshift_kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + cifmw_path: "{{ cifmw_path }}" + vars_dir: "{{ fvt_dir }}/ci/vars" + dest: "{{ env_vars_file }}" + + - name: Include vars from the extra_vars files + ansible.builtin.include_vars: + dir: "{{ cifmw_basedir }}/artifacts/parameters" + + - name: Run telemetry-logging tests on OSP18 + block: + - name: "Run local logging tests" + ansible.builtin.shell: + cmd: | + ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v -e @"{{ env_vars_file }}" ci/logging_tests_computes.yml + chdir: "{{ fvt_dir }}" + register: output + + - name: Save ansible output to a file + ansible.builtin.copy: + content: "{{ output.stdout }}" + dest: "{{ logs_dir }}/ansible_run.log" + + - name: "Run cltplane logging tests" + ansible.builtin.shell: + cmd: | + ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v -e @"{{ env_vars_file }}" ci/logging_tests_local.yml + chdir: "{{ fvt_dir }}" + register: output2 + + - name: Save ansible output to a file + ansible.builtin.lineinfile: + line: "{{ output2.stdout }}" + path: "{{ logs_dir }}/ansible_run.log" + + - name: Include report result + ansible.builtin.include_tasks: + file: report_result.yml From d6ab53415d54b3a9e58be9f830dbfaab8004a712 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 9 Aug 2024 17:18:51 +0100 Subject: [PATCH 02/18] Try to use new hooks to run test playbooks directly --- ci/logging_tests_computes.yml | 3 +-- ci/run_playbooks_logging.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index b208ce0cc..007aa987f 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -32,7 +32,6 @@ tasks: - name: "Run file and container tests" - ansible.builtin.import_role: + ansible.builtin.import_role: name: common - diff --git a/ci/run_playbooks_logging.yml b/ci/run_playbooks_logging.yml index 1e144f4f1..35e0a0cbc 100644 --- a/ci/run_playbooks_logging.yml +++ b/ci/run_playbooks_logging.yml @@ -52,6 +52,6 @@ line: "{{ output2.stdout }}" path: "{{ logs_dir }}/ansible_run.log" - - name: Include report result + - name: Include report result ansible.builtin.include_tasks: file: report_result.yml From 5ed43ef85ab6cfa47fded39d9a67fe07a2cc82fd Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 13 Aug 2024 19:48:45 +0100 Subject: [PATCH 03/18] Fix syntax error in variable definition --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index eac54504b..c5c13be10 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -121,7 +121,7 @@ name: infrawatch/feature-verification-tests github-check: jobs: - - feature-verification-tests-noop + #- feature-verification-tests-noop - openstack-k8s-operators-content-provider: override-checkout: main irrelevant-files: *irrelevant_files From dc4433bf3f9e06efdef8f3af0a5124ef9d7da6b1 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 14 Aug 2024 13:51:27 +0100 Subject: [PATCH 04/18] Use ignore_errors in plays for test_logging The logging plugins we use need all the tests to run in order to report properly. Adding ignore_errors to the plays that run the tests allows all the tests to run even when one fails. Also added verbosity to the ansible config to get more info about the failing tests --- ci/logging_tests_computes.yml | 3 +-- ci/logging_tests_local.yml | 24 +++++++++--------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index 007aa987f..921da7af7 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -19,6 +19,7 @@ - name: "Verify ctlplane logging containers and files" hosts: computes gather_facts: no + ignore_errors: true vars: container_test_id: "RHOSO-12753" container_list: @@ -29,9 +30,7 @@ file_test_id: "RHOSO-12754" file_list: - /etc/rsyslog.d/10-telemetry.conf - tasks: - name: "Run file and container tests" ansible.builtin.import_role: name: common - diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index a2dee02ec..d20f45b59 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -1,6 +1,8 @@ +--- - name: Prepare Logging Tests hosts: controller gather_facts: false + ignore_errors: true vars: id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" environment: @@ -32,6 +34,7 @@ - name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" hosts: controller gather_facts: no + ignore_errors: true vars: proj_test_id: "RHOSO-12668" proj_list: @@ -39,8 +42,6 @@ - openshift - openstack-operators - openshift-logging - - endpoint_test_id: "RHOSO-12682" endpoint_list: - [nova,compute,public] @@ -60,7 +61,6 @@ - [neutron,network,public] - [neutron,network,internal] - cred_test_id: "RHOSO-12670" cred_list: - alertingrules.loki.grafana.com @@ -68,14 +68,12 @@ - recordingrules.loki.grafana.com - rulerconfigs.loki.grafana.com - node_test_id: "RHOSO-12671" node_list: - edpm-compute-0 - edpm-compute-1 - crc - pod_test_id: "RHOS0-12672" pod_status_str: "Running" pod_nspace: openstack-operators @@ -83,7 +81,6 @@ - telemetry-operator-controller-manager - dataplane-operator-controller-manager - service_test_id: "RHOSO-12675" service_nspace: openshift-logging service_list: @@ -104,36 +101,30 @@ - logging-loki-query-frontend-http - logging-view-plugin - openstack-logging - - manifest_test_id: "RHOSO-12677" manifest_list: - "loki-operator 2" - "loki-helm-operator 1" - - subscription_test_id: "RHOSO-12678" subscription_nspace: openshift-operators-redhat subscription_list: - loki-operator - tasks: - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests" ansible.builtin.import_role: name: common - - name: "Verify logging pods are running in openstack" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12752" pod_status_str: "Running" pod_nspace: openstack pod_list: - openstackclient - tasks: - name: "Verify Running Pods" ansible.builtin.import_role: @@ -143,6 +134,7 @@ - name: "Verify logging pods are running in openshift-operators-redhat" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12673" pod_status_str: "Running" @@ -159,6 +151,7 @@ - name: "Verify logging pods are running in openshift-logging" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12676" pod_status_str: "Running" @@ -185,6 +178,7 @@ - name: "Verify logging pods are running in minio-dev" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12674" pod_status_str: "Running" @@ -201,6 +195,7 @@ - name: "Verify logging pods have complete status in openstack" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12679" pod_nspace: openstack @@ -223,7 +218,6 @@ - ssh-known-hosts-edpm-deployment-logging - telemetry-edpm-deployment-logging - validate-network-edpm-deployment-logging - tasks: - name: "Run pods completed tests" ansible.builtin.import_role: @@ -233,6 +227,7 @@ - name: "Verify logging services are running in openstack" hosts: controller gather_facts: no + ignore_errors: true vars: service_test_id: "RHOSO-12749" service_nspace: openstack @@ -241,7 +236,6 @@ - nova-metadata-internal - nova-novncproxy-cell1-public - nova-public - tasks: - name: "Run Services tests" ansible.builtin.import_role: From c9e11f553bdb64603bdb434f074faae3fc93c5b5 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 16 Aug 2024 11:58:34 +0100 Subject: [PATCH 05/18] Update playbooks to remove login play --- ci/logging_tests_local.yml | 71 +++++++++++++++++++++----------------- ci/vars-logging-test.yml | 1 - 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index d20f45b59..82c9cce45 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -1,35 +1,40 @@ --- -- name: Prepare Logging Tests - hosts: controller - gather_facts: false - ignore_errors: true - vars: - id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - - tasks: - - name: "Log into OCP" - ansible.builtin.shell: - cmd: | - oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443 - failed_when: false - changed_when: false - - - name: "Create id rsa file for ctlplane access" - ansible.builtin.shell: - cmd: | - oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}" - register: rsa_results - failed_when: rsa_results.rc != 0 - changed_when: false - - - name: "Change files permissions" - ansible.builtin.shell: - cmd: | - chmod 600 "{{ id_rsa_path }}" - changed_when: false +#- name: Prepare Logging Tests +# hosts: controller +# gather_facts: true +# ignore_errors: false +# vars: +# id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" +# environment: +# KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" +# PATH: "{{ cifmw_path }}" +# +# tasks: +# # This needs to be updated; what do we use elsewhere? +# # Elsewhere, we assume that we're already logged in +# - name: "Log into OCP" +# ansible.builtin.shell: +# cmd: | +# oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443 +# failed_when: false +# changed_when: false +# +# # I don't think we need this, Zuul __should__ have taken care of this already +# - name: "Create id rsa file for ctlplane access" +# ansible.builtin.shell: +# cmd: | +# oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}" +# register: rsa_results +# failed_when: rsa_results.rc != 0 +# changed_when: false +# +# # (efoley): Once again, I don't think this is necessary; if it's not needed for zuul, because it's already done, but is needed before running this on other systems, then the README should be updated to reflect this; +# # A troubleshooting section can be added too so users can identify when they are missing these configuration steps +# - name: "Change files permissions" +# ansible.builtin.shell: +# cmd: | +# chmod 600 "{{ id_rsa_path }}" +# changed_when: false - name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" hosts: controller @@ -146,7 +151,9 @@ - name: "Verify Pods running" ansible.builtin.import_role: name: common - + # vars can be passed here, so we can use the same play for this and the next play. + # vars: + # - - name: "Verify logging pods are running in openshift-logging" hosts: controller diff --git a/ci/vars-logging-test.yml b/ci/vars-logging-test.yml index 89b17d92f..d76ecadf2 100644 --- a/ci/vars-logging-test.yml +++ b/ci/vars-logging-test.yml @@ -2,7 +2,6 @@ # pre_tests or post_tests are also options for when the FVT roles are run post_deploy_00_fvt_logging: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_all.yml" - config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" type: playbook post_deploy_99_collect_results: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml" From cceb3e21526f1df4873583b6c3576d63734c3e48 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 16 Aug 2024 20:39:11 +0100 Subject: [PATCH 06/18] [ci/logging_tests] Add KUBECONFIG and PATH to play environment --- ci/logging_tests_computes.yml | 3 +++ ci/logging_tests_local.yml | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index 921da7af7..38f61e5c6 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -20,6 +20,9 @@ hosts: computes gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: container_test_id: "RHOSO-12753" container_list: diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index 82c9cce45..cbf93e296 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -40,6 +40,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: proj_test_id: "RHOSO-12668" proj_list: @@ -124,6 +127,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: pod_test_id: "RHOSO-12752" pod_status_str: "Running" @@ -140,6 +146,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: pod_test_id: "RHOSO-12673" pod_status_str: "Running" @@ -159,6 +168,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: pod_test_id: "RHOSO-12676" pod_status_str: "Running" @@ -186,6 +198,11 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + + vars: pod_test_id: "RHOSO-12674" pod_status_str: "Running" @@ -203,6 +220,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: pod_test_id: "RHOSO-12679" pod_nspace: openstack @@ -235,6 +255,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: service_test_id: "RHOSO-12749" service_nspace: openstack From 2798e03210d74d30d83987797230ec9dc88c3151 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 19 Aug 2024 18:12:12 +0100 Subject: [PATCH 07/18] Add report_results to logging tests --- callback_plugins/custom_logger.py | 1 + ci/logging_tests_local.yml | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/callback_plugins/custom_logger.py b/callback_plugins/custom_logger.py index 50dd0e00f..e77ea8ae1 100644 --- a/callback_plugins/custom_logger.py +++ b/callback_plugins/custom_logger.py @@ -53,6 +53,7 @@ def playbook_on_stats(self, stats): self.log_summary_results(host) def log_task_result(self, host, result, task_name): + print("logging task result: %s - %s - %s" % (host, result, task_name)) # test_run_result.out only interested in the test tasks, not setup or debug. if "RHELOSP" in task_name or "RHOSO" in task_name or "UI" in task_name: if "RHELOSP" in task_name: diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index cbf93e296..2258ee2e4 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -1,15 +1,22 @@ --- -#- name: Prepare Logging Tests -# hosts: controller -# gather_facts: true -# ignore_errors: false +- name: Prepare Logging Tests + hosts: controller + gather_facts: true + ignore_errors: false # vars: # id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" # environment: # KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" # PATH: "{{ cifmw_path }}" -# -# tasks: + vars_files: + - vars/common.yml + tasks: + - name: Create log dir + ansible.builtin.file: + path: "{{ logs_dir }}/logging_local" + state: directory + mode: "0755" + # # This needs to be updated; what do we use elsewhere? # # Elsewhere, we assume that we're already logged in # - name: "Log into OCP" @@ -270,3 +277,11 @@ - name: "Run Services tests" ansible.builtin.import_role: name: common + +- name: Include report result + hosts: controller + tasks: + - ansible.builtin.include_tasks: + file: report_result.yml + vars: + test_dir: "{{ logs_dir }}/logging_local" From e2932776234274b240770dff86a22fe966bbde64 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 19 Aug 2024 21:09:53 +0100 Subject: [PATCH 08/18] remove trailing spaces --- ci/logging_tests_local.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index 2258ee2e4..e10cc1a60 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -41,7 +41,7 @@ # ansible.builtin.shell: # cmd: | # chmod 600 "{{ id_rsa_path }}" -# changed_when: false +# changed_when: false - name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" hosts: controller @@ -54,7 +54,7 @@ proj_test_id: "RHOSO-12668" proj_list: - openshift-openstack-infra - - openshift + - openshift - openstack-operators - openshift-logging endpoint_test_id: "RHOSO-12682" @@ -90,7 +90,7 @@ - crc pod_test_id: "RHOS0-12672" - pod_status_str: "Running" + pod_status_str: "Running" pod_nspace: openstack-operators pod_list: - telemetry-operator-controller-manager From 486336067ae1eba810f987c0ff912fe758f5f614 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 20 Aug 2024 16:59:01 +0100 Subject: [PATCH 09/18] [zuul][logging] Update var names for the telemetry_logging role --- ci/logging_tests_computes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index 38f61e5c6..393711c18 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -37,3 +37,4 @@ - name: "Run file and container tests" ansible.builtin.import_role: name: common + From ccce02846ba5e79b585d60e57e9958e4269bb13c Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 21 Aug 2024 15:31:33 +0100 Subject: [PATCH 10/18] debug: add print statements to custom_logger --- callback_plugins/custom_logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/callback_plugins/custom_logger.py b/callback_plugins/custom_logger.py index e77ea8ae1..50dd0e00f 100644 --- a/callback_plugins/custom_logger.py +++ b/callback_plugins/custom_logger.py @@ -53,7 +53,6 @@ def playbook_on_stats(self, stats): self.log_summary_results(host) def log_task_result(self, host, result, task_name): - print("logging task result: %s - %s - %s" % (host, result, task_name)) # test_run_result.out only interested in the test tasks, not setup or debug. if "RHELOSP" in task_name or "RHOSO" in task_name or "UI" in task_name: if "RHELOSP" in task_name: From 26776abe334034487a35e363d639f4fe634478d5 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 21 Aug 2024 15:47:26 +0100 Subject: [PATCH 11/18] add gather_facts: true so report result knos about logs_dir --- ci/logging_tests_local.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index e10cc1a60..b7e375120 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -280,6 +280,7 @@ - name: Include report result hosts: controller + gather_facts: true tasks: - ansible.builtin.include_tasks: file: report_result.yml From d3983de2c7272726ebf90f274e1859323eb8a7b1 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 21 Jan 2025 18:09:28 +0000 Subject: [PATCH 12/18] USe the zuul.yaml from master --- .zuul.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index c5c13be10..b44ab4740 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -75,21 +75,6 @@ required-projects: *required_projects irrelevant-files: *irrelevant_files -- job: - name: functional-logging-tests-osp18 - parent: telemetry-operator-multinode-logging - description: | - Run the logging functional test on osp18+patched version - vars: - cifmw_extras: - - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" - - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml" - - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml" - roles: - - zuul: github.com/openstack-k8s-operators/ci-framework - required-projects: *required_projects - irrelevant-files: *irrelevant_files - - job: name: feature-verification-tests-noop parent: noop @@ -121,7 +106,7 @@ name: infrawatch/feature-verification-tests github-check: jobs: - #- feature-verification-tests-noop + - feature-verification-tests-noop - openstack-k8s-operators-content-provider: override-checkout: main irrelevant-files: *irrelevant_files From f91ed696ce4483682780ea3b1ac290b014876bee Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 21 Jan 2025 18:12:25 +0000 Subject: [PATCH 13/18] use ci/vars-logging-test from master --- ci/vars-logging-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/vars-logging-test.yml b/ci/vars-logging-test.yml index d76ecadf2..89b17d92f 100644 --- a/ci/vars-logging-test.yml +++ b/ci/vars-logging-test.yml @@ -2,6 +2,7 @@ # pre_tests or post_tests are also options for when the FVT roles are run post_deploy_00_fvt_logging: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_all.yml" + config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" type: playbook post_deploy_99_collect_results: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml" From 2de5d0e656bcd6af7da634e797a7a5c1988ccb20 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 21 Jan 2025 18:18:53 +0000 Subject: [PATCH 14/18] Update the logging test playbooks * rename loggin_tests_local playbook to logging_tests_controller * Rename vars in ci/logging_tests_compute and loggin_tests_controller --- ci/logging_tests_computes.yml | 12 +- ci/logging_tests_controller.yml | 113 ++++++++++--- ci/logging_tests_local.yml | 288 -------------------------------- 3 files changed, 93 insertions(+), 320 deletions(-) delete mode 100644 ci/logging_tests_local.yml diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index 393711c18..bd7ca9f54 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -7,8 +7,8 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: - journal_test_id: "RHOSO-12681" - journal_list: + common_journal_test_id: "RHOSO-12681" + common_journal_list: - ceilometer_agent_compute - nova_compute tasks: @@ -24,14 +24,14 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: - container_test_id: "RHOSO-12753" - container_list: + common_container_test_id: "RHOSO-12753" + common_container_list: - ceilometer_agent_compute - ceilometer_agent_ipmi - node_exporter - file_test_id: "RHOSO-12754" - file_list: + common_file_test_id: "RHOSO-12754" + common_file_list: - /etc/rsyslog.d/10-telemetry.conf tasks: - name: "Run file and container tests" diff --git a/ci/logging_tests_controller.yml b/ci/logging_tests_controller.yml index 9c314a4e3..b018af3cc 100644 --- a/ci/logging_tests_controller.yml +++ b/ci/logging_tests_controller.yml @@ -7,22 +7,51 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: - common_pod_test_id: "RHOS0-12672" - common_pod_status_str: "Running" - common_pod_nspace: openstack-operators - common_pod_list: - - telemetry-operator-controller-manager - - openstack-operator-controller-manager - common_subscription_test_id: "RHOSO-12678" - common_subscription_nspace: openshift-operators-redhat - common_subscription_list: - - loki-operator common_project_test_id: "RHOSO-12668" common_project_list: - openshift-openstack-infra - openshift - openstack-operators - openshift-logging + common_endpoint_test_id: "RHOSO-12682" + common_endpoint_list: + - [nova,compute,public] + - [nova,compute,internal] + - [placement,placement,public] + - [placement,placement,internal] + - [swift,object-store,public] + - [swift,object-store,internal] + - [cinderv3,volumev3,public] + - [cinderv3,volumev3,internal] + - [barbican,key-manager,public] + - [barbican,key-manager,internal] + - [keystone,identity,public] + - [keystone,identity,internal] + - [glance,image,public] + - [glance,image,internal] + - [neutron,network,public] + - [neutron,network,internal] + + common_crd_test_id: "RHOSO-12670" + common_crd_list: + - alertingrules.loki.grafana.com + - lokistacks.loki.grafana.com + - recordingrules.loki.grafana.com + - rulerconfigs.loki.grafana.com + + common_node_test_id: "RHOSO-12671" + common_node_list: + - edpm-compute-0 + - edpm-compute-1 + - crc + + common_pod_test_id: "RHOS0-12672" + common_pod_status_str: "Running" + common_pod_nspace: openstack-operators + common_pod_list: + - telemetry-operator-controller-manager + - dataplane-operator-controller-manager + common_service_test_id: "RHOSO-12675" common_service_nspace: openshift-logging common_service_list: @@ -41,15 +70,22 @@ - logging-loki-querier-http - logging-loki-query-frontend-grpc - logging-loki-query-frontend-http + - logging-view-plugin - openstack-logging - common_file_test_id: "RHOSO-12754" - common_file_list: - - /etc/rsyslog.d/10-telemetry.conf + common_manifest_test_id: "RHOSO-12677" + common_manifest_list: + - "loki-operator 2" + - "loki-helm-operator 1" + common_subscription_test_id: "RHOSO-12678" + common_subscription_nspace: openshift-operators-redhat + common_subscription_list: + - loki-operator tasks: - - name: "Verify logging infrastructure components" + - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests" ansible.builtin.import_role: name: common + - name: "Verify logging pods are running in openstack" hosts: controller gather_facts: no @@ -82,6 +118,7 @@ common_pod_nspace: openshift-operators-redhat common_pod_list: - loki-operator-controller-manager + tasks: - name: "Verify Pods running" ansible.builtin.import_role: @@ -100,21 +137,23 @@ common_pod_nspace: openshift-logging common_pod_list: - cluster-logging-operator + - collector - logging-loki-compactor - logging-loki-distributor - #- logging-loki-gateway + - logging-loki-gateway - logging-loki-index-gateway - logging-loki-ingester - logging-loki-querier - logging-loki-query-frontend - - collector + - logging-view-plugin ### see JIRA LOG-5431 if pods not running tasks: - - name: "Verify Pods running" + - name: "Verify Pods running" ansible.builtin.import_role: name: common + - name: "Verify logging pods are running in minio-dev" hosts: controller gather_facts: no @@ -122,12 +161,15 @@ environment: KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" + + vars: common_pod_test_id: "RHOSO-12674" common_pod_status_str: "Running" common_pod_nspace: minio-dev common_pod_list: - minio + tasks: - name: "Run pod running tests" ansible.builtin.import_role: @@ -146,13 +188,30 @@ common_pod_nspace: openstack common_pod_status_str: "Completed" common_pod_list: - - logging-edpm-deployment-openstack-edpm-ipam + - bootstrap-edpm-deployment-logging + - configure-network-edpm-deployment-logging + - configure-os-edpm-deployment-logging + - download-cache-edpm-deployment-logging + - install-certs-edpm-deployment-logging + - install-os-edpm-deployment-logging + - libvirt-edpm-deployment-logging + - logging-edpm-deployment-logging-openstack-edpm + - neutron-metadata-edpm-deployment-logging + - nova-custom-edpm-deployment + - ovn-edpm-deployment-logging + - reboot-os-edpm-deployment-logging + - repo-setup-edpm-deployment-logging + - run-os-edpm-deployment-logging + - ssh-known-hosts-edpm-deployment-logging + - telemetry-edpm-deployment-logging + - validate-network-edpm-deployment-logging tasks: - name: "Run pods completed tests" ansible.builtin.import_role: name: common -- name: "Verify the crds exist" + +- name: "Verify logging services are running in openstack" hosts: controller gather_facts: no ignore_errors: true @@ -160,13 +219,15 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: - common_crd_test_id: "RHOSO-12670" - common_crd_list: - - alertingrules.loki.grafana.com - - lokistacks.loki.grafana.com - - recordingrules.loki.grafana.com - - rulerconfigs.loki.grafana.com + common_service_test_id: "RHOSO-12749" + common_service_nspace: openstack + common_service_list: + - nova-internal + - nova-metadata-internal + - nova-novncproxy-cell1-public + - nova-public tasks: - - name: "Run the crds tests" + - name: "Run Services tests" ansible.builtin.import_role: name: common + diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml deleted file mode 100644 index b7e375120..000000000 --- a/ci/logging_tests_local.yml +++ /dev/null @@ -1,288 +0,0 @@ ---- -- name: Prepare Logging Tests - hosts: controller - gather_facts: true - ignore_errors: false -# vars: -# id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" -# environment: -# KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" -# PATH: "{{ cifmw_path }}" - vars_files: - - vars/common.yml - tasks: - - name: Create log dir - ansible.builtin.file: - path: "{{ logs_dir }}/logging_local" - state: directory - mode: "0755" - -# # This needs to be updated; what do we use elsewhere? -# # Elsewhere, we assume that we're already logged in -# - name: "Log into OCP" -# ansible.builtin.shell: -# cmd: | -# oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443 -# failed_when: false -# changed_when: false -# -# # I don't think we need this, Zuul __should__ have taken care of this already -# - name: "Create id rsa file for ctlplane access" -# ansible.builtin.shell: -# cmd: | -# oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}" -# register: rsa_results -# failed_when: rsa_results.rc != 0 -# changed_when: false -# -# # (efoley): Once again, I don't think this is necessary; if it's not needed for zuul, because it's already done, but is needed before running this on other systems, then the README should be updated to reflect this; -# # A troubleshooting section can be added too so users can identify when they are missing these configuration steps -# - name: "Change files permissions" -# ansible.builtin.shell: -# cmd: | -# chmod 600 "{{ id_rsa_path }}" -# changed_when: false - -- name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" - hosts: controller - gather_facts: no - ignore_errors: true - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - vars: - proj_test_id: "RHOSO-12668" - proj_list: - - openshift-openstack-infra - - openshift - - openstack-operators - - openshift-logging - endpoint_test_id: "RHOSO-12682" - endpoint_list: - - [nova,compute,public] - - [nova,compute,internal] - - [placement,placement,public] - - [placement,placement,internal] - - [swift,object-store,public] - - [swift,object-store,internal] - - [cinderv3,volumev3,public] - - [cinderv3,volumev3,internal] - - [barbican,key-manager,public] - - [barbican,key-manager,internal] - - [keystone,identity,public] - - [keystone,identity,internal] - - [glance,image,public] - - [glance,image,internal] - - [neutron,network,public] - - [neutron,network,internal] - - cred_test_id: "RHOSO-12670" - cred_list: - - alertingrules.loki.grafana.com - - lokistacks.loki.grafana.com - - recordingrules.loki.grafana.com - - rulerconfigs.loki.grafana.com - - node_test_id: "RHOSO-12671" - node_list: - - edpm-compute-0 - - edpm-compute-1 - - crc - - pod_test_id: "RHOS0-12672" - pod_status_str: "Running" - pod_nspace: openstack-operators - pod_list: - - telemetry-operator-controller-manager - - dataplane-operator-controller-manager - - service_test_id: "RHOSO-12675" - service_nspace: openshift-logging - service_list: - - cluster-logging-operator-metrics - - logging-loki-compactor-grpc - - logging-loki-compactor-http - - logging-loki-distributor-grpc - - logging-loki-distributor-http - - logging-loki-gateway-http - - logging-loki-gossip-ring - - logging-loki-index-gateway-grpc - - logging-loki-index-gateway-http - - logging-loki-ingester-grpc - - logging-loki-ingester-http - - logging-loki-querier-grpc - - logging-loki-querier-http - - logging-loki-query-frontend-grpc - - logging-loki-query-frontend-http - - logging-view-plugin - - openstack-logging - manifest_test_id: "RHOSO-12677" - manifest_list: - - "loki-operator 2" - - "loki-helm-operator 1" - subscription_test_id: "RHOSO-12678" - subscription_nspace: openshift-operators-redhat - subscription_list: - - loki-operator - tasks: - - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests" - ansible.builtin.import_role: - name: common - - -- name: "Verify logging pods are running in openstack" - hosts: controller - gather_facts: no - ignore_errors: true - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - vars: - pod_test_id: "RHOSO-12752" - pod_status_str: "Running" - pod_nspace: openstack - pod_list: - - openstackclient - tasks: - - name: "Verify Running Pods" - ansible.builtin.import_role: - name: common - - -- name: "Verify logging pods are running in openshift-operators-redhat" - hosts: controller - gather_facts: no - ignore_errors: true - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - vars: - pod_test_id: "RHOSO-12673" - pod_status_str: "Running" - pod_nspace: openshift-operators-redhat - pod_list: - - loki-operator-controller-manager - - tasks: - - name: "Verify Pods running" - ansible.builtin.import_role: - name: common - # vars can be passed here, so we can use the same play for this and the next play. - # vars: - # - - -- name: "Verify logging pods are running in openshift-logging" - hosts: controller - gather_facts: no - ignore_errors: true - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - vars: - pod_test_id: "RHOSO-12676" - pod_status_str: "Running" - pod_nspace: openshift-logging - pod_list: - - cluster-logging-operator - - collector - - logging-loki-compactor - - logging-loki-distributor - - logging-loki-gateway - - logging-loki-index-gateway - - logging-loki-ingester - - logging-loki-querier - - logging-loki-query-frontend - - logging-view-plugin - - ### see JIRA LOG-5431 if pods not running - tasks: - - name: "Verify Pods running" - ansible.builtin.import_role: - name: common - - -- name: "Verify logging pods are running in minio-dev" - hosts: controller - gather_facts: no - ignore_errors: true - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - - - vars: - pod_test_id: "RHOSO-12674" - pod_status_str: "Running" - pod_nspace: minio-dev - pod_list: - - minio - - tasks: - - name: "Run pod running tests" - ansible.builtin.import_role: - name: common - - -- name: "Verify logging pods have complete status in openstack" - hosts: controller - gather_facts: no - ignore_errors: true - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - vars: - pod_test_id: "RHOSO-12679" - pod_nspace: openstack - pod_status_str: "Completed" - pod_list: - - bootstrap-edpm-deployment-logging - - configure-network-edpm-deployment-logging - - configure-os-edpm-deployment-logging - - download-cache-edpm-deployment-logging - - install-certs-edpm-deployment-logging - - install-os-edpm-deployment-logging - - libvirt-edpm-deployment-logging - - logging-edpm-deployment-logging-openstack-edpm - - neutron-metadata-edpm-deployment-logging - - nova-custom-edpm-deployment - - ovn-edpm-deployment-logging - - reboot-os-edpm-deployment-logging - - repo-setup-edpm-deployment-logging - - run-os-edpm-deployment-logging - - ssh-known-hosts-edpm-deployment-logging - - telemetry-edpm-deployment-logging - - validate-network-edpm-deployment-logging - tasks: - - name: "Run pods completed tests" - ansible.builtin.import_role: - name: common - - -- name: "Verify logging services are running in openstack" - hosts: controller - gather_facts: no - ignore_errors: true - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - vars: - service_test_id: "RHOSO-12749" - service_nspace: openstack - service_list: - - nova-internal - - nova-metadata-internal - - nova-novncproxy-cell1-public - - nova-public - tasks: - - name: "Run Services tests" - ansible.builtin.import_role: - name: common - -- name: Include report result - hosts: controller - gather_facts: true - tasks: - - ansible.builtin.include_tasks: - file: report_result.yml - vars: - test_dir: "{{ logs_dir }}/logging_local" From 21945c646466a9b7f5963927e74402645f7b1799 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 22 Jan 2025 16:56:29 +0000 Subject: [PATCH 15/18] revert rename of telemetry_logging role vars --- ci/logging_tests_computes.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index bd7ca9f54..c87ce392b 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -7,8 +7,8 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: - common_journal_test_id: "RHOSO-12681" - common_journal_list: + journal_test_id: "RHOSO-12681" + journal_list: - ceilometer_agent_compute - nova_compute tasks: From 0f6b8a69dc62148eeeec0842df937dbc6645e582 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 22 Jan 2025 17:15:17 +0000 Subject: [PATCH 16/18] Resolve differences between master and this branch --- ci/logging_tests_controller.yml | 59 +++++++++++++++------------------ 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/ci/logging_tests_controller.yml b/ci/logging_tests_controller.yml index b018af3cc..7a35d51dd 100644 --- a/ci/logging_tests_controller.yml +++ b/ci/logging_tests_controller.yml @@ -7,6 +7,16 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: + common_pod_test_id: "RHOS0-12672" + common_pod_status_str: "Running" + common_pod_nspace: openstack-operators + common_pod_list: + - telemetry-operator-controller-manager + - openstack-operator-controller-manager + common_subscription_test_id: "RHOSO-12678" + common_subscription_nspace: openshift-operators-redhat + common_subscription_list: + - loki-operator common_project_test_id: "RHOSO-12668" common_project_list: - openshift-openstack-infra @@ -32,25 +42,12 @@ - [neutron,network,public] - [neutron,network,internal] - common_crd_test_id: "RHOSO-12670" - common_crd_list: - - alertingrules.loki.grafana.com - - lokistacks.loki.grafana.com - - recordingrules.loki.grafana.com - - rulerconfigs.loki.grafana.com - common_node_test_id: "RHOSO-12671" common_node_list: - edpm-compute-0 - edpm-compute-1 - crc - common_pod_test_id: "RHOS0-12672" - common_pod_status_str: "Running" - common_pod_nspace: openstack-operators - common_pod_list: - - telemetry-operator-controller-manager - - dataplane-operator-controller-manager common_service_test_id: "RHOSO-12675" common_service_nspace: openshift-logging @@ -70,22 +67,19 @@ - logging-loki-querier-http - logging-loki-query-frontend-grpc - logging-loki-query-frontend-http - - logging-view-plugin - openstack-logging + common_file_test_id: "RHOSO-12754" + common_file_list: + - /etc/rsyslog.d/10-telemetry.conf common_manifest_test_id: "RHOSO-12677" common_manifest_list: - "loki-operator 2" - "loki-helm-operator 1" - common_subscription_test_id: "RHOSO-12678" - common_subscription_nspace: openshift-operators-redhat - common_subscription_list: - - loki-operator tasks: - - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests" + - name: "Verify logging infrastructure components" ansible.builtin.import_role: name: common - - name: "Verify logging pods are running in openstack" hosts: controller gather_facts: no @@ -118,7 +112,6 @@ common_pod_nspace: openshift-operators-redhat common_pod_list: - loki-operator-controller-manager - tasks: - name: "Verify Pods running" ansible.builtin.import_role: @@ -137,23 +130,21 @@ common_pod_nspace: openshift-logging common_pod_list: - cluster-logging-operator - - collector - logging-loki-compactor - logging-loki-distributor - - logging-loki-gateway + #- logging-loki-gateway - logging-loki-index-gateway - logging-loki-ingester - logging-loki-querier - logging-loki-query-frontend - - logging-view-plugin + - collector ### see JIRA LOG-5431 if pods not running tasks: - - name: "Verify Pods running" + - name: "Verify Pods running" ansible.builtin.import_role: name: common - - name: "Verify logging pods are running in minio-dev" hosts: controller gather_facts: no @@ -161,15 +152,12 @@ environment: KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" - - vars: common_pod_test_id: "RHOSO-12674" common_pod_status_str: "Running" common_pod_nspace: minio-dev common_pod_list: - minio - tasks: - name: "Run pod running tests" ansible.builtin.import_role: @@ -188,6 +176,8 @@ common_pod_nspace: openstack common_pod_status_str: "Completed" common_pod_list: + - logging-edpm-deployment-openstack-edpm-ipam + # I think that we deliberately removed the rest of this list - bootstrap-edpm-deployment-logging - configure-network-edpm-deployment-logging - configure-os-edpm-deployment-logging @@ -210,8 +200,7 @@ ansible.builtin.import_role: name: common - -- name: "Verify logging services are running in openstack" +- name: "Verify the crds exist" hosts: controller gather_facts: no ignore_errors: true @@ -226,8 +215,14 @@ - nova-metadata-internal - nova-novncproxy-cell1-public - nova-public + common_crd_test_id: "RHOSO-12670" + common_crd_list: + - alertingrules.loki.grafana.com + - lokistacks.loki.grafana.com + - recordingrules.loki.grafana.com + - rulerconfigs.loki.grafana.com tasks: - - name: "Run Services tests" + - name: "Run Services and CRD tests" ansible.builtin.import_role: name: common From 9547d76e1f0baba0bde5bc3728b34c1e302a079d Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 22 Jan 2025 17:23:06 +0000 Subject: [PATCH 17/18] Remove the ci/run_playbooks_logging.yml playbook It was replaced with the hook and the logging_tests_all playbook --- ci/run_playbooks_logging.yml | 57 ------------------------------------ 1 file changed, 57 deletions(-) delete mode 100644 ci/run_playbooks_logging.yml diff --git a/ci/run_playbooks_logging.yml b/ci/run_playbooks_logging.yml deleted file mode 100644 index 35e0a0cbc..000000000 --- a/ci/run_playbooks_logging.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -- name: "Run logging functional test playbooks" - hosts: "{{ cifmw_target_hook_host | default('localhost') }}" - gather_facts: true - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - vars_files: - - vars/common.yml - tasks: - - name: "Create log dir" - ansible.builtin.file: - path: "{{ logs_dir }}" - state: directory - mode: "0755" - - - name: "Set dynamic KUBECONFIG and PATH vars" - copy: - content: | - cifmw_openshift_kubeconfig: "{{ cifmw_openshift_kubeconfig }}" - cifmw_path: "{{ cifmw_path }}" - vars_dir: "{{ fvt_dir }}/ci/vars" - dest: "{{ env_vars_file }}" - - - name: Include vars from the extra_vars files - ansible.builtin.include_vars: - dir: "{{ cifmw_basedir }}/artifacts/parameters" - - - name: Run telemetry-logging tests on OSP18 - block: - - name: "Run local logging tests" - ansible.builtin.shell: - cmd: | - ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v -e @"{{ env_vars_file }}" ci/logging_tests_computes.yml - chdir: "{{ fvt_dir }}" - register: output - - - name: Save ansible output to a file - ansible.builtin.copy: - content: "{{ output.stdout }}" - dest: "{{ logs_dir }}/ansible_run.log" - - - name: "Run cltplane logging tests" - ansible.builtin.shell: - cmd: | - ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v -e @"{{ env_vars_file }}" ci/logging_tests_local.yml - chdir: "{{ fvt_dir }}" - register: output2 - - - name: Save ansible output to a file - ansible.builtin.lineinfile: - line: "{{ output2.stdout }}" - path: "{{ logs_dir }}/ansible_run.log" - - - name: Include report result - ansible.builtin.include_tasks: - file: report_result.yml From 5eadc5ff88a495daa1d51fc4402faaa6f1d79909 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 10 Feb 2025 21:29:31 +0000 Subject: [PATCH 18/18] Update ci/logging_tests_computes.yml --- ci/logging_tests_computes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index f91be5200..17a48eab0 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -19,7 +19,7 @@ ansible.builtin.import_role: name: telemetry_logging - - name: "Run the tests from the common role" + - name: "Run the tests from the common role" ansible.builtin.import_role: name: common