Merge pull request #64493 from JoeAldinger/OSDOCS-6269-trouble-shooting

JoeAldinger · web-flow · commit 16c22e47f1b3 · 2023-10-18T11:52:37.000-04:00
/lgtm
diff --git a/modules/nw-ovn-kubernetes-alerts-cli.adoc b/modules/nw-ovn-kubernetes-alerts-cli.adoc
@@ -26,14 +26,11 @@ $ ALERT_MANAGER=$(oc get route alertmanager-main -n openshift-monitoring \
 -o jsonpath='{@.spec.host}')
 ----
 
-.. Issue a `curl` request to the alert manager route API with the correct authorization details requesting specific fields by running the following command:
+.. Issue a `curl` request to the alert manager route API by running the following command, replacing `$ALERT_MANAGER` with the URL of your `Alertmanager` instance:
 +
 [source,terminal]
 ----
-$ curl -s -k -H "Authorization: Bearer \
-$(oc create token prometheus-k8s -n openshift-monitoring)" \
-https://$ALERT_MANAGER/api/v1/alerts \
-| jq '.data[] | "\(.labels.severity) \(.labels.alertname) \(.labels.pod) \(.labels.container) \(.labels.endpoint) \(.labels.instance)"'
+$ curl -s -k -H "Authorization: Bearer $(oc create token prometheus-k8s -n openshift-monitoring)" https://$ALERT_MANAGER/api/v1/alerts | jq '.data[] | "\(.labels.severity) \(.labels.alertname) \(.labels.pod) \(.labels.container) \(.labels.endpoint) \(.labels.instance)"'
 ----
 
 . View alerting rules by running the following command:
diff --git a/modules/nw-ovn-kubernetes-change-log-levels.adoc b/modules/nw-ovn-kubernetes-change-log-levels.adoc
@@ -6,7 +6,7 @@
 [id="nw-ovn-kubernetes-change-log-levels_{context}"]
 = Changing the OVN-Kubernetes log levels
 
-The default log level for OVN-Kubernetes is 2. To debug OVN-Kubernetes set the log level to 5.
+The default log level for OVN-Kubernetes is 4. To debug OVN-Kubernetes, set the log level to 5.
 Follow this procedure to increase the log level of the OVN-Kubernetes to help you debug an issue.
 
 .Prerequisites
@@ -26,16 +26,16 @@ $ oc get po -o wide -n openshift-ovn-kubernetes
 .Example output
 [source,terminal]
 ----
-NAME                   READY   STATUS    RESTARTS      AGE   IP             NODE                           NOMINATED NODE   READINESS GATES
-ovnkube-master-84nc9   6/6     Running   0             50m   10.0.134.156   ip-10-0-134-156.ec2.internal   <none>           <none>
-ovnkube-master-gmlqv   6/6     Running   0             50m   10.0.209.180   ip-10-0-209-180.ec2.internal   <none>           <none>
-ovnkube-master-nhts2   6/6     Running   1 (48m ago)   50m   10.0.147.31    ip-10-0-147-31.ec2.internal    <none>           <none>
-ovnkube-node-2cbh8     5/5     Running   0             43m   10.0.217.114   ip-10-0-217-114.ec2.internal   <none>           <none>
-ovnkube-node-6fvzl     5/5     Running   0             50m   10.0.147.31    ip-10-0-147-31.ec2.internal    <none>           <none>
-ovnkube-node-f4lzz     5/5     Running   0             24m   10.0.146.76    ip-10-0-146-76.ec2.internal    <none>           <none>
-ovnkube-node-jf67d     5/5     Running   0             50m   10.0.209.180   ip-10-0-209-180.ec2.internal   <none>           <none>
-ovnkube-node-np9mf     5/5     Running   0             40m   10.0.165.191   ip-10-0-165-191.ec2.internal   <none>           <none>
-ovnkube-node-qjldg     5/5     Running   0             50m   10.0.134.156   ip-10-0-134-156.ec2.internal   <none>           <none>
+NAME                                     READY   STATUS    RESTARTS       AGE    IP           NODE                                       NOMINATED NODE   READINESS GATES
+ovnkube-control-plane-65497d4548-9ptdr   2/2     Running   2 (128m ago)   147m   10.0.0.3     ci-ln-3njdr9b-72292-5nwkp-master-0         <none>           <none>
+ovnkube-control-plane-65497d4548-j6zfk   2/2     Running   0              147m   10.0.0.5     ci-ln-3njdr9b-72292-5nwkp-master-2         <none>           <none>
+ovnkube-control-plane-65497d4548-k7xqt   2/2     Running   0              147m   10.0.0.4     ci-ln-3njdr9b-72292-5nwkp-master-1         <none>           <none>
+ovnkube-node-5dx44                       8/8     Running   0              146m   10.0.0.3     ci-ln-3njdr9b-72292-5nwkp-master-0         <none>           <none>
+ovnkube-node-dpfn4                       8/8     Running   0              146m   10.0.0.4     ci-ln-3njdr9b-72292-5nwkp-master-1         <none>           <none>
+ovnkube-node-kwc9l                       8/8     Running   0              134m   10.0.128.2   ci-ln-3njdr9b-72292-5nwkp-worker-a-2fjcj   <none>           <none>
+ovnkube-node-mcrhl                       8/8     Running   0              134m   10.0.128.4   ci-ln-3njdr9b-72292-5nwkp-worker-c-v9x5v   <none>           <none>
+ovnkube-node-nsct4                       8/8     Running   0              146m   10.0.0.5     ci-ln-3njdr9b-72292-5nwkp-master-2         <none>           <none>
+ovnkube-node-zrj9f                       8/8     Running   0              134m   10.0.128.3   ci-ln-3njdr9b-72292-5nwkp-worker-b-v78h7   <none>           <none>
 ----
 
 . Create a `ConfigMap` file similar to the following example and use a filename such as `env-overrides.yaml`:
@@ -49,12 +49,12 @@ metadata:
   name: env-overrides
   namespace: openshift-ovn-kubernetes
 data:
-  ip-10-0-217-114.ec2.internal: | <1>
+  ci-ln-3njdr9b-72292-5nwkp-master-0: | <1>
     # This sets the log level for the ovn-kubernetes node process:
     OVN_KUBE_LOG_LEVEL=5
     # You might also/instead want to enable debug logging for ovn-controller:
     OVN_LOG_LEVEL=dbg
-  ip-10-0-209-180.ec2.internal: |
+  ci-ln-3njdr9b-72292-5nwkp-master-2: |
     # This sets the log level for the ovn-kubernetes node process:
     OVN_KUBE_LOG_LEVEL=5
     # You might also/instead want to enable debug logging for ovn-controller:
@@ -86,16 +86,66 @@ configmap/env-overrides.yaml created
 [source,terminal]
 ----
 $ oc delete pod -n openshift-ovn-kubernetes \
---field-selector spec.nodeName=ip-10-0-217-114.ec2.internal -l app=ovnkube-node
+--field-selector spec.nodeName=ci-ln-3njdr9b-72292-5nwkp-master-0 -l app=ovnkube-node
 ----
 +
 [source,terminal]
 ----
 $ oc delete pod -n openshift-ovn-kubernetes \
---field-selector spec.nodeName=ip-10-0-209-180.ec2.internal -l app=ovnkube-node
+--field-selector spec.nodeName=ci-ln-3njdr9b-72292-5nwkp-master-2 -l app=ovnkube-node
 ----
 +
 [source,terminal]
 ----
-$ oc delete pod -n openshift-ovn-kubernetes -l app=ovnkube-master
+$ oc delete pod -n openshift-ovn-kubernetes -l app=ovnkube-node
+----
+
+. To verify that the `ConfigMap`file has been applied to all nodes for a specific pod, run the following command:
++
+[source,terminal]
+----
+$ oc logs -n openshift-ovn-kubernetes --all-containers --prefix ovnkube-node-<xxxx> | grep -E -m 10 '(Logging config:|vconsole|DBG)'
+----
++
+where:
+
+`<XXXX>`:: Specifies the random sequence of letters for a pod from the previous step.
++
+.Example output
+[source,terminal]
+----
+[pod/ovnkube-node-2cpjc/sbdb] + exec /usr/share/ovn/scripts/ovn-ctl --no-monitor '--ovn-sb-log=-vconsole:info -vfile:off -vPATTERN:console:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m' run_sb_ovsdb
+[pod/ovnkube-node-2cpjc/ovnkube-controller] I1012 14:39:59.984506   35767 config.go:2247] Logging config: {File: CNIFile:/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log LibovsdbFile:/var/log/ovnkube/libovsdb.log Level:5 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:0 ACLLoggingRateLimit:20}
+[pod/ovnkube-node-2cpjc/northd] + exec ovn-northd --no-chdir -vconsole:info -vfile:off '-vPATTERN:console:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m' --pidfile /var/run/ovn/ovn-northd.pid --n-threads=1
+[pod/ovnkube-node-2cpjc/nbdb] + exec /usr/share/ovn/scripts/ovn-ctl --no-monitor '--ovn-nb-log=-vconsole:info -vfile:off -vPATTERN:console:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m' run_nb_ovsdb
+[pod/ovnkube-node-2cpjc/ovn-controller] 2023-10-12T14:39:54.552Z|00002|hmap|DBG|lib/shash.c:114: 1 bucket with 6+ nodes, including 1 bucket with 6 nodes (32 nodes total across 32 buckets)
+[pod/ovnkube-node-2cpjc/ovn-controller] 2023-10-12T14:39:54.553Z|00003|hmap|DBG|lib/shash.c:114: 1 bucket with 6+ nodes, including 1 bucket with 6 nodes (64 nodes total across 64 buckets)
+[pod/ovnkube-node-2cpjc/ovn-controller] 2023-10-12T14:39:54.553Z|00004|hmap|DBG|lib/shash.c:114: 1 bucket with 6+ nodes, including 1 bucket with 7 nodes (32 nodes total across 32 buckets)
+[pod/ovnkube-node-2cpjc/ovn-controller] 2023-10-12T14:39:54.553Z|00005|reconnect|DBG|unix:/var/run/openvswitch/db.sock: entering BACKOFF
+[pod/ovnkube-node-2cpjc/ovn-controller] 2023-10-12T14:39:54.553Z|00007|reconnect|DBG|unix:/var/run/openvswitch/db.sock: entering CONNECTING
+[pod/ovnkube-node-2cpjc/ovn-controller] 2023-10-12T14:39:54.553Z|00008|ovsdb_cs|DBG|unix:/var/run/openvswitch/db.sock: SERVER_SCHEMA_REQUESTED -> SERVER_SCHEMA_REQUESTED at lib/ovsdb-cs.c:423
+----
+
+. Optional: Check the `ConfigMap` file has been applied by running the following command:
++
+[source,terminal]
+----
+for f in $(oc -n openshift-ovn-kubernetes get po -l 'app=ovnkube-node' --no-headers -o custom-columns=N:.metadata.name) ; do echo "---- $f ----" ; oc -n openshift-ovn-kubernetes exec -c ovnkube-controller $f --  pgrep -a -f  init-ovnkube-controller | grep -P -o '^.*loglevel\s+\d' ; done
+----
++
+.Example output
+[source,terminal]
+----
+---- ovnkube-node-2dt57 ----
+60981 /usr/bin/ovnkube --init-ovnkube-controller xpst8-worker-c-vmh5n.c.openshift-qe.internal --init-node xpst8-worker-c-vmh5n.c.openshift-qe.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4
+---- ovnkube-node-4zznh ----
+178034 /usr/bin/ovnkube --init-ovnkube-controller xpst8-master-2.c.openshift-qe.internal --init-node xpst8-master-2.c.openshift-qe.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4
+---- ovnkube-node-548sx ----
+77499 /usr/bin/ovnkube --init-ovnkube-controller xpst8-worker-a-fjtnb.c.openshift-qe.internal --init-node xpst8-worker-a-fjtnb.c.openshift-qe.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4
+---- ovnkube-node-6btrf ----
+73781 /usr/bin/ovnkube --init-ovnkube-controller xpst8-worker-b-p8rww.c.openshift-qe.internal --init-node xpst8-worker-b-p8rww.c.openshift-qe.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4
+---- ovnkube-node-fkc9r ----
+130707 /usr/bin/ovnkube --init-ovnkube-controller xpst8-master-0.c.openshift-qe.internal --init-node xpst8-master-0.c.openshift-qe.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 5
+---- ovnkube-node-tk9l4 ----
+181328 /usr/bin/ovnkube --init-ovnkube-controller xpst8-master-1.c.openshift-qe.internal --init-node xpst8-master-1.c.openshift-qe.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4
 ----
diff --git a/modules/nw-ovn-kubernetes-logs-cli.adoc b/modules/nw-ovn-kubernetes-logs-cli.adoc
@@ -36,32 +36,32 @@ For example:
 +
 [source,terminal]
 ----
-$ oc logs ovnkube-master-7h4q7 -n openshift-ovn-kubernetes
+$ oc logs ovnkube-node-5dx44 -n openshift-ovn-kubernetes
 ----
 +
 [source,terminal]
 ----
-$ oc logs -f ovnkube-master-7h4q7 -n openshift-ovn-kubernetes -c ovn-dbchecker
+$ oc logs -f ovnkube-node-5dx44 -c ovnkube-controller -n openshift-ovn-kubernetes
 ----
 +
 The contents of log files are printed out.
 
-. Examine the most recent entries in all the containers in the `ovnkube-master` pods:
+. Examine the most recent entries in all the containers in the `ovnkube-node` pods:
 +
 [source,terminal]
 ----
-$ for p in $(oc get pods --selector app=ovnkube-master -n openshift-ovn-kubernetes \
+$ for p in $(oc get pods --selector app=ovnkube-node -n openshift-ovn-kubernetes \
 -o jsonpath='{range.items[*]}{" "}{.metadata.name}'); \
 do echo === $p ===; for container in $(oc get pods -n openshift-ovn-kubernetes $p \
 -o json | jq -r '.status.containerStatuses[] | .name');do echo ---$container---; \
 oc logs -c $container $p -n openshift-ovn-kubernetes --tail=5; done; done
 ----
 
-. View the last 5 lines of every log in every container in an `ovnkube-master` pod using the following command:
+. View the last 5 lines of every log in every container in an `ovnkube-node` pod using the following command:
 +
 [source,terminal]
 ----
-$ oc logs -l app=ovnkube-master -n openshift-ovn-kubernetes --all-containers --tail 5
+$ oc logs -l app=ovnkube-node -n openshift-ovn-kubernetes --all-containers --tail 5
 ----
 
 
diff --git a/modules/nw-ovn-kubernetes-readiness-probes.adoc b/modules/nw-ovn-kubernetes-readiness-probes.adoc
@@ -6,7 +6,7 @@
 [id="nw-ovn-kubernetes-readiness-probes_{context}"]
 = Monitoring OVN-Kubernetes health by using readiness probes
 
-The `ovnkube-master` and `ovnkube-node` pods have containers configured with readiness probes.
+The `ovnkube-control-plane` and `ovnkube-node` pods have containers configured with readiness probes.
 
 .Prerequisites
 
@@ -16,25 +16,18 @@ The `ovnkube-master` and `ovnkube-node` pods have containers configured with rea
 
 .Procedure
 
-. Review the details of the `ovnkube-master` readiness probe by running the following command:
+. Review the details of the `ovnkube-node` readiness probe by running the following command:
 +
 [source,terminal]
 ----
-$ oc get pods -n openshift-ovn-kubernetes -l app=ovnkube-master \
+$ oc get pods -n openshift-ovn-kubernetes -l app=ovnkube-node \
 -o json | jq '.items[0].spec.containers[] | .name,.readinessProbe'
 ----
 +
-The readiness probe for the northbound and southbound database containers in the `ovnkube-master` pod checks for the health of the Raft cluster hosting the databases.
+The readiness probe for the northbound and southbound database containers in the `ovnkube-node` pod checks for the health of the databases and the `ovnkube-controller` container.
 
-. Review the details of the `ovnkube-node` readiness probe by running the following command:
-+
-[source,terminal]
-----
-$ oc get pods -n openshift-ovn-kubernetes -l app=ovnkube-master \
--o json | jq '.items[0].spec.containers[] | .name,.readinessProbe'
-----
 +
-The `ovnkube-node` container in the `ovnkube-node` pod has a readiness probe to verify the presence of the ovn-kubernetes CNI configuration file, the absence of which would indicate that the pod is not running or is not ready to accept requests to configure pods.
+The `ovnkube-node` container in the `ovnkube-node` pod has a readiness probe to verify the presence of the OVN-Kubernetes CNI configuration file, the absence of which would indicate that the pod is not running or is not ready to accept requests to configure pods.
 
 . Show all events including the probe failures, for the namespace by using the following command:
 +
@@ -43,11 +36,11 @@ The `ovnkube-node` container in the `ovnkube-node` pod has a readiness probe to
 $ oc get events -n openshift-ovn-kubernetes
 ----
 
-. Show the events for just this pod:
+. Show the events for just a specific pod:
 +
 [source,terminal]
 ----
-$ oc describe pod ovnkube-master-tp2z8 -n openshift-ovn-kubernetes
+$ oc describe pod ovnkube-node-9lqfk -n openshift-ovn-kubernetes
 ----
 
 . Show the messages and statuses from the cluster network operator:
@@ -57,11 +50,11 @@ $ oc describe pod ovnkube-master-tp2z8 -n openshift-ovn-kubernetes
 $ oc get co/network -o json | jq '.status.conditions[]'
 ----
 
-. Show the `ready` status of each container in `ovnkube-master` pods by running the following script:
+. Show the `ready` status of each container in `ovnkube-node` pods by running the following script:
 +
 [source,terminal]
 ----
-$ for p in $(oc get pods --selector app=ovnkube-master -n openshift-ovn-kubernetes \
+$ for p in $(oc get pods --selector app=ovnkube-node -n openshift-ovn-kubernetes \
 -o jsonpath='{range.items[*]}{" "}{.metadata.name}'); do echo === $p ===;  \
 oc get pods -n openshift-ovn-kubernetes $p -o json | jq '.status.containerStatuses[] | .name, .ready'; \
 done
diff --git a/networking/ovn_kubernetes_network_provider/ovn-kubernetes-troubleshooting-sources.adoc b/networking/ovn_kubernetes_network_provider/ovn-kubernetes-troubleshooting-sources.adoc
@@ -32,7 +32,6 @@ include::modules/nw-ovn-kubernetes-pod-connectivity-checks.adoc[leveloffset=+1]
 [id="additional-resources_ovn-kubernetes-sources-of-troubleshooting-information"]
 == Additional resources
 
-* link:https://access.redhat.com/solutions/5892971[How do I change the ovn-kubernetes loglevel in OpenShift 4?]
 * xref:../../networking/verifying-connectivity-endpoint.adoc#nw-pod-network-connectivity-implementation_verifying-connectivity-endpoint[Implementation of connection health checks]
 * xref:../../networking/verifying-connectivity-endpoint.adoc#nw-pod-network-connectivity-verify_verifying-connectivity-endpoint[Verifying network connectivity for an endpoint]
 
