Merge pull request #64406 from abrennan89/log-forwarding

OBSDOCS-206: Clean up log collection and forwarding docs

Author: abrennan89

_topic_maps/_topic_map.yml

@@ -2580,8 +2580,6 @@ Topics:
   Topics:
   - Name: About the Cluster Logging custom resource
     File: cluster-logging-configuring-cr
-  - Name: Configuring the logging collector
-    File: cluster-logging-collector
   - Name: Configuring the log store
     File: cluster-logging-log-store
   - Name: Configuring the log visualizer
@@ -2603,17 +2601,17 @@ Topics:
 - Name: Viewing cluster logs in Kibana
   File: cluster-logging-visualizer
   Distros: openshift-enterprise,openshift-origin
-- Name: Forwarding logs to third party systems
-  File: cluster-logging-external
-  Distros: openshift-enterprise,openshift-origin
-- Name: Enabling JSON logging
-  File: cluster-logging-enabling-json-logging
-- Name: Collecting and storing Kubernetes events
-  File: cluster-logging-eventrouter
-  Distros: openshift-enterprise,openshift-origin
-# - Name: Forwarding logs using ConfigMaps
-#  File: cluster-logging-external-configmap
-#  Distros: openshift-enterprise,openshift-origin
+- Name: Log collection and forwarding
+  Dir: log_collection_forwarding
+  Topics:
+  - Name: About log forwarding
+    File: log-forwarding
+  - Name: Enabling JSON log forwarding
+    File: cluster-logging-enabling-json-logging
+  - Name: Configuring the logging collector
+    File: cluster-logging-collector
+  - Name: Collecting and storing Kubernetes events
+    File: cluster-logging-eventrouter
 - Name: Updating Logging
   File: cluster-logging-upgrading
 - Name: Viewing cluster dashboards

_topic_maps/_topic_map_osd.yml

@@ -374,8 +374,6 @@ Topics:
   Topics:
   - Name: About the Cluster Logging custom resource
     File: cluster-logging-configuring-cr
-  - Name: Configuring the logging collector
-    File: cluster-logging-collector
   - Name: Configuring the log store
     File: cluster-logging-log-store
   - Name: Configuring the log visualizer
@@ -397,15 +395,17 @@ Topics:
 - Name: Viewing cluster logs in Kibana
   File: cluster-logging-visualizer
   Distros: openshift-dedicated
-- Name: Forwarding logs to third party systems
-  File: cluster-logging-external
-- Name: Enabling JSON logging
-  File: cluster-logging-enabling-json-logging
-- Name: Collecting and storing Kubernetes events
-  File: cluster-logging-eventrouter
-# - Name: Forwarding logs using ConfigMaps
-#  File: cluster-logging-external-configmap
-#  Distros: openshift-dedicated
+- Name: Log collection and forwarding
+  Dir: log_collection_forwarding
+  Topics:
+  - Name: About log forwarding
+    File: log-forwarding
+  - Name: Enabling JSON log forwarding
+    File: cluster-logging-enabling-json-logging
+  - Name: Configuring the logging collector
+    File: cluster-logging-collector
+  - Name: Collecting and storing Kubernetes events
+    File: cluster-logging-eventrouter
 - Name: Updating Logging
   File: cluster-logging-upgrading
 - Name: Viewing cluster dashboards

_topic_maps/_topic_map_rosa.yml

@@ -498,8 +498,6 @@ Topics:
   Topics:
   - Name: About the Cluster Logging custom resource
     File: cluster-logging-configuring-cr
-  - Name: Configuring the logging collector
-    File: cluster-logging-collector
   - Name: Configuring the log store
     File: cluster-logging-log-store
   - Name: Configuring the log visualizer
@@ -520,14 +518,17 @@ Topics:
   File: viewing-resource-logs
 - Name: Viewing cluster logs in Kibana
   File: cluster-logging-visualizer
-- Name: Forwarding logs to third party systems
-  File: cluster-logging-external
-- Name: Enabling JSON logging
-  File: cluster-logging-enabling-json-logging
-- Name: Collecting and storing Kubernetes events
-  File: cluster-logging-eventrouter
-# - Name: Forwarding logs using ConfigMaps
-#  File: cluster-logging-external-configmap
+- Name: Log collection and forwarding
+  Dir: log_collection_forwarding
+  Topics:
+  - Name: About log forwarding
+    File: log-forwarding
+  - Name: Enabling JSON log forwarding
+    File: cluster-logging-enabling-json-logging
+  - Name: Configuring the logging collector
+    File: cluster-logging-collector
+  - Name: Collecting and storing Kubernetes events
+    File: cluster-logging-eventrouter
 - Name: Updating Logging
   File: cluster-logging-upgrading
 - Name: Viewing cluster dashboards

_unused_topics/cluster-logging-log-forwarding-disable.adoc

@@ -6,9 +6,9 @@ include::_attributes/attributes-openshift-dedicated.adoc[]
 
 toc::[]
 
-You can add, access, and remove add-on services for your {product-title} 
+You can add, access, and remove add-on services for your {product-title}
 ifdef::openshift-rosa[]
-(ROSA) 
+(ROSA)
 endif::openshift-rosa[]
 cluster by using {cluster-manager-first}.
 
@@ -25,5 +25,5 @@ include::modules/deleting-service.adoc[leveloffset=+1]
 ifdef::openshift-rosa[]
 [role="_additional-resources"]
 == Additional resources
-* For information about the `cluster-logging-operator` and the AWS CloudWatch log forwarding service, see xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-cloudwatch_cluster-logging-external[Forwarding logs to Amazon CloudWatch]
+* xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-cloudwatch_log-forwarding[Forwarding logs to Amazon CloudWatch]
 endif::[]

adding_service_cluster/adding-service.adoc

@@ -16,7 +16,7 @@ include::modules/aws-cloudwatch.adoc[leveloffset=+1]
 .Additional resources
 
 * link:https://aws.amazon.com/cloudwatch/[Amazon CloudWatch product information]
-* xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-cloudwatch_cluster-logging-external[Forwarding logs to Amazon CloudWatch]
+* xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-cloudwatch_log-forwarding[Forwarding logs to Amazon CloudWatch]
 
 include::modules/osd-rhoam.adoc[leveloffset=+1]
 

adding_service_cluster/rosa-available-services.adoc

@@ -34,11 +34,11 @@ include::modules/cluster-logging-deploy-console.adoc[leveloffset=+1]
 
 ifdef::openshift-enterprise,openshift-origin[]
 * xref:../operators/admin/olm-adding-operators-to-cluster.adoc#olm-installing-operators-from-operatorhub_olm-adding-operators-to-a-cluster[Installing Operators from the OperatorHub]
-* xref:../logging/config/cluster-logging-collector.adoc#cluster-logging-removing-unused-components-if-no-elasticsearch_cluster-logging-collector[Removing unused components if you do not use the default Elasticsearch log store]
+* xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-removing-unused-components-if-no-elasticsearch_cluster-logging-log-store[Removing unused components if you do not use the default Elasticsearch log store]
 endif::[]
 ifdef::openshift-rosa,openshift-dedicated[]
 * link:https://docs.openshift.com/container-platform/latest/operators/admin/olm-adding-operators-to-cluster.html[Installing Operators from OperatorHub]
-* link:https://docs.openshift.com/container-platform/latest/logging/config/cluster-logging-collector.html#cluster-logging-removing-unused-components-if-no-elasticsearch_cluster-logging-collector[Removing unused components if you do not use the default Elasticsearch log store]
+* link:https://docs.openshift.com/container-platform/latest/logging/config/cluster-logging-log-store.html#cluster-logging-removing-unused-components-if-no-elasticsearch_cluster-logging-log-store[Removing unused components if you do not use the default Elasticsearch log store]
 endif::[]
 
 == Post-installation tasks

logging/cluster-logging-deploying.adoc

@@ -519,8 +519,8 @@ In OpenShift Logging 5.3, the legacy methods of forwarding logs to Syslog and Fl
 
 Instead, use the following non-legacy methods:
 
-* xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-fluentd_cluster-logging-external[Forwarding logs using the Fluentd forward protocol]
-* xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-syslog_cluster-logging-external[Forwarding logs using the syslog protocol]
+* xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-fluentd_log-forwarding[Forwarding logs using the Fluentd forward protocol]
+* xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-syslog_log-forwarding[Forwarding logs using the syslog protocol]
 
 [id="openshift-logging-5-3-0-legacy-forwarding-config_{context}"]
 ==== Configuration mechanisms for legacy forwarding methods have been removed
@@ -980,13 +980,13 @@ This release includes link:https://access.redhat.com/errata/RHBA-2021:3393[RHBA-
 [id="openshift-logging-5-2-0-new-features-and-enhancements_{context}"]
 === New features and enhancements
 
-* With this update, you can forward log data to Amazon CloudWatch, which provides application and infrastructure monitoring. For more information, see xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-cloudwatch_cluster-logging-external[Forwarding logs to Amazon CloudWatch]. (link:https://issues.redhat.com/browse/LOG-1173[LOG-1173])
+* With this update, you can forward log data to Amazon CloudWatch, which provides application and infrastructure monitoring. For more information, see xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-cloudwatch_log-forwarding[Forwarding logs to Amazon CloudWatch]. (link:https://issues.redhat.com/browse/LOG-1173[LOG-1173])
 
-* With this update, you can forward log data to Loki, a horizontally scalable, highly available, multi-tenant log aggregation system. For more information, see xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-loki_cluster-logging-external[Forwarding logs to Loki]. (link:https://issues.redhat.com/browse/LOG-684[LOG-684])
+* With this update, you can forward log data to Loki, a horizontally scalable, highly available, multi-tenant log aggregation system. For more information, see xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-loki_log-forwarding[Forwarding logs to Loki]. (link:https://issues.redhat.com/browse/LOG-684[LOG-684])
 
-* With this update, if you use the Fluentd forward protocol to forward log data over a TLS-encrypted connection, now you can use a password-encrypted private key file and specify the passphrase in the Cluster Log Forwarder configuration. For more information, see xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-fluentd_cluster-logging-external[Forwarding logs using the Fluentd forward protocol]. (link:https://issues.redhat.com/browse/LOG-1525[LOG-1525])
+* With this update, if you use the Fluentd forward protocol to forward log data over a TLS-encrypted connection, now you can use a password-encrypted private key file and specify the passphrase in the Cluster Log Forwarder configuration. For more information, see xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-fluentd_log-forwarding[Forwarding logs using the Fluentd forward protocol]. (link:https://issues.redhat.com/browse/LOG-1525[LOG-1525])
 
-* This enhancement enables you to use a username and password to authenticate a log forwarding connection to an external Elasticsearch instance. For example, if you cannot use mutual TLS (mTLS) because a third-party operates the Elasticsearch instance, you can use HTTP or HTTPS and set a secret that contains the username and password. For more information, see xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-es_cluster-logging-external[Forwarding logs to an external Elasticsearch instance]. (link:https://issues.redhat.com/browse/LOG-1022[LOG-1022])
+* This enhancement enables you to use a username and password to authenticate a log forwarding connection to an external Elasticsearch instance. For example, if you cannot use mutual TLS (mTLS) because a third-party operates the Elasticsearch instance, you can use HTTP or HTTPS and set a secret that contains the username and password. For more information, see xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-es_log-forwarding[Forwarding logs to an external Elasticsearch instance]. (link:https://issues.redhat.com/browse/LOG-1022[LOG-1022])
 
 * With this update, you can collect OVN network policy audit logs for forwarding to a logging server. (link:https://issues.redhat.com/browse/LOG-1526[LOG-1526])
 
@@ -1072,9 +1072,9 @@ From {product-title} 4.6 to the present, forwarding logs by using the following
 
 Instead, use the following non-legacy methods:
 
-* xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-fluentd_cluster-logging-external[Forwarding logs using the Fluentd forward protocol]
+* xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-fluentd_log-forwarding[Forwarding logs using the Fluentd forward protocol]
 
-* xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-syslog_cluster-logging-external[Forwarding logs using the syslog protocol]
+* xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-syslog_log-forwarding[Forwarding logs using the syslog protocol]
 
 [id="openshift-logging-5-2-0-CVEs_{context}"]
 === CVEs

logging/cluster-logging-release-notes.adoc

@@ -19,13 +19,8 @@ on using the interface, see the link:https://www.elastic.co/guide/en/kibana/6.8/
 
 [NOTE]
 ====
-The audit logs are not stored in the internal {product-title} Elasticsearch instance by default. To view the audit logs in Kibana, you must use the xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-store[Log Forwarding API] to configure a pipeline that uses the `default` output for audit logs.
+The audit logs are not stored in the internal {product-title} Elasticsearch instance by default. To view the audit logs in Kibana, you must use the xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-log-store[Log Forwarding API] to configure a pipeline that uses the `default` output for audit logs.
 ====
 
-// The following include statements pull in the module files that comprise
-// the assembly. Include any combination of concept, procedure, or reference
-// modules required to cover the user story. You can also include other
-// assemblies.
-
 include::modules/cluster-logging-visualizer-indices.adoc[leveloffset=+1]
 include::modules/cluster-logging-visualizer-kibana.adoc[leveloffset=+1]

logging/cluster-logging-visualizer.adoc

@@ -12,7 +12,7 @@ As a cluster administrator, you can deploy the {logging} to aggregate all the lo
 
 [NOTE]
 ====
-Because the internal {product-title} Elasticsearch log store does not provide secure storage for audit logs, audit logs are not stored in the internal Elasticsearch instance by default. If you want to send the audit logs to the default internal Elasticsearch log store, for example to view the audit logs in Kibana, you must use the Log Forwarding API as described in xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-store[Forward audit logs to the log store].
+Because the internal {product-title} Elasticsearch log store does not provide secure storage for audit logs, audit logs are not stored in the internal Elasticsearch instance by default. If you want to send the audit logs to the default internal Elasticsearch log store, for example to view the audit logs in Kibana, you must use the Log Forwarding API as described in xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-log-store[Forward audit logs to the log store].
 ====
 endif::[]
 
@@ -21,7 +21,7 @@ include::modules/logging-architecture-overview.adoc[leveloffset=+1]
 ifdef::openshift-rosa,openshift-dedicated[]
 include::modules/cluster-logging-cloudwatch.adoc[leveloffset=+1]
 .Next steps
-* See xref:../logging/cluster-logging-external.adoc#cluster-logging-collector-log-forward-cloudwatch_cluster-logging-external[Forwarding logs to Amazon CloudWatch] for instructions.
+* See xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-cloudwatch_log-forwarding[Forwarding logs to Amazon CloudWatch] for instructions.
 endif::[]
 
 include::modules/logging-common-terms.adoc[leveloffset=+1]
@@ -33,7 +33,7 @@ include::modules/cluster-logging-json-logging-about.adoc[leveloffset=+2]
 
 include::modules/cluster-logging-collecting-storing-kubernetes-events.adoc[leveloffset=+2]
 
-For information, see xref:../logging/cluster-logging-eventrouter.adoc#cluster-logging-eventrouter[About collecting and storing Kubernetes events].
+For information, see xref:../logging/log_collection_forwarding/cluster-logging-eventrouter.adoc#cluster-logging-eventrouter[About collecting and storing Kubernetes events].
 
 include::modules/cluster-logging-update-logging.adoc[leveloffset=+2]
 
@@ -57,22 +57,18 @@ include::modules/cluster-logging-about-components.adoc[leveloffset=+2]
 
 include::modules/cluster-logging-about-collector.adoc[leveloffset=+2]
 
-For information, see xref:../logging/config/cluster-logging-collector.adoc#cluster-logging-collector[Configuring the logging collector].
+For information, see xref:../logging/log_collection_forwarding/cluster-logging-collector.adoc#cluster-logging-collector[Configuring the logging collector].
 
 include::modules/cluster-logging-about-logstore.adoc[leveloffset=+2]
 
-For information, see xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-store[Configuring the log store].
+For information, see xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-log-store[Configuring the log store].
 
 include::modules/cluster-logging-about-visualizer.adoc[leveloffset=+2]
 
 For information, see xref:../logging/config/cluster-logging-visualizer.adoc#cluster-logging-visualizer[Configuring the log visualizer].
 
 include::modules/cluster-logging-eventrouter-about.adoc[leveloffset=+2]
 
-For information, see xref:../logging/cluster-logging-eventrouter.adoc#cluster-logging-eventrouter[Collecting and storing Kubernetes events].
-
-include::modules/cluster-logging-forwarding-about.adoc[leveloffset=+2]
-
-For information, see xref:../logging/cluster-logging-external.adoc#cluster-logging-external[Forwarding logs to third-party systems].
+For information, see xref:../logging/log_collection_forwarding/cluster-logging-eventrouter.adoc#cluster-logging-eventrouter[Collecting and storing Kubernetes events].
 
 include::modules/cluster-logging-feature-reference.adoc[leveloffset=+1]