Merge pull request #64935 from abrennan89/OBSDOCS-206

OBSDOCS-206: Add multi ClusterLogForwarder docs and clean up existing documentation

Author: abrennan89

_topic_maps/_topic_map.yml

@@ -2544,8 +2544,10 @@ Topics:
 - Name: Log collection and forwarding
   Dir: log_collection_forwarding
   Topics:
-  - Name: About log forwarding
+  - Name: About log collection and forwarding
     File: log-forwarding
+  - Name: Log output types
+    File: logging-output-types
   - Name: Enabling JSON log forwarding
     File: cluster-logging-enabling-json-logging
   - Name: Configuring the logging collector
@@ -2573,6 +2575,8 @@ Topics:
 - Name: Exported fields
   File: cluster-logging-exported-fields
   Distros: openshift-enterprise,openshift-origin
+- Name: Glossary
+  File: logging-common-terms
 ---
 Name: Monitoring
 Dir: monitoring

_topic_maps/_topic_map_osd.yml

@@ -577,8 +577,10 @@ Topics:
 - Name: Log collection and forwarding
   Dir: log_collection_forwarding
   Topics:
-  - Name: About log forwarding
+  - Name: About log collection and forwarding
     File: log-forwarding
+  - Name: Log output types
+    File: logging-output-types
   - Name: Enabling JSON log forwarding
     File: cluster-logging-enabling-json-logging
   - Name: Configuring the logging collector
@@ -604,6 +606,8 @@ Topics:
   File: cluster-logging-uninstall
 - Name: Exported fields
   File: cluster-logging-exported-fields
+- Name: Glossary
+  File: logging-common-terms
 ---
 Name: Monitoring
 Dir: monitoring

_topic_maps/_topic_map_rosa.yml

@@ -731,8 +731,10 @@ Topics:
 - Name: Log collection and forwarding
   Dir: log_collection_forwarding
   Topics:
-  - Name: About log forwarding
+  - Name: About log collection and forwarding
     File: log-forwarding
+  - Name: Log output types
+    File: logging-output-types
   - Name: Enabling JSON log forwarding
     File: cluster-logging-enabling-json-logging
   - Name: Configuring the logging collector
@@ -758,6 +760,8 @@ Topics:
   File: cluster-logging-uninstall
 - Name: Exported fields
   File: cluster-logging-exported-fields
+- Name: Glossary
+  File: logging-common-terms
 ---
 Name: Monitoring
 Dir: monitoring

logging/cluster-logging.adoc

@@ -2,33 +2,37 @@
 include::_attributes/common-attributes.adoc[]
 include::_attributes/attributes-openshift-dedicated.adoc[]
 [id="cluster-logging"]
-= Understanding the {logging-title}
+= About Logging
 :context: cluster-logging
 
 toc::[]
 
-ifdef::openshift-enterprise,openshift-rosa,openshift-dedicated,openshift-webscale,openshift-origin[]
-As a cluster administrator, you can deploy the {logging} to aggregate all the logs from your {product-title} cluster, such as node system audit logs, application container logs, and infrastructure logs. The {logging} aggregates these logs from throughout your cluster and stores them in a default log store. You can xref:../logging/cluster-logging-visualizer.adoc#cluster-logging-visualizer[use the Kibana web console to visualize log data].
+As a cluster administrator, you can deploy {logging} on an {product-title} cluster, and use it to collect and aggregate node system audit logs, application container logs, and infrastructure logs. You can forward logs to your chosen log outputs, including on-cluster, Red{nbsp}Hat managed log storage. You can also visualize your log data in the {product-title} web console, or xref:../logging/cluster-logging-visualizer.adoc#cluster-logging-visualizer[the Kibana web console], depending on your deployed log storage solution.
+
+[IMPORTANT]
+====
+The Kibana web console is now deprecated and will be removed in a future logging release.
+====
+
+{product-title} cluster administrators can deploy the {logging} by using Operators. For information, see xref:../logging/cluster-logging-deploying.adoc#cluster-logging-deploying[Installing the {logging-title}].
+
+The Operators are responsible for deploying, upgrading, and maintaining the {logging}. After the Operators are installed, you can create a `ClusterLogging` custom resource (CR) to schedule {logging} pods and other resources necessary to support the {logging}. You can also create a `ClusterLogForwarder` CR to specify which logs are collected, how they are transformed, and where they are forwarded to.
 
 [NOTE]
 ====
 Because the internal {product-title} Elasticsearch log store does not provide secure storage for audit logs, audit logs are not stored in the internal Elasticsearch instance by default. If you want to send the audit logs to the default internal Elasticsearch log store, for example to view the audit logs in Kibana, you must use the Log Forwarding API as described in xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-log-store[Forward audit logs to the log store].
 ====
-endif::[]
 
 include::modules/logging-architecture-overview.adoc[leveloffset=+1]
 
+include::modules/cluster-logging-about.adoc[leveloffset=+1]
+
 ifdef::openshift-rosa,openshift-dedicated[]
 include::modules/cluster-logging-cloudwatch.adoc[leveloffset=+1]
 .Next steps
 * See xref:../logging/log_collection_forwarding/log-forwarding.adoc#cluster-logging-collector-log-forward-cloudwatch_log-forwarding[Forwarding logs to Amazon CloudWatch] for instructions.
 endif::[]
 
-include::modules/logging-common-terms.adoc[leveloffset=+1]
-include::modules/cluster-logging-about.adoc[leveloffset=+1]
-
-For information, see xref:../logging/cluster-logging-deploying.adoc#cluster-logging-deploying[Installing the {logging-title}].
-
 include::modules/cluster-logging-json-logging-about.adoc[leveloffset=+2]
 
 include::modules/cluster-logging-collecting-storing-kubernetes-events.adoc[leveloffset=+2]

logging/config/cluster-logging-configuring-cr.adoc

@@ -8,9 +8,35 @@ toc::[]
 
 To configure {logging-title} you customize the `ClusterLogging` custom resource (CR).
 
-// The following include statements pull in the module files that comprise
-// the assembly. Include any combination of concept, procedure, or reference
-// modules required to cover the user story. You can also include other
-// assemblies.
-
 include::modules/cluster-logging-about-crd.adoc[leveloffset=+1]
+
+////
+// collecting this information here for a future PR
+
+If you want to specify collector resources or scheduling, you must create a `ClusterLogging` CR:
+
+.ClusterLogging resource example
+[source,yaml]
+----
+apiVersion: "logging.openshift.io/v1"
+kind: "ClusterLogging"
+metadata:
+  name: audit-collector <1>
+  namespace: openshift-kube-apiserver <2>
+spec:
+  collection:
+    type: "vector" <3>
+    resources:
+      limits:
+        memory: 2G
+# ...
+----
+<1> The name of the `ClusterLogging` CR must be the same as the `ClusterLogForwarder` CR.
+<2> The namespace of the `ClusterLogging` CR must be the same as the `ClusterLogForwarder` CR.
+<3> The collector type that you want to use. This example uses the Vector collector.
+
+[NOTE]
+====
+The relevant `spec` fields for this CR in multiple log forwarder mode are the `managmentState` and `collection` fields. All other `spec` fields are ignored.
+====
+////

logging/log_collection_forwarding/_attributes

@@ -0,0 +1 @@
+../../_attributes/

logging/log_collection_forwarding/images

@@ -0,0 +1 @@
+../../images/

logging/log_collection_forwarding/log-forwarding.adoc

@@ -2,42 +2,51 @@
 include::_attributes/common-attributes.adoc[]
 include::_attributes/attributes-openshift-dedicated.adoc[]
 [id="log-forwarding"]
-= About log forwarding
+= About log collection and forwarding
 :context: log-forwarding
 
 toc::[]
 
-By default, the {logging} sends container and infrastructure logs to the default internal log store defined in the `ClusterLogging` custom resource. However, it does not send audit logs to the internal store because it does not provide secure storage. If this default configuration meets your needs, you do not need to configure the Cluster Log Forwarder.
+Administrators can create `ClusterLogForwarder` resources that specify which logs are collected, how they are transformed, and where they are forwarded to.
 
-To send logs to other log aggregators, you use the {product-title} Cluster Log Forwarder. This API enables you to send container, infrastructure, and audit logs to specific endpoints within or outside your cluster. In addition, you can send different types of logs to various systems so that various individuals can access each type. You can also enable Transport Layer Security (TLS) support to send logs securely, as required by your organization.
+`ClusterLogForwarder` resources can be used up to forward container, infrastructure, and audit logs to specific endpoints within or outside of a cluster. Transport Layer Security (TLS) is supported so that log forwarders can be configured to send logs securely.
 
-[NOTE]
-====
-To send audit logs to the default internal Elasticsearch log store, use the Cluster Log Forwarder as described in xref:../../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-log-store[Forward audit logs to the log store].
-====
+Administrators can also authorize RBAC permissions that define which service accounts and users can access and forward which types of logs.
 
-When you forward logs externally, the {logging} creates or modifies a Fluentd config map to send logs using your desired protocols. You are responsible for configuring the protocol on the external log aggregator.
+////
+include::modules/log-forwarding-modes.adoc[leveloffset=+1]
 
-// unused files - either include or delete
-// cluster-logging-log-forwarding-disable.adoc
+[id="log-forwarding-enabling-multi-clf-mode"]
+== Enabling multi log forwarder mode for a cluster
 
-include::modules/cluster-logging-collector-log-forwarding-about.adoc[leveloffset=+1]
+To use multi log forwarder mode, you must create a service account and cluster role bindings for that service account. You can then reference the service account in the `ClusterLogForwarder` resource to control access permissions.
 
-include::modules/cluster-logging-forwarding-separate-indices.adoc[leveloffset=+1]
+include::modules/log-collection-rbac-permissions.adoc[leveloffset=+2]
 
-include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-1.adoc[leveloffset=+1]
+[role="_additional-resources"]
+.Additional resources
+ifdef::openshift-enterprise[]
+* xref:../../authentication/using-rbac.adoc#using-rbac[Using RBAC to define and apply permissions]
+* xref:../../authentication/using-service-accounts-in-applications.adoc#using-service-accounts-in-applications[Using service accounts in applications]
+endif::[]
+* link:https://kubernetes.io/docs/reference/access-authn-authz/rbac/[Using RBAC Authorization Kubernetes documentation]
 
-include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-2.adoc[leveloffset=+1]
+include::modules/logging-create-clf.adoc[leveloffset=+1]
+////
 
-include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-3.adoc[leveloffset=+1]
+[id="log-forwarding-audit-logs"]
+== Sending audit logs to the internal log store
 
-include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-4.adoc[leveloffset=+1]
+By default, the {logging} sends container and infrastructure logs to the default internal log store defined in the `ClusterLogging` custom resource. However, it does not send audit logs to the internal store because it does not provide secure storage. If this default configuration meets your needs, you do not need to configure the Cluster Log Forwarder.
 
-include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-5.adoc[leveloffset=+1]
+[NOTE]
+====
+To send audit logs to the internal Elasticsearch log store, use the Cluster Log Forwarder as described in xref:../../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-log-store[Forward audit logs to the log store].
+====
 
-include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-6.adoc[leveloffset=+1]
+include::modules/cluster-logging-collector-log-forwarding-about.adoc[leveloffset=+1]
 
-include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-7.adoc[leveloffset=+1]
+include::modules/cluster-logging-forwarding-separate-indices.adoc[leveloffset=+1]
 
 include::modules/cluster-logging-collector-log-forward-es.adoc[leveloffset=+1]
 

logging/log_collection_forwarding/logging-output-types.adoc

@@ -0,0 +1,37 @@
+:_content-type: ASSEMBLY
+include::_attributes/common-attributes.adoc[]
+include::_attributes/attributes-openshift-dedicated.adoc[]
+[id="logging-output-types"]
+= Log output types
+:context: logging-output-types
+
+toc::[]
+
+Log outputs specified in the `ClusterLogForwarder` CR can be any of the following types:
+
+`default`:: The on-cluster, Red{nbsp}Hat managed log store. You are not required to configure the default output.
++
+[NOTE]
+====
+If you configure a `default` output, you receive an error message, because the `default` output name is reserved for referencing the on-cluster, Red{nbsp}Hat managed log store.
+====
+`loki`:: Loki, a horizontally scalable, highly available, multi-tenant log aggregation system.
+`kafka`:: A Kafka broker. The `kafka` output can use a TCP or TLS connection.
+`elasticsearch`:: An external Elasticsearch instance. The `elasticsearch` output can use a TLS connection.
+`fluentdForward`:: An external log aggregation solution that supports Fluentd. This option uses the Fluentd *forward* protocols. The `fluentForward` output can use a TCP or TLS connection and supports shared-key authentication by providing a *shared_key* field in a secret. Shared-key authentication can be used with or without TLS.
++
+[IMPORTANT]
+====
+The `fluentdForward` output is only supported if you are using the Fluentd collector. It is not supported if you are using the Vector collector. If you are using the Vector collector, you can forward logs to Fluentd by using the `http` output.
+====
+`syslog`:: An external log aggregation solution that supports the syslog link:https://tools.ietf.org/html/rfc3164[RFC3164] or link:https://tools.ietf.org/html/rfc5424[RFC5424] protocols. The `syslog` output can use a UDP, TCP, or TLS connection.
+`cloudwatch`:: Amazon CloudWatch, a monitoring and log storage service hosted by Amazon Web Services (AWS).
+
+// supported outputs by version
+include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-7.adoc[leveloffset=+1]
+include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-6.adoc[leveloffset=+1]
+include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-5.adoc[leveloffset=+1]
+include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-4.adoc[leveloffset=+1]
+include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-3.adoc[leveloffset=+1]
+include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-2.adoc[leveloffset=+1]
+include::modules/cluster-logging-collector-log-forwarding-supported-plugins-5-1.adoc[leveloffset=+1]

logging/log_collection_forwarding/modules

@@ -0,0 +1 @@
+../../modules/