diff --git a/.gitignore b/.gitignore index ca818ba0..f9f11598 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ recipes/common/bin/* training/cloud/examples training/instructlab/instructlab vector_dbs/milvus/volumes/milvus/* +**/mTLS/** diff --git a/observability/Containerfile b/observability/Containerfile new file mode 100644 index 00000000..7093d904 --- /dev/null +++ b/observability/Containerfile @@ -0,0 +1,26 @@ +# Embed Kepler systemd service +# and opentelemetry-collector systemd service with collector config. +# +# This is meant to be built as an add-on build to any other bootc/Containerfile or bootc/Containerfile.nocache +# See common/README_bootc_observability.md + +FROM quay.io/replace/centos-bootc:replace + +# Add Kepler service and opentelemetry-collector +RUN curl -L -O https://github.com/sustainable-computing-io/kepler/releases/download/v0.7.10/kepler.rpm.tar.gz && \ + tar xvzf kepler.rpm.tar.gz && \ + dnf -y install RPMS/noarch/container-kepler-0.7.10-1.noarch.rpm && \ + sudo systemctl enable container-kepler + +# See common/observability/mTLS for example files or adjust as necessary +COPY mTLS /usr/share/mTLS + +# Add OpenTelemetry Collector with configuration file +RUN sudo dnf copr enable -y frzifus/redhat-opentelemetry-collector-main && \ + sudo dnf install -y opentelemetry-collector && \ + sudo mkdir /etc/otelcol-logs && \ + sudo chown -R observability:observability /etc/otelcol-logs && \ + sudo systemctl enable opentelemetry-collector + +# See common/observability/kepler for example opentelemetry-collector config and adjust as necessary +COPY kepler/otelcol-config.yaml /etc/opentelemetry-collector/configs/10-custom.yaml diff --git a/observability/README.md b/observability/README.md new file mode 100644 index 00000000..ba20f6fe --- /dev/null +++ b/observability/README.md @@ -0,0 +1,26 @@ +# Bootc image with Kepler and OpenTelemetry Collector + +Follow this to build a bootc image with a chatbot, kepler, and opentelemetry-collector systemd services. +This example assumes mTLS authentication. The Containerfile assumes certificate and keys +are in `./mTLS` directory. Adjust the Containerfile accordingly. For an example setting up mTLS to send OTLP data to +an OpenTelemetry Collector running in OpenShift, see +[redhat-et/edge-ocp-observability](https://github.com/redhat-et/edge-ocp-observability/blob/main/observability-hub/mtls/generate_certs.sh). +[kepler/otelcol-config.yaml](./kepler/otelcol-config.yaml) is an example collector configuration. Replace the OTLP_EXPORT_ENDPOINT with a real value and +uncomment the otlphttp exporter in the metrics pipeline to export the data if sending to an external endpoint. +As is, OTLP metrics from Kepler will be visible in opentelemetry-collector logs with `systemctl status opentelemetry-collector`. + +Before running the below, update the Containerfile or populate a `mTLS` directory with expected files. +Also, update the opentelemetry collector configuration file to match your needs. + +```bash +cd recipes/natural_language_processing/chatbot +make BOOTC_IMAGE=quay.io/sallyom/centos-bootc:chatbot ARCH=x86_64 CONTAINERFILE=bootc/Containerfile.nocache bootc + +cd ../../../observability +podman build \ + --from quay.io/sallyom/centos-bootc:chatbot \ + --arch x86_64 \ + --security-opt label=disable \ + --cap-add SYS_ADMIN \ + -t quay.io/sallyom/centos-bootc:chatbot-kepler . +``` diff --git a/observability/kepler/otelcol-config.yaml b/observability/kepler/otelcol-config.yaml new file mode 100644 index 00000000..9bed076c --- /dev/null +++ b/observability/kepler/otelcol-config.yaml @@ -0,0 +1,57 @@ +--- +extensions: + file_storage: + # this dir must exist on filesystem where otelcollector is running + directory: /etc/otelcol-logs + timeout: 10s +receivers: + journald: + units: + - container-kepler + - opentelemetry-collector + #- ovn-kubernetes + #- crio + priority: info + prometheus: + config: + scrape_configs: + - job_name: 'rhel-kepler-local' + scrape_interval: 5s + static_configs: + - targets: ['0.0.0.0:8888'] # container-kepler service +processors: + batch: + memory_limiter: + check_interval: 5s + limit_percentage: 95 + spike_limit_percentage: 25 +exporters: + otlphttp: + endpoint: OTLP_EXPORT_ENDPOINT + tls: + insecure: false + cert_file: /usr/share/mTLS/server.crt + key_file: /usr/share/mTLS/server.key + ca_file: /usr/share/mTLS/ca.crt + otlp: + endpoint: OTLP_EXPORT_ENDPOINT + tls: + insecure: false + cert_file: /usr/share/mTLS/server.crt + key_file: /usr/share/mTLS/server.key + ca_file: /usr/share/mTLS/ca.crt + debug: + verbosity: detailed +service: + extensions: [file_storage] + telemetry: + metrics: + address: 0.0.0.0:9999 + pipelines: + logs: + receivers: [journald] + exporters: [debug] + metrics: + receivers: [prometheus] + #exporters: [otlphttp] + exporters: [debug]