Skip to content

Commit abe28ea

Browse files
csatib02csatib02
committed
feat: move in node-exporter image
Signed-off-by: Bence Csati <[email protected]>
1 parent c11e2db commit abe28ea

File tree

12 files changed

+257
-25
lines changed

12 files changed

+257
-25
lines changed

‎.github/workflows/config-reloader.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ jobs:
103103
for tag in ${TAGS[@]}; do
104104
cosign verify "${tag}@${DIGEST}" \
105105
--rekor-url "https://rekor.sigstore.dev/" \
106-
--certificate-identity "https://github.com/${{ github.repository }}/.github/workflows/config-reloader-image.yaml@${{ github.ref }}" \
106+
--certificate-identity "https://github.com/${{ github.repository }}/.github/workflows/config-reloader.yaml@${{ github.ref }}" \
107107
--certificate-oidc-issuer "https://token.actions.githubusercontent.com" | jq
108108
done
109109

‎.github/workflows/dependency-images.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,14 @@ jobs:
6262
packages: write
6363
id-token: write
6464
security-events: write
65+
66+
node-exporter:
67+
name: Node exporter
68+
uses: ./.github/workflows/node-exporter.yaml
69+
with:
70+
publish: ${{ inputs.publish }}
71+
permissions:
72+
contents: read
73+
packages: write
74+
id-token: write
75+
security-events: write

‎.github/workflows/node-exporter.yaml

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
name: Node exporter
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
publish:
7+
description: Publish artifacts to the artifact store
8+
default: false
9+
required: false
10+
type: boolean
11+
12+
permissions:
13+
contents: read
14+
15+
jobs:
16+
node-exporter-image:
17+
name: Node exporter image
18+
runs-on: ubuntu-latest
19+
20+
permissions:
21+
contents: read
22+
packages: write
23+
id-token: write
24+
security-events: write
25+
26+
steps:
27+
- name: Checkout repository
28+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
29+
30+
- name: Set up QEMU
31+
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
32+
33+
- name: Set up Docker Buildx
34+
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
35+
36+
- name: Set up Cosign
37+
uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1
38+
if: ${{ inputs.publish }}
39+
40+
- name: Set image name
41+
id: image-name
42+
run: echo "value=ghcr.io/${{ github.repository }}/node-exporter" >> "$GITHUB_OUTPUT"
43+
44+
- name: Gather build metadata
45+
id: meta
46+
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
47+
with:
48+
images: ${{ steps.image-name.outputs.value }}
49+
flavor: |
50+
latest = false
51+
tags: |
52+
type=ref,event=branch
53+
type=ref,event=pr
54+
type=semver,pattern={{raw}}
55+
type=raw,value=latest,enable={{is_default_branch}}
56+
labels: |
57+
org.opencontainers.image.description=Node exporter image for the Logging operator.
58+
org.opencontainers.image.title=Logging operator Node exporter image
59+
org.opencontainers.image.authors=Kube logging authors
60+
org.opencontainers.image.documentation=https://kube-logging.dev/docs/
61+
62+
- name: Login to GitHub Container Registry
63+
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
64+
with:
65+
registry: ghcr.io
66+
username: ${{ github.actor }}
67+
password: ${{ github.token }}
68+
if: ${{ inputs.publish }}
69+
70+
- name: Build and push Node exporter image
71+
id: build
72+
uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
73+
with:
74+
context: images/node-exporter
75+
platforms: linux/amd64,linux/arm64
76+
tags: ${{ steps.meta.outputs.tags }}
77+
labels: ${{ steps.meta.outputs.labels }}
78+
cache-from: type=gha
79+
cache-to: type=gha,mode=max
80+
outputs: |
81+
type=image,push=${{ inputs.publish }},name=target,annotation-index.org.opencontainers.image.description=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.description'] }}
82+
type=oci,dest=image.tar,name=target,annotation-index.org.opencontainers.image.description=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.description'] }}
83+
84+
- name: Sign image with GitHub OIDC Token
85+
if: ${{ inputs.publish && github.repository_owner == 'kube-logging' }} # Check if the workflow is called by the same GitHub organization
86+
env:
87+
DIGEST: ${{ steps.build.outputs.digest }}
88+
TAGS: ${{ steps.meta.outputs.tags }}
89+
run: |
90+
images=""
91+
for tag in ${TAGS[@]}; do
92+
images+="${tag}@${DIGEST} "
93+
done
94+
95+
cosign sign --yes --rekor-url "https://rekor.sigstore.dev/" ${images}
96+
97+
- name: Verify signed image with cosign
98+
if: ${{ inputs.publish && github.repository_owner == 'kube-logging' }} # Check if the workflow is called by the same GitHub organization
99+
env:
100+
DIGEST: ${{ steps.build.outputs.digest }}
101+
TAGS: ${{ steps.meta.outputs.tags }}
102+
run: |
103+
for tag in ${TAGS[@]}; do
104+
cosign verify "${tag}@${DIGEST}" \
105+
--rekor-url "https://rekor.sigstore.dev/" \
106+
--certificate-identity "https://github.com/${{ github.repository }}/.github/workflows/node-exporter.yaml@${{ github.ref }}" \
107+
--certificate-oidc-issuer "https://token.actions.githubusercontent.com" | jq
108+
done
109+
110+
- name: Extract OCI tarball
111+
run: |
112+
mkdir -p image
113+
tar -xf image.tar -C image
114+
115+
- name: Run Trivy vulnerability scanner
116+
uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # 0.30.0
117+
env:
118+
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2
119+
TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1
120+
with:
121+
input: image
122+
format: sarif
123+
output: trivy-results.sarif
124+
125+
- name: Upload Trivy scan results as artifact
126+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
127+
with:
128+
name: "[${{ github.job }}] Trivy scan results"
129+
path: trivy-results.sarif
130+
retention-days: 5
131+
132+
- name: Upload Trivy scan results to GitHub Security tab
133+
uses: github/codeql-action/upload-sarif@6bb031afdd8eb862ea3fc1848194185e076637e5 # v3.28.11
134+
with:
135+
sarif_file: trivy-results.sarif

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ test-e2e-nodeps:
233233
CONFIG_RELOADER_IMAGE="${CONFIG_RELOADER_IMG}" \
234234
SYSLOG_NG_RELOADER_IMAGE="${SYSLOG_NG_RELOADER_IMG}" \
235235
FLUENTD_DRAIN_WATCH_IMAGE="${FLUENTD_DRAIN_WATCH_IMG}" \
236+
NODE_EXPORTER_IMAGE="${NODE_EXPORTER_IMG}" \
236237
FLUENTD_IMAGE="${FLUENTD_IMG}" \
237238
KIND_PATH="$(KIND)" \
238239
KIND_IMAGE="$(KIND_IMAGE)" \

‎config/samples/logging_logging_with_monitoring.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ spec:
1111
enableRecreateWorkloadOnImmutableFieldChange: true
1212
fluentd:
1313
bufferVolumeImage:
14-
repository: ghcr.io/kube-logging/node-exporter
14+
repository: ghcr.io/kube-logging/logging-operator/node-exporter
1515
bufferVolumeMetrics:
1616
prometheusRules: true
1717
serviceMonitor: true

‎images/node-exporter/Dockerfile

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
FROM ghcr.io/kube-logging/custom-runner:v0.12.0 AS custom-runner
2+
3+
FROM quay.io/prometheus/node-exporter:v1.9.0
4+
5+
COPY --from=custom-runner /runner /
6+
7+
USER root
8+
9+
RUN mkdir -p /prometheus/node_exporter/textfile_collector
10+
11+
COPY buffer-size.sh /prometheus/buffer-size.sh
12+
RUN chmod 0744 /prometheus/buffer-size.sh
13+
14+
WORKDIR /
15+
16+
ENTRYPOINT ["/runner"]

‎images/node-exporter/README.md

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Node Exporter Image
2+
3+
Node Exporter Image is a monitoring script that collects buffer size and file count metrics for Prometheus Node Exporter.
4+
5+
## Features
6+
7+
- Tracks disk usage of buffer files.
8+
- Reports buffer file count.
9+
- Generates Prometheus-compatible metrics.
10+
- Supports a configurable buffer path.
11+
12+
## Usage
13+
14+
Set the required environment variable before running:
15+
16+
```sh
17+
export BUFFER_PATH=/path/to/buffers # Optional, default is /buffers
18+
```
19+
20+
### Prometheus Integration
21+
22+
The script generates the following metrics for Prometheus Node Exporter:
23+
24+
- `node_buffer_size_bytes`: Deprecated metric for buffer disk usage.
25+
- `logging_buffer_size_bytes`: New metric for buffer disk usage, including the host label.
26+
- `logging_buffer_files`: Number of buffer files.
27+
28+
Metrics are stored in:
29+
30+
```sh
31+
/prometheus/node_exporter/textfile_collector/
32+
```
33+
34+
Ensure Node Exporter is configured to read from this directory.

‎images/node-exporter/buffer-size.sh

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/sh
2+
3+
[ -z "$BUFFER_PATH" ] && BUFFER_PATH=/buffers
4+
5+
while true; do
6+
# Deprecated: node_buffer_size_bytes will soon be replaced by logging_buffer_size_bytes
7+
# logging_buffer_size_bytes includes the host label
8+
echo "# HELP node_buffer_size_bytes Disk space used [deprecated]" > /prometheus/node_exporter/textfile_collector/buffer_size.prom.$$
9+
echo "# TYPE node_buffer_size_bytes gauge" >> /prometheus/node_exporter/textfile_collector/buffer_size.prom.$$
10+
du -sb ${BUFFER_PATH} | sed -ne 's/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/node_buffer_size_bytes{entity="\2"} \1/p' >> /prometheus/node_exporter/textfile_collector/buffer_size.prom.$$
11+
mv /prometheus/node_exporter/textfile_collector/buffer_size.prom.$$ /prometheus/node_exporter/textfile_collector/buffer_size.prom
12+
13+
echo "# HELP logging_buffer_size_bytes Disk space used" > /prometheus/node_exporter/textfile_collector/logging_buffer_size_bytes.prom.$$
14+
echo "# TYPE logging_buffer_size_bytes gauge" >> /prometheus/node_exporter/textfile_collector/logging_buffer_size_bytes.prom.$$
15+
du -sb ${BUFFER_PATH} | sed -ne 's/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/logging_buffer_size_bytes{entity="\2", host="'$(hostname)'"} \1/p' >> /prometheus/node_exporter/textfile_collector/logging_buffer_size_bytes.prom.$$
16+
mv /prometheus/node_exporter/textfile_collector/logging_buffer_size_bytes.prom.$$ /prometheus/node_exporter/textfile_collector/logging_buffer_size_bytes.prom
17+
18+
echo "# HELP logging_buffer_files File count" > /prometheus/node_exporter/textfile_collector/logging_buffer_files.prom.$$
19+
echo "# TYPE logging_buffer_files gauge" >> /prometheus/node_exporter/textfile_collector/logging_buffer_files.prom.$$
20+
echo -e "$(find "${BUFFER_PATH}" -type f 2>/dev/null | wc -l)\t${BUFFER_PATH}" | sed -ne 's/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/logging_buffer_files{entity="\2", host="'$(hostname)'"} \1/p' >> /prometheus/node_exporter/textfile_collector/logging_buffer_files.prom.$$
21+
mv /prometheus/node_exporter/textfile_collector/logging_buffer_files.prom.$$ /prometheus/node_exporter/textfile_collector/logging_buffer_files.prom
22+
23+
sleep 15
24+
done

‎images/syslog-ng-reloader/README.md

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,3 @@ It watches mounted volume dirs and notifies the target process changed files on
55
If changes exist - send webhook.
66

77
It is available as a Docker image at `ghcr.io/kube-logging/logging-operator/syslog-ng-reloader`
8-
9-
## License
10-
11-
The project is licensed under the [Apache License, Version 2.0](LICENSE).

‎pkg/sdk/logging/api/v1beta1/fluentd_types.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,11 @@ func (f *FluentdSpec) SetDefaults() error {
252252
f.BufferVolumeImage.Repository = DefaultFluentdBufferVolumeImageRepository
253253
}
254254
if f.BufferVolumeImage.Tag == "" {
255-
f.BufferVolumeImage.Tag = DefaultFluentdBufferVolumeImageTag
255+
if Version == "" {
256+
f.BufferVolumeImage.Tag = DefaultFluentdBufferVolumeImageTag
257+
} else {
258+
f.BufferVolumeImage.Tag = Version
259+
}
256260
}
257261
if f.BufferVolumeImage.PullPolicy == "" {
258262
f.BufferVolumeImage.PullPolicy = "IfNotPresent"

0 commit comments

Comments
 (0)