Skip to content

Commit c7d7b66

Browse files
committed
refactor: reorganize examples and add docker-compose setups
- Move k8s/ to examples/kubernetes/ with sidecar/ and daemonset/ subdirs - Add docker-compose examples for both plugins with MinIO integration - Add build-docker.sh script for cross-platform Docker builds - Update out_clp_s3_v2 README with comprehensive documentation - Update plugin-config.conf with cleaner defaults
1 parent 2965bf6 commit c7d7b66

24 files changed

+1016
-351
lines changed
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Docker Compose Example
2+
3+
Local development setup with MinIO (S3-compatible storage) for testing the out_clp_s3 plugin.
4+
5+
> **See also:** [Plugin README](../../README.md) for configuration options |
6+
> [Main README](../../../../README.md) for project overview
7+
8+
## Quick Start
9+
10+
```shell
11+
docker compose up
12+
```
13+
14+
This starts:
15+
- **MinIO** - S3-compatible storage
16+
- **Fluent Bit** - with the CLP plugin configured
17+
- **Log generator** - produces sample log lines
18+
19+
## Services
20+
21+
| Service | URL | Credentials |
22+
|---------|-----|-------------|
23+
| MinIO Console | http://localhost:9001 | minioadmin / minioadmin |
24+
| MinIO API | http://localhost:9000 | - |
25+
26+
## Verify Logs
27+
28+
Logs are uploaded when the buffer reaches `upload_size_mb` (default: 16 MB). To see uploads sooner,
29+
generate more logs or lower the threshold in `fluent-bit.conf`.
30+
31+
```shell
32+
# Check uploaded files
33+
docker compose exec minio mc ls local/logs/ --recursive
34+
35+
# Or open MinIO Console
36+
open http://localhost:9001
37+
```
38+
39+
## Configuration
40+
41+
Edit `fluent-bit.conf` to customize:
42+
- Input paths and parsers
43+
- Upload size threshold (`upload_size_mb`)
44+
- Output bucket and prefix
45+
46+
See [plugin documentation](../../README.md) for all options.
47+
48+
## Cleanup
49+
50+
```shell
51+
docker compose down -v
52+
```
53+
54+
## Files
55+
56+
| File | Description |
57+
|------|-------------|
58+
| `docker-compose.yaml` | Service definitions |
59+
| `fluent-bit.conf` | Fluent Bit configuration |
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Docker Compose for local development and testing
2+
# Usage: docker compose up
3+
#
4+
# Services:
5+
# - minio: S3-compatible storage (console at http://localhost:9001)
6+
# - fluent-bit: Log collector with CLP plugin
7+
# - log-generator: Generates sample logs for testing
8+
9+
services:
10+
minio:
11+
image: minio/minio:latest
12+
command: server /data --console-address ":9001"
13+
ports:
14+
- "9000:9000" # S3 API
15+
- "9001:9001" # Console
16+
environment:
17+
MINIO_ROOT_USER: minioadmin
18+
MINIO_ROOT_PASSWORD: minioadmin
19+
volumes:
20+
- minio-data:/data
21+
healthcheck:
22+
test: ["CMD", "mc", "ready", "local"]
23+
interval: 5s
24+
timeout: 5s
25+
retries: 5
26+
27+
minio-setup:
28+
image: minio/mc:latest
29+
depends_on:
30+
minio:
31+
condition: service_healthy
32+
entrypoint: >
33+
/bin/sh -c "
34+
mc alias set local http://minio:9000 minioadmin minioadmin;
35+
mc mb local/logs --ignore-existing;
36+
mc anonymous set download local/logs;
37+
echo 'Bucket created and configured';
38+
"
39+
40+
fluent-bit:
41+
# Pre-built image (uncomment when available):
42+
# image: ghcr.io/y-scope/fluent-bit-clp-s3:latest
43+
# Build locally:
44+
build:
45+
context: ../../../..
46+
dockerfile: plugins/out_clp_s3/Dockerfile
47+
depends_on:
48+
minio-setup:
49+
condition: service_completed_successfully
50+
environment:
51+
AWS_REGION: us-east-1
52+
AWS_ENDPOINT_URL: http://minio:9000
53+
AWS_ACCESS_KEY_ID: minioadmin
54+
AWS_SECRET_ACCESS_KEY: minioadmin
55+
volumes:
56+
- ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro
57+
- logs:/logs:ro
58+
restart: unless-stopped
59+
60+
# Generates sample logs for testing
61+
log-generator:
62+
image: alpine:latest
63+
volumes:
64+
- logs:/logs
65+
entrypoint: >
66+
/bin/sh -c "
67+
mkdir -p /logs;
68+
echo 'Starting log generator...';
69+
i=0;
70+
while true; do
71+
echo \"$$(date -u +%Y-%m-%dT%H:%M:%SZ) INFO Log message $$i from container\" >> /logs/app.log;
72+
i=$$((i + 1));
73+
sleep 0.1;
74+
done
75+
"
76+
77+
volumes:
78+
minio-data:
79+
logs:
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Fluent Bit configuration for Docker Compose
2+
# Watches /logs directory and uploads to MinIO
3+
4+
[SERVICE]
5+
flush 1
6+
daemon off
7+
log_level info
8+
plugins_file /fluent-bit/etc/plugin-config.conf
9+
10+
[INPUT]
11+
name tail
12+
path /logs/*.log
13+
tag app
14+
refresh_interval 5
15+
read_from_head true
16+
17+
[OUTPUT]
18+
name out_clp_s3
19+
match *
20+
s3_bucket logs
21+
s3_region us-east-1
22+
s3_bucket_prefix app/
23+
upload_size_mb 16
24+
use_disk_buffer true
Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
# Plugin configuration referenced in Fluent Bit configuration
22
[PLUGINS]
3-
path /fluent-bit/bin/out_clp_s3.so
4-
#path ./out_clp_s3.so
3+
path /fluent-bit/lib/out_clp_s3.so

0 commit comments

Comments
 (0)