Skip to content

Commit 73e409d

Browse files
authored
Merge pull request rsyslog#6656 from rgerhards/codex-container-dockerhub-helper
packaging/docker: add Docker Hub metadata helper
2 parents 5fb668c + f9f822f commit 73e409d

File tree

3 files changed

+178
-0
lines changed

3 files changed

+178
-0
lines changed

packaging/docker/rsyslog/README.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ container image family:
99
- `rsyslog/rsyslog-dockerlogs`
1010
- `rsyslog/rsyslog-etl`
1111

12+
Docker Hub descriptions for these repos can be maintained from this
13+
subtree with `sync_dockerhub_metadata.py` and `dockerhub_metadata.json`.
14+
The script reads credentials from `DOCKERHUB_USERNAME` /
15+
`DOCKERHUB_PASSWORD` or the local `~/.docker/config.json` and defaults
16+
to a dry run.
17+
1218
## Version and tag contract
1319

1420
Local builds default to a non-release tag on purpose:
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"rsyslog-minimal": {
3+
"description": "Lean Ubuntu-based rsyslog image with core modules and stdout output for custom configurations.",
4+
"full_description": "# rsyslog/rsyslog-minimal\n\n`rsyslog/rsyslog-minimal` is a lean Ubuntu-based image that provides the rsyslog core with a very small default configuration.\n\n## What it is for\n\nUse this image when you want to:\n\n- run rsyslog in a container with minimal defaults\n- provide your own configuration under `/etc/rsyslog.d/`\n- build your own downstream rsyslog-based image\n\n## Default behavior\n\nThe image:\n\n- starts `rsyslogd` in the foreground\n- validates the configuration with `rsyslogd -N1` before startup\n- writes logs to standard output by default\n- includes configuration snippets from `/etc/rsyslog.d/*.conf`\n\n## Useful environment variables\n\n- `RSYSLOG_HOSTNAME`: sets the hostname used inside rsyslog\n- `PERMIT_UNCLEAN_START`: skips startup config validation when set\n- `RSYSLOG_ROLE`: entrypoint role selector, normally left as `minimal`\n\n## Notes\n\nThis image is the foundation for the other user-facing rsyslog container images.\n\nDocumentation:\nhttps://www.rsyslog.com/doc/containers/minimal.html\n"
5+
},
6+
"rsyslog": {
7+
"description": "General-purpose rsyslog image with common HTTP-capable modules for containerized log processing.",
8+
"full_description": "# rsyslog/rsyslog\n\n`rsyslog/rsyslog` is the general-purpose image in the rsyslog container family.\n\n## What it is for\n\nUse this image when you want to:\n\n- run rsyslog in a container with broader defaults than the minimal image\n- use HTTP-capable rsyslog modules out of the box\n- build your own downstream rsyslog image on a more featureful base\n\n## Default behavior\n\nThe image:\n\n- builds on `rsyslog/rsyslog-minimal`\n- includes the `imhttp` and `omhttp` modules\n- keeps the same entrypoint and config-validation behavior as the minimal image\n- writes logs to standard output unless you mount additional rules\n\n## Useful environment variables\n\n- `RSYSLOG_HOSTNAME`: sets the hostname used inside rsyslog\n- `PERMIT_UNCLEAN_START`: skips startup config validation when set\n- `RSYSLOG_ROLE`: entrypoint role selector, normally left as `standard`\n\n## Notes\n\nThis image is the common base for the collector and other specialized rsyslog container roles.\n\nDocumentation:\nhttps://www.rsyslog.com/doc/containers/standard.html\n"
9+
},
10+
"rsyslog-collector": {
11+
"description": "Central rsyslog collector image with UDP/TCP inputs, optional RELP/TLS, and file-based defaults.",
12+
"full_description": "# rsyslog/rsyslog-collector\n\n`rsyslog/rsyslog-collector` is the central receiver role in the rsyslog container image family.\n\n## What it is for\n\nUse this image when you want to:\n\n- receive syslog traffic in a container\n- build a central log collector or relay service\n- start from a reusable rsyslog receiver image and add your own routing rules\n\n## Default behavior\n\nThe image:\n\n- listens on UDP syslog (`514/udp`) by default\n- listens on TCP syslog (`514/tcp`) by default\n- can enable RELP on `2514/tcp`\n- can enable TLS syslog on `6514/tcp`\n- writes received messages to local files by default\n\n## Useful environment variables\n\n- `ENABLE_UDP`: enable UDP syslog reception, default `on`\n- `ENABLE_TCP`: enable TCP syslog reception, default `on`\n- `ENABLE_RELP`: enable RELP reception, default `off`\n- `ENABLE_TLS`: enable TLS syslog reception, default `off`\n- `WRITE_ALL_FILE`: write `/var/log/all.log`, default `on`\n- `WRITE_JSON_FILE`: write `/var/log/all-json.log`, default `on`\n- `TLS_CA_FILE`, `TLS_CERT_FILE`, `TLS_KEY_FILE`, `TLS_AUTH_MODE`: TLS listener settings\n\n## Notes\n\nThe packaged configuration writes to local files. You can mount additional rules into `/etc/rsyslog.d/` to forward to other backends or reshape the pipeline.\n\nDocumentation:\nhttps://www.rsyslog.com/doc/containers/collector.html\n"
13+
},
14+
"rsyslog-dockerlogs": {
15+
"description": "Rsyslog image for reading Docker daemon logs and forwarding them to a central collector over TCP.",
16+
"full_description": "# rsyslog/rsyslog-dockerlogs\n\n`rsyslog/rsyslog-dockerlogs` is a forwarding role in the rsyslog container image family that reads Docker log events and ships them to a central collector.\n\n## What it is for\n\nUse this image when you want to:\n\n- collect logs from the Docker daemon with `imdocker`\n- forward container log events to a central rsyslog receiver\n- run an edge-side log forwarder in Docker environments\n\n## Default behavior\n\nThe image:\n\n- reads Docker log events with `imdocker`\n- forwards everything to a remote rsyslog collector over TCP\n- uses an in-memory action queue with infinite retry\n- does not store logs locally as its primary role\n\n## Useful environment variables\n\n- `REMOTE_SERVER_NAME`: required hostname or IP of the destination collector\n- `REMOTE_SERVER_PORT`: TCP port on the collector, default `514`\n- `RSYSLOG_HOSTNAME`: sets the hostname used inside rsyslog\n- `PERMIT_UNCLEAN_START`: skips startup config validation when set\n- `RSYSLOG_ROLE`: entrypoint role selector, normally left as `docker`\n\n## Notes\n\nThis image is designed to feed a central rsyslog collector, not to act as a standalone storage endpoint.\n\nDocumentation:\nhttps://www.rsyslog.com/doc/containers/dockerlogs.html\n"
17+
},
18+
"rsyslog-etl": {
19+
"description": "Specialized rsyslog ETL image that receives syslog and forwards events to a Vespa HTTP endpoint.",
20+
"full_description": "# rsyslog/rsyslog-etl\n\n`rsyslog/rsyslog-etl` is a specialized ingestion role that accepts syslog over the network and forwards events to a Vespa HTTP API.\n\n## What it is for\n\nUse this image when you want to:\n\n- receive syslog over UDP, TCP, or RELP\n- forward events to a Vespa document API with `omhttp`\n- use rsyslog as a transport and delivery component in a Vespa-oriented ETL pipeline\n\n## Default behavior\n\nThe image:\n\n- listens on UDP syslog (`514/udp`) by default\n- listens on TCP syslog (`514/tcp`) by default\n- enables RELP on `2514/tcp` by default\n- forwards events to Vespa over HTTP\n- is specialized for Vespa-oriented pipelines rather than generic all-destinations ETL use\n\n## Useful environment variables\n\n- `ENABLE_UDP`, `ENABLE_TCP`, `ENABLE_RELP`: control network listeners\n- `ENABLE_VESPA`: enables the packaged Vespa HTTP action\n- `VESPA_NAMESPACE`, `VESPA_DOCTYPE`: define the generated Vespa document path\n- `VESPA_SERVER`, `VESPA_PORT`: define the Vespa HTTP endpoint\n- `RSYSLOG_HOSTNAME`: sets the hostname used inside rsyslog\n- `PERMIT_UNCLEAN_START`: skips startup config validation when set\n\n## Notes\n\nThe packaged ETL output is HTTP to Vespa. The repository also includes a sample Docker Compose deployment for this image.\n\nDocumentation:\nhttps://www.rsyslog.com/doc/containers/etl.html\n"
21+
}
22+
}
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/usr/bin/env python3
2+
"""Sync Docker Hub repository descriptions for the rsyslog image family.
3+
4+
By default the script runs in dry-run mode and prints the intended updates.
5+
Use --apply to write metadata to Docker Hub.
6+
"""
7+
8+
from __future__ import annotations
9+
10+
import argparse
11+
import base64
12+
import binascii
13+
import json
14+
import os
15+
from pathlib import Path
16+
import sys
17+
import urllib.error
18+
import urllib.request
19+
20+
21+
HUB_LOGIN_URL = "https://hub.docker.com/v2/users/login/"
22+
HUB_REPO_URL = "https://hub.docker.com/v2/repositories/{namespace}/{repo}/"
23+
DEFAULT_NAMESPACE = "rsyslog"
24+
DEFAULT_METADATA_FILE = Path(__file__).with_name("dockerhub_metadata.json")
25+
26+
27+
def load_metadata(path: Path) -> dict[str, dict[str, str]]:
28+
return json.loads(path.read_text())
29+
30+
31+
def load_credentials() -> tuple[str, str]:
32+
username = os.getenv("DOCKERHUB_USERNAME")
33+
password = os.getenv("DOCKERHUB_PASSWORD")
34+
if username and password:
35+
return username, password
36+
37+
cfg_path = Path.home() / ".docker" / "config.json"
38+
if not cfg_path.exists():
39+
raise RuntimeError(
40+
"No Docker Hub credentials found. Set DOCKERHUB_USERNAME and "
41+
"DOCKERHUB_PASSWORD or login with docker first."
42+
)
43+
44+
cfg = json.loads(cfg_path.read_text())
45+
auths = cfg.get("auths", {})
46+
auth = auths.get("https://index.docker.io/v1/", {}).get("auth")
47+
if not auth:
48+
raise RuntimeError(
49+
"Docker config does not contain https://index.docker.io/v1/ auth."
50+
)
51+
52+
try:
53+
decoded = base64.b64decode(auth).decode()
54+
except (binascii.Error, UnicodeDecodeError) as err:
55+
raise RuntimeError("Docker config auth for Docker Hub is malformed.") from err
56+
57+
if ":" not in decoded:
58+
raise RuntimeError("Docker config auth for Docker Hub is malformed.")
59+
60+
return tuple(decoded.split(":", 1))
61+
62+
63+
def hub_request(
64+
url: str, token: str | None = None, method: str = "GET", payload: dict | None = None
65+
) -> dict:
66+
data = None if payload is None else json.dumps(payload).encode()
67+
req = urllib.request.Request(url, data=data, method=method)
68+
req.add_header("Content-Type", "application/json")
69+
if token:
70+
req.add_header("Authorization", f"JWT {token}")
71+
with urllib.request.urlopen(req, timeout=30) as resp:
72+
return json.loads(resp.read().decode())
73+
74+
75+
def login() -> str:
76+
username, password = load_credentials()
77+
resp = hub_request(HUB_LOGIN_URL, method="POST", payload={"username": username, "password": password})
78+
token = resp.get("token")
79+
if not token:
80+
raise RuntimeError("Docker Hub login succeeded but no token was returned.")
81+
return token
82+
83+
84+
def normalize_repo_selection(selected: list[str] | None, metadata: dict[str, dict[str, str]]) -> list[str]:
85+
if not selected:
86+
return sorted(metadata.keys())
87+
missing = [name for name in selected if name not in metadata]
88+
if missing:
89+
raise RuntimeError(f"Metadata file does not define repos: {', '.join(missing)}")
90+
return selected
91+
92+
93+
def main() -> int:
94+
parser = argparse.ArgumentParser(description=__doc__)
95+
parser.add_argument("--apply", action="store_true", help="Write metadata to Docker Hub.")
96+
parser.add_argument("--namespace", default=DEFAULT_NAMESPACE, help="Docker Hub namespace. Default: rsyslog")
97+
parser.add_argument(
98+
"--metadata-file",
99+
default=str(DEFAULT_METADATA_FILE),
100+
help="Path to the JSON metadata file.",
101+
)
102+
parser.add_argument(
103+
"--repo",
104+
action="append",
105+
help="Limit sync to one or more repos defined in the metadata file.",
106+
)
107+
args = parser.parse_args()
108+
109+
metadata_file = Path(args.metadata_file)
110+
metadata = load_metadata(metadata_file)
111+
repos = normalize_repo_selection(args.repo, metadata)
112+
113+
token = login()
114+
115+
for repo in repos:
116+
payload = metadata[repo]
117+
url = HUB_REPO_URL.format(namespace=args.namespace, repo=repo)
118+
current = hub_request(url, token=token)
119+
summary = {
120+
"repo": f"{args.namespace}/{repo}",
121+
"current_description": current.get("description") or "",
122+
"new_description": payload["description"],
123+
"apply": args.apply,
124+
}
125+
print(json.dumps(summary, ensure_ascii=True))
126+
if args.apply:
127+
updated = hub_request(url, token=token, method="PATCH", payload=payload)
128+
print(
129+
json.dumps(
130+
{
131+
"repo": f"{args.namespace}/{repo}",
132+
"updated_description": updated.get("description") or "",
133+
"has_full_description": updated.get("full_description") is not None,
134+
},
135+
ensure_ascii=True,
136+
)
137+
)
138+
139+
return 0
140+
141+
142+
if __name__ == "__main__":
143+
try:
144+
raise SystemExit(main())
145+
except urllib.error.HTTPError as err:
146+
print(f"HTTP error from Docker Hub: {err.code} {err.reason}", file=sys.stderr)
147+
raise
148+
except Exception as err: # pragma: no cover - CLI path
149+
print(f"ERROR: {err}", file=sys.stderr)
150+
raise SystemExit(1)

0 commit comments

Comments
 (0)