Skip to content

Commit 9970c94

Browse files
tjmoore4tjmoore4
committed
PGDG Components Build Files
Issue: PGO-2703
1 parent e489ca5 commit 9970c94

File tree

24 files changed

+1181
-0
lines changed

24 files changed

+1181
-0
lines changed

‎cmd/pgbackrest/main.go‎

Lines changed: 347 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,347 @@
1+
// Copyright 2018 - 2025 Crunchy Data Solutions, Inc.
2+
//
3+
// SPDX-License-Identifier: Apache-2.0
4+
5+
package main
6+
7+
import (
8+
"bytes"
9+
"context"
10+
"io"
11+
"os"
12+
"os/exec"
13+
"strconv"
14+
"strings"
15+
16+
log "github.com/sirupsen/logrus"
17+
corev1 "k8s.io/api/core/v1"
18+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19+
"k8s.io/apimachinery/pkg/runtime"
20+
"k8s.io/client-go/kubernetes"
21+
"k8s.io/client-go/rest"
22+
"k8s.io/client-go/tools/clientcmd"
23+
"k8s.io/client-go/tools/remotecommand"
24+
)
25+
26+
type KubeAPI struct {
27+
Client *kubernetes.Clientset
28+
Config *rest.Config
29+
}
30+
31+
const backrestCommand = "pgbackrest"
32+
33+
const (
34+
backrestBackupCommand = "backup"
35+
backrestInfoCommand = "info"
36+
backrestStanzaCreateCommand = "stanza-create"
37+
)
38+
39+
const (
40+
repoTypeFlagGCS = "--repo1-type=gcs"
41+
repoTypeFlagS3 = "--repo1-type=s3"
42+
noRepoS3VerifyTLS = "--no-repo1-s3-verify-tls"
43+
)
44+
45+
const containerNameDefault = "database"
46+
47+
const (
48+
pgtaskBackrestStanzaCreate = "stanza-create"
49+
pgtaskBackrestInfo = "info"
50+
pgtaskBackrestBackup = "backup"
51+
)
52+
53+
type config struct {
54+
command, commandOpts, container, namespace, podName, repoType, selector string
55+
compareHash, localGCSStorage, localS3Storage, s3VerifyTLS bool
56+
}
57+
58+
func main() {
59+
ctx := context.Background()
60+
log.Info("crunchy-pgbackrest starts")
61+
62+
debugFlag, _ := strconv.ParseBool(os.Getenv("CRUNCHY_DEBUG"))
63+
if debugFlag {
64+
log.SetLevel(log.DebugLevel)
65+
}
66+
log.Infof("debug flag set to %t", debugFlag)
67+
68+
config, err := NewConfig()
69+
if err != nil {
70+
panic(err)
71+
}
72+
73+
k, err := NewForConfig(config)
74+
if err != nil {
75+
panic(err)
76+
}
77+
78+
// first load any configuration provided (e.g. via environment variables)
79+
cfg := loadConfiguration(ctx, k)
80+
81+
// create the proper pgBackRest command
82+
cmd := createPGBackRestCommand(cfg)
83+
log.Infof("command to execute is [%s]", strings.Join(cmd, " "))
84+
85+
var output, stderr string
86+
// now run the proper exec command depending on whether or not the config hashes should first
87+
// be compared prior to executing the PGBackRest command
88+
if !cfg.compareHash {
89+
output, stderr, err = runCommand(ctx, k, cfg, cmd)
90+
} else {
91+
output, stderr, err = compareHashAndRunCommand(ctx, k, cfg, cmd)
92+
}
93+
94+
// log any output and check for errors
95+
log.Info("output=[" + output + "]")
96+
log.Info("stderr=[" + stderr + "]")
97+
if err != nil {
98+
log.Fatal(err)
99+
}
100+
101+
log.Info("crunchy-pgbackrest ends")
102+
}
103+
104+
// Exec returns the stdout and stderr from running a command inside an existing
105+
// container.
106+
func (k *KubeAPI) Exec(ctx context.Context, namespace, pod, container string, stdin io.Reader, command []string) (string, string, error) {
107+
var stdout, stderr bytes.Buffer
108+
109+
var Scheme = runtime.NewScheme()
110+
if err := corev1.AddToScheme(Scheme); err != nil {
111+
log.Error(err)
112+
return "", "", err
113+
}
114+
var ParameterCodec = runtime.NewParameterCodec(Scheme)
115+
116+
request := k.Client.CoreV1().RESTClient().Post().
117+
Resource("pods").SubResource("exec").
118+
Namespace(namespace).Name(pod).
119+
VersionedParams(&corev1.PodExecOptions{
120+
Container: container,
121+
Command: command,
122+
Stdin: stdin != nil,
123+
Stdout: true,
124+
Stderr: true,
125+
}, ParameterCodec)
126+
127+
exec, err := remotecommand.NewSPDYExecutor(k.Config, "POST", request.URL())
128+
129+
if err == nil {
130+
err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
131+
Stdin: stdin,
132+
Stdout: &stdout,
133+
Stderr: &stderr,
134+
})
135+
}
136+
137+
return stdout.String(), stderr.String(), err
138+
}
139+
140+
func NewConfig() (*rest.Config, error) {
141+
// The default loading rules try to read from the files specified in the
142+
// environment or from the home directory.
143+
loader := clientcmd.NewDefaultClientConfigLoadingRules()
144+
145+
// The deferred loader tries an in-cluster config if the default loading
146+
// rules produce no results.
147+
return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
148+
loader, &clientcmd.ConfigOverrides{}).ClientConfig()
149+
}
150+
151+
func NewForConfig(config *rest.Config) (*KubeAPI, error) {
152+
var api KubeAPI
153+
var err error
154+
155+
api.Config = config
156+
api.Client, err = kubernetes.NewForConfig(api.Config)
157+
158+
return &api, err
159+
}
160+
161+
// getEnvRequired attempts to get an environmental variable that is required
162+
// by this program. If this cannot happen, we fatally exit
163+
func getEnvRequired(envVar string) string {
164+
val := strings.TrimSpace(os.Getenv(envVar))
165+
166+
if val == "" {
167+
log.Fatalf("required environmental variable %q not set, exiting.", envVar)
168+
}
169+
170+
log.Debugf("%s set to: %s", envVar, val)
171+
172+
return val
173+
}
174+
175+
// loadConfiguration loads configuration from the environment as needed to run a pgBackRest
176+
// command
177+
func loadConfiguration(ctx context.Context, kubeapi *KubeAPI) config {
178+
cfg := config{}
179+
180+
cfg.namespace = getEnvRequired("NAMESPACE")
181+
cfg.command = getEnvRequired("COMMAND")
182+
183+
cfg.container = os.Getenv("CONTAINER")
184+
if cfg.container == "" {
185+
cfg.container = containerNameDefault
186+
}
187+
log.Debugf("CONTAINER set to: %s", cfg.container)
188+
189+
cfg.commandOpts = os.Getenv("COMMAND_OPTS")
190+
log.Debugf("COMMAND_OPTS set to: %s", cfg.commandOpts)
191+
192+
cfg.selector = os.Getenv("SELECTOR")
193+
log.Debugf("SELECTOR set to: %s", cfg.selector)
194+
195+
compareHashEnv := os.Getenv("COMPARE_HASH")
196+
log.Debugf("COMPARE_HASH set to: %s", compareHashEnv)
197+
if compareHashEnv != "" {
198+
// default to false if an error parsing
199+
cfg.compareHash, _ = strconv.ParseBool(compareHashEnv)
200+
}
201+
202+
if cfg.selector == "" {
203+
// if no selector then a Pod name must be provided
204+
cfg.podName = getEnvRequired("PODNAME")
205+
} else {
206+
// if a selector is provided, then lookup the Pod via the provided selector to get the
207+
// Pod name
208+
pods, err := kubeapi.Client.CoreV1().Pods(cfg.namespace).List(ctx,
209+
metav1.ListOptions{LabelSelector: cfg.selector})
210+
if err != nil {
211+
log.Fatal(err)
212+
}
213+
if len(pods.Items) != 1 {
214+
log.Fatalf("found %d Pods using selector, but only expected one", len(pods.Items))
215+
}
216+
cfg.podName = pods.Items[0].GetName()
217+
}
218+
log.Debugf("PODNAME set to: %s", cfg.podName)
219+
220+
cfg.repoType = os.Getenv("PGBACKREST_REPO1_TYPE")
221+
log.Debugf("PGBACKREST_REPO1_TYPE set to: %s", cfg.repoType)
222+
223+
// determine the setting of PGHA_PGBACKREST_LOCAL_S3_STORAGE
224+
// we will discard the error and treat the value as "false" if it is not
225+
// explicitly set
226+
cfg.localS3Storage, _ = strconv.ParseBool(os.Getenv("PGHA_PGBACKREST_LOCAL_S3_STORAGE"))
227+
log.Debugf("PGHA_PGBACKREST_LOCAL_S3_STORAGE set to: %t", cfg.localS3Storage)
228+
229+
// determine the setting of PGHA_PGBACKREST_LOCAL_GCS_STORAGE
230+
// we will discard the error and treat the value as "false" if it is not
231+
// explicitly set
232+
cfg.localGCSStorage, _ = strconv.ParseBool(os.Getenv("PGHA_PGBACKREST_LOCAL_GCS_STORAGE"))
233+
log.Debugf("PGHA_PGBACKREST_LOCAL_GCS_STORAGE set to: %t", cfg.localGCSStorage)
234+
235+
// parse the environment variable and store the appropriate boolean value
236+
// we will discard the error and treat the value as "false" if it is not
237+
// explicitly set
238+
cfg.s3VerifyTLS, _ = strconv.ParseBool(os.Getenv("PGHA_PGBACKREST_S3_VERIFY_TLS"))
239+
log.Debugf("PGHA_PGBACKREST_S3_VERIFY_TLS set to: %t", cfg.s3VerifyTLS)
240+
241+
return cfg
242+
}
243+
244+
// createPGBackRestCommand form the proper pgBackRest command based on the configuration provided
245+
func createPGBackRestCommand(cfg config) []string {
246+
cmd := []string{backrestCommand}
247+
248+
switch cfg.command {
249+
default:
250+
log.Fatalf("unsupported backup command specified: %s", cfg.command)
251+
case pgtaskBackrestStanzaCreate:
252+
log.Info("backrest stanza-create command requested")
253+
cmd = append(cmd, backrestStanzaCreateCommand, cfg.commandOpts)
254+
case pgtaskBackrestInfo:
255+
log.Info("backrest info command requested")
256+
cmd = append(cmd, backrestInfoCommand, cfg.commandOpts)
257+
case pgtaskBackrestBackup:
258+
log.Info("backrest backup command requested")
259+
cmd = append(cmd, backrestBackupCommand, cfg.commandOpts)
260+
}
261+
262+
if cfg.localS3Storage {
263+
// if the first backup fails, still attempt the 2nd one
264+
cmd = append(cmd, ";")
265+
cmd = append(cmd, cmd...)
266+
cmd[len(cmd)-1] = repoTypeFlagS3 // a trick to overwite the second ";"
267+
// pass in the flag to disable TLS verification, if set
268+
// otherwise, maintain default behavior and verify TLS
269+
if !cfg.s3VerifyTLS {
270+
cmd = append(cmd, noRepoS3VerifyTLS)
271+
}
272+
log.Info("backrest command will be executed for both local and s3 storage")
273+
} else if cfg.repoType == "s3" {
274+
cmd = append(cmd, repoTypeFlagS3)
275+
// pass in the flag to disable TLS verification, if set
276+
// otherwise, maintain default behavior and verify TLS
277+
if !cfg.s3VerifyTLS {
278+
cmd = append(cmd, noRepoS3VerifyTLS)
279+
}
280+
log.Info("s3 flag enabled for backrest command")
281+
}
282+
283+
if cfg.localGCSStorage {
284+
// if the first backup fails, still attempt the 2nd one
285+
cmd = append(cmd, ";")
286+
cmd = append(cmd, cmd...)
287+
cmd[len(cmd)-1] = repoTypeFlagGCS // a trick to overwite the second ";"
288+
log.Info("backrest command will be executed for both local and gcs storage")
289+
} else if cfg.repoType == "gcs" {
290+
cmd = append(cmd, repoTypeFlagGCS)
291+
log.Info("gcs flag enabled for backrest command")
292+
}
293+
294+
return cmd
295+
}
296+
297+
// compareHashAndRunCommand calculates the hash of the pgBackRest configuration locally against
298+
// a hash of the pgBackRest configuration for the container being exec'd into to run a pgBackRest
299+
// command. Only if the hashes match will the pgBackRest command be run, otherwise and error will
300+
// be written and exit code 1 will be returned. This is done to ensure a pgBackRest command is only
301+
// run when it can be verified that the exepected configuration is present.
302+
func compareHashAndRunCommand(ctx context.Context, kubeapi *KubeAPI, cfg config, cmd []string) (string, string, error) {
303+
304+
// the base script used in both the local and exec commands created below
305+
baseScript := `
306+
export LC_ALL=C
307+
shopt -s globstar
308+
files=(/etc/pgbackrest/conf.d/**)
309+
for i in "${!files[@]}"; do
310+
[[ -f "${files[$i]}" ]] || unset -v "files[$i]"
311+
done`
312+
313+
// the script run locally to get the local hash
314+
localScript := baseScript + `
315+
sha1sum "${files[@]}" | sha1sum
316+
`
317+
318+
// the script to run remotely via exec
319+
execScript := baseScript + `
320+
declare -r hash="$1"
321+
local_hash="$(sha1sum "${files[@]}" | sha1sum)"
322+
if [[ "${local_hash}" != "${hash}" ]]; then
323+
printf >&2 "hash %s does not match local hash %s" "${hash}" "${local_hash}"; exit 1;
324+
else
325+
` + strings.Join(cmd, " ") + `
326+
fi
327+
`
328+
329+
localHashCmd := exec.CommandContext(ctx, "bash", "-ceu", "--", localScript)
330+
hashOutput, err := localHashCmd.Output()
331+
if err != nil {
332+
log.Fatalf("unable to calculate hash for pgBackRest config: %v", err)
333+
}
334+
configHash := strings.TrimSuffix(string(hashOutput), "\n")
335+
log.Debugf("calculated config hash %s", configHash)
336+
337+
execCmd := []string{"bash", "-ceu", "--", execScript, "-", configHash}
338+
return kubeapi.Exec(ctx, cfg.namespace, cfg.podName, cfg.container, nil, execCmd)
339+
}
340+
341+
// runCommand runs the provided pgBackRest command according to the configuration
342+
// provided
343+
func runCommand(ctx context.Context, kubeapi *KubeAPI, cfg config, cmd []string) (string, string, error) {
344+
bashCmd := []string{"bash"}
345+
reader := strings.NewReader(strings.Join(cmd, " "))
346+
return kubeapi.Exec(ctx, cfg.namespace, cfg.podName, cfg.container, reader, bashCmd)
347+
}

‎components/image-collector/Dockerfile‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright 2025 Crunchy Data Solutions, Inc.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
# Rather than build the binary, retrieve the already-built binary from
6+
# the OpenTelemetry image
7+
FROM otel/opentelemetry-collector-contrib:0.139.0 AS collector
8+
9+
# Aggregate the collector licenses from binary
10+
# and from root of the PGO repo
11+
FROM registry.access.redhat.com/ubi9/go-toolset AS build
12+
13+
# Run as user 0 to allow later chmod'ing
14+
USER 0
15+
16+
COPY --from=collector --chmod=0777 /otelcol-contrib /otelcol-contrib
17+
COPY hack/extract-licenses.go .
18+
RUN go run ./extract-licenses.go /licenses /otelcol-contrib
19+
COPY licenses/LICENSE.txt /licenses/LICENSE.txt
20+
21+
# Make sure all licenses are readable by owner:group:other
22+
RUN \
23+
<<-SHELL
24+
set -e
25+
26+
find /licenses '(' -type d -exec chmod 0555 '{}' + ')'
27+
find /licenses '(' -type f -exec chmod 0444 '{}' + ')'
28+
SHELL
29+
30+
# Assemble the complete image with required packages for OpenTelemetry
31+
# and related activity (e.g., log retention, process restarting)
32+
FROM registry.access.redhat.com/ubi9/ubi-minimal
33+
34+
COPY --from=build --chmod=0777 /otelcol-contrib /otelcol-contrib
35+
COPY --from=build /licenses /licenses
36+
37+
RUN microdnf install -y 'logrotate' 'procps-ng'
38+
39+
USER 2

0 commit comments

Comments
 (0)