Skip to content

Commit 9f76518

Browse files
authored
Merge pull request kubernetes#90804 from dims/new-go-runner-image-for-distroless
New go-runner image for distroless scenarios
2 parents be02fe5 + 393e095 commit 9f76518

File tree

10 files changed

+333
-0
lines changed

10 files changed

+333
-0
lines changed

build/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ filegroup(
2020
name = "all-srcs",
2121
srcs = [
2222
":package-srcs",
23+
"//build/go-runner:all-srcs",
2324
"//build/release-tars:all-srcs",
2425
"//build/visible_to:all-srcs",
2526
],

build/go-runner/BUILD

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["go-runner.go"],
6+
importpath = "k8s.io/kubernetes/build/go-runner",
7+
visibility = ["//visibility:private"],
8+
deps = ["//vendor/github.com/pkg/errors:go_default_library"],
9+
)
10+
11+
go_binary(
12+
name = "go-runner",
13+
embed = [":go_default_library"],
14+
visibility = ["//visibility:public"],
15+
)
16+
17+
filegroup(
18+
name = "package-srcs",
19+
srcs = glob(["**"]),
20+
tags = ["automanaged"],
21+
visibility = ["//visibility:private"],
22+
)
23+
24+
filegroup(
25+
name = "all-srcs",
26+
srcs = [":package-srcs"],
27+
tags = ["automanaged"],
28+
visibility = ["//visibility:public"],
29+
)

build/go-runner/Dockerfile

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Copyright 2020 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Build the manager binary
16+
FROM golang:1.13 as builder
17+
WORKDIR /workspace
18+
19+
# Run this with docker build --build_arg goproxy=$(go env GOPROXY) to override the goproxy
20+
ARG goproxy=https://proxy.golang.org
21+
# Run this with docker build --build_arg package=./controlplane/kubeadm or --build_arg package=./bootstrap/kubeadm
22+
ENV GOPROXY=$goproxy
23+
24+
# Copy the sources
25+
COPY ./ ./
26+
27+
# Cache the go build
28+
RUN go build .
29+
30+
# Build
31+
ARG package=.
32+
ARG ARCH
33+
34+
# Do not force rebuild of up-to-date packages (do not use -a)
35+
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
36+
go build -ldflags '-s -w -buildid= -extldflags "-static"' \
37+
-o go-runner ${package}
38+
39+
# Production image
40+
FROM gcr.io/distroless/static:latest
41+
LABEL maintainers="Kubernetes Authors"
42+
LABEL description="go based runner for distroless scenarios"
43+
WORKDIR /
44+
COPY --from=builder /workspace/go-runner .
45+
ENTRYPOINT ["/go-runner"]

build/go-runner/Makefile

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Copyright 2020 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# set default shell
16+
SHELL=/bin/bash -o pipefail
17+
18+
TAG ?= 0.1.0
19+
REGISTRY ?= k8s.gcr.io
20+
21+
IMGNAME = go-runner
22+
IMAGE = $(REGISTRY)/$(IMGNAME)
23+
24+
PLATFORMS = linux/amd64 linux/arm64 linux/arm linux/ppc64le linux/s390x
25+
26+
HOST_GOOS ?= $(shell go env GOOS)
27+
HOST_GOARCH ?= $(shell go env GOARCH)
28+
GO_BUILD ?= go build
29+
30+
.PHONY: all build clean
31+
32+
.PHONY: all
33+
all: build
34+
35+
.PHONY: build
36+
build:
37+
$(GO_BUILD)
38+
39+
.PHONY: clean
40+
clean:
41+
rm go-runner
42+
43+
.PHONY: container
44+
container: init-docker-buildx
45+
# https://github.com/docker/buildx/issues/59
46+
$(foreach PLATFORM,$(PLATFORMS), \
47+
DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build \
48+
--load \
49+
--progress plain \
50+
--platform $(PLATFORM) \
51+
--tag $(IMAGE)-$(PLATFORM):$(TAG) .;)
52+
53+
.PHONY: push
54+
push: container
55+
$(foreach PLATFORM,$(PLATFORMS), \
56+
docker push $(IMAGE)-$(PLATFORM):$(TAG);)
57+
58+
.PHONY: manifest
59+
manifest: push
60+
docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(PLATFORMS) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g")
61+
@for arch in $(PLATFORMS); do docker manifest annotate --arch "$${arch##*/}" ${IMAGE}:${TAG} ${IMAGE}-$${arch}:${TAG}; done
62+
docker manifest push --purge $(IMAGE):$(TAG)
63+
64+
.PHONY: init-docker-buildx
65+
init-docker-buildx:
66+
ifneq ($(shell docker buildx 2>&1 >/dev/null; echo $?),)
67+
$(error "buildx not vailable. Docker 19.03 or higher is required")
68+
endif
69+
docker run --rm --privileged linuxkit/binfmt:4ea3b9b0938cbd19834c096aa31ff475cc75d281
70+
docker buildx create --name multiarch-go-runner --use || true
71+
docker buildx inspect --bootstrap

build/go-runner/OWNERS

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# See the OWNERS docs at https://go.k8s.io/owners
2+
3+
approvers:
4+
- build-image-approvers
5+
reviewers:
6+
- build-image-reviewers

build/go-runner/README.md

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Kubernetes go-runner image
2+
3+
The Kubernetes go-runner image wraps the gcr.io/distroless/static image and provides a go based
4+
binary that can run commands and wrap stdout/stderr etc.
5+
6+
Why do we need this? Some of our images like kube-apiserver currently use bash for collecting
7+
logs, so we are not able to switch to distroless images directly for these images. The klog's
8+
`--log-file` was supposed to fix this problem, but we ran into trouble in scalability CI jobs
9+
around log rotation and picked this option instead. we essentially publish a multi-arch
10+
manifest with support for various platforms. This can be used as a base for other kubernetes
11+
components.
12+
13+
For example instead of running kube-apiserver like this:
14+
```bash
15+
"/bin/sh",
16+
"-c",
17+
"exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1"
18+
```
19+
20+
we would use go-runner like so:
21+
```bash
22+
"/go-runner", "--log-file=/var/log/kube-apiserver.log", "--also-stdout=false", "--redirect-stderr=true",
23+
"/usr/local/bin/kube-apiserver",
24+
"--allow-privileged={{pillar['allow_privileged']}}",
25+
{{params}}
26+
```
27+
28+
The go-runner would then ensure that we run the `/usr/local/bin/kube-apiserver` with the
29+
specified parameters and redirect stdout ONLY to the log file specified and ensure anything
30+
logged to stderr also ends up in the log file.

build/go-runner/cloudbuild.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md for more details on image pushing process
2+
3+
# this must be specified in seconds. If omitted, defaults to 600s (10 mins)
4+
timeout: 1200s
5+
# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
6+
# or any new substitutions added in the future.
7+
options:
8+
substitution_option: ALLOW_LOOSE
9+
machineType: 'N1_HIGHCPU_8'
10+
steps:
11+
- name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200422-b25d964'
12+
entrypoint: 'bash'
13+
dir: ./build/go-runner
14+
env:
15+
- DOCKER_CLI_EXPERIMENTAL=enabled
16+
- REGISTRY=gcr.io/$PROJECT_ID
17+
- HOME=/root
18+
args:
19+
- '-c'
20+
- |
21+
gcloud auth configure-docker \
22+
&& make manifest

build/go-runner/go-runner.go

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"flag"
21+
"fmt"
22+
"io"
23+
"log"
24+
"os"
25+
"os/exec"
26+
"os/signal"
27+
"strings"
28+
"syscall"
29+
30+
"github.com/pkg/errors"
31+
)
32+
33+
var (
34+
logFilePath = flag.String("log-file", "", "If non-empty, save stdout to this file")
35+
alsoToStdOut = flag.Bool("also-stdout", false, "useful with log-file, log to standard output as well as the log file")
36+
redirectStderr = flag.Bool("redirect-stderr", true, "treat stderr same as stdout")
37+
)
38+
39+
func main() {
40+
flag.Parse()
41+
42+
if err := configureAndRun(); err != nil {
43+
log.Fatal(err)
44+
}
45+
}
46+
47+
func configureAndRun() error {
48+
var (
49+
outputStream io.Writer = os.Stdout
50+
errStream io.Writer = os.Stderr
51+
)
52+
53+
args := flag.Args()
54+
if len(args) == 0 {
55+
return errors.Errorf("not enough arguments to run")
56+
}
57+
58+
if logFilePath != nil && *logFilePath != "" {
59+
logFile, err := os.OpenFile(*logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
60+
if err != nil {
61+
return errors.Wrapf(err, "failed to create log file %v", *logFilePath)
62+
}
63+
if *alsoToStdOut {
64+
outputStream = io.MultiWriter(os.Stdout, logFile)
65+
} else {
66+
outputStream = logFile
67+
}
68+
}
69+
70+
if *redirectStderr {
71+
errStream = outputStream
72+
}
73+
74+
exe := args[0]
75+
var exeArgs []string
76+
if len(args) > 1 {
77+
exeArgs = args[1:]
78+
}
79+
cmd := exec.Command(exe, exeArgs...)
80+
cmd.Stdout = outputStream
81+
cmd.Stderr = errStream
82+
83+
log.Printf("Running command:\n%v", cmdInfo(cmd))
84+
err := cmd.Start()
85+
if err != nil {
86+
return errors.Wrap(err, "starting command")
87+
}
88+
89+
// Handle signals and shutdown process gracefully.
90+
go setupSigHandler(cmd.Process)
91+
return errors.Wrap(cmd.Wait(), "running command")
92+
}
93+
94+
// cmdInfo generates a useful look at what the command is for printing/debug.
95+
func cmdInfo(cmd *exec.Cmd) string {
96+
return fmt.Sprintf(
97+
`Command env: (log-file=%v, also-stdout=%v, redirect-stderr=%v)
98+
Run from directory: %v
99+
Executable path: %v
100+
Args (comma-delimited): %v`, *logFilePath, *alsoToStdOut, *redirectStderr,
101+
cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","),
102+
)
103+
}
104+
105+
// setupSigHandler will forward any termination signals to the process
106+
func setupSigHandler(process *os.Process) {
107+
// terminationSignals are signals that cause the program to exit in the
108+
// supported platforms (linux, darwin, windows).
109+
terminationSignals := []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT}
110+
111+
c := make(chan os.Signal, 1)
112+
signal.Notify(c, terminationSignals...)
113+
114+
// Block until a signal is received.
115+
log.Println("Now listening for interrupts")
116+
s := <-c
117+
log.Printf("Got signal: %v. Sending down to process (PID: %v)", s, process.Pid)
118+
if err := process.Signal(s); err != nil {
119+
log.Fatalf("Failed to signal process: %v", err)
120+
}
121+
log.Printf("Signalled process %v successfully.", process.Pid)
122+
}

build/go-runner/go.mod

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module k8s.io/kubernetes/build/go-runner
2+
3+
go 1.13
4+
5+
require github.com/pkg/errors v0.9.1

build/go-runner/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
2+
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

0 commit comments

Comments
 (0)