Skip to content

Commit 32f3126

Browse files
Stuart133Stuart133
authored
Merge pull request #5 from circleci/sr/add-fake-agent
[ONPREM-1221] Add the fake task agent for acceptance tests & publish images
2 parents 7e3363e + 5226a2f commit 32f3126

File tree

9 files changed

+284
-47
lines changed

9 files changed

+284
-47
lines changed

‎.circleci/config.yml

Lines changed: 32 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -6,59 +6,52 @@ orbs:
66
workflows:
77
main-workflow:
88
jobs:
9-
- download-taskagent:
9+
- prepare-agents:
1010
context: org-global
11-
- scan-image:
12-
context: org-global
13-
requires:
14-
- download-taskagent
15-
- build-and-publish-image:
11+
- build-and-publish-images:
1612
name: build-and-publish-image-amd64
1713
context: org-global
1814
requires:
19-
- scan-image
20-
- build-and-publish-image:
15+
- prepare-agents
16+
- build-and-publish-images:
2117
name: build-and-publish-image-arm64
2218
resource: arm.medium
2319
arch: arm64
2420
context: org-global
2521
requires:
26-
- scan-image
27-
28-
executors:
29-
ccc:
30-
docker:
31-
- image: circleci/command-convenience:0.1
32-
auth:
33-
username: $DOCKER_HUB_USER
34-
password: $DOCKER_HUB_PASSWORD
35-
environment:
36-
NAME: << pipeline.parameters.release-name >>
37-
DOCKERFILE_PATH: Dockerfile
38-
TEAM: on-prem
22+
- prepare-agents
23+
- publish-manifest:
24+
context: org-global
25+
filters:
26+
branches:
27+
only:
28+
- main
29+
requires:
30+
- build-and-publish-image-amd64
31+
- build-and-publish-image-arm64
3932

4033
parameters:
4134
release-name:
4235
type: string
4336
default: "runner-init"
4437

4538
jobs:
46-
download-taskagent:
39+
prepare-agents:
4740
machine:
4841
image: ubuntu-2204:2024.02.7
49-
resource_class: small
42+
resource_class: large
5043
steps:
5144
- checkout
5245
- docker_login
53-
- run: ./do download-taskagent amd64
54-
- run: ./do download-taskagent arm64
46+
- run: ./do build-fake-agents
47+
- run: ./do download-taskagents
5548
- persist_to_workspace:
5649
root: .
5750
paths:
58-
- "./circleci-agent-*"
51+
- "./bin/circleci-*"
5952
- notify_failing_main
6053

61-
build-and-publish-image:
54+
build-and-publish-images:
6255
parameters:
6356
resource:
6457
type: string
@@ -74,22 +67,22 @@ jobs:
7467
- attach_workspace:
7568
at: .
7669
- docker_login
77-
- run: docker build -t circleci/<< pipeline.parameters.release-name >>:agent-<< parameters.arch >> --build-arg ARCH=<< parameters.arch >> .
78-
# TODO: Publish images (on main) once the repo is set up
70+
- run: ./do build-docker-images << pipeline.parameters.release-name >> << parameters.arch >>
71+
- when:
72+
condition:
73+
equal: [ main, << pipeline.git.branch >> ]
74+
steps:
75+
- run: ./do publish-docker-images << pipeline.parameters.release-name >> << parameters.arch >>
7976
- notify_failing_main
8077

81-
scan-image:
82-
executor: ccc
78+
publish-manifest:
79+
machine:
80+
image: ubuntu-2204:2024.02.7
81+
resource_class: small
8382
steps:
8483
- checkout
85-
- setup_remote_docker
86-
- attach_workspace:
87-
at: .
88-
- run:
89-
name: Scan Docker images
90-
command: scan
91-
- store_artifacts:
92-
path: ccc-image-scan-results
84+
- docker_login
85+
- run: ./do publish-docker-manifest << pipeline.parameters.release-name >>
9386
- notify_failing_main
9487

9588
commands:

‎.gitignore

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Binaries for programs and plugins
2+
*.exe
3+
*.exe~
4+
*.dll
5+
*.so
6+
*.dylib
7+
coverage.txt
8+
/test-reports
9+
10+
# Test binary, build with `go test -c`
11+
*.test
12+
13+
# Output of the go coverage tool, specifically when used with LiteIDE
14+
*.out
15+
16+
# file system and ide files
17+
.DS_Store
18+
/.idea
19+
/*.iml
20+
/bin
21+
/target
22+
23+
secrets
24+
pkg_test.go
25+
api/docs/*.html

‎README.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
# Runner Init
22

3-
An image for bootstrapping container runner tasks
3+
This repo contains code and images responsible for bootstrapping Runner docker tasks. At present there are two components:
4+
5+
- The runner init image definition
6+
- A fake task agent used in Kubernetes acceptance testing

‎do

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,65 @@ set -eu -o pipefail
44

55
# This variable is used, but shellcheck can't tell.
66
# shellcheck disable=SC2034
7-
help_download_taskagent="Download task agent via the picard image"
8-
download-taskagent() {
9-
arch=${1:?'task agent arch must be specified'}
10-
7+
help_download_taskagents="Download task agents via the picard image"
8+
download-taskagents() {
119
id=$(docker create circleci/picard:agent)
12-
docker cp "$id":/opt/circleci/linux/"$arch"/circleci-agent ./circleci-agent-"$arch"
10+
11+
docker cp "$id":/opt/circleci/linux/amd64/circleci-agent ./bin/circleci-agent-amd64
12+
docker cp "$id":/opt/circleci/linux/arm64/circleci-agent ./bin/circleci-agent-arm64
13+
1314
docker rm -v "$id"
1415
}
1516

17+
# This variable is used, but shellcheck can't tell.
18+
# shellcheck disable=SC2034
19+
help_build_fake_agents="Build the fake agent go binaries"
20+
build-fake-agents() {
21+
GOOS=linux GOARCH=amd64 go build -C ./fake-agent -o ../bin/circleci-fake-agent-amd64 ./
22+
GOOS=linux GOARCH=arm64 go build -C ./fake-agent -o ../bin/circleci-fake-agent-arm64 ./
23+
}
24+
25+
# This variable is used, but shellcheck can't tell.
26+
# shellcheck disable=SC2034
27+
help_build_docker_images="Build the runner init images"
28+
build-docker-images() {
29+
repo=${1:?'image repo name must be specified'}
30+
arch=${2:?'image arch must be specified'}
31+
32+
docker build -t circleci/"$repo":agent-"$arch" --build-arg ARCH="$arch" -f ./runner-init/Dockerfile .
33+
docker build -t circleci/"$repo":test-agent-"$arch" --build-arg ARCH="$arch" -f ./runner-init/fake-agent.Dockerfile .
34+
}
35+
36+
# This variable is used, but shellcheck can't tell.
37+
# shellcheck disable=SC2034
38+
help_publish_docker_images="Publish the runner init images"
39+
publish-docker-images() {
40+
repo=${1:?'image repo name must be specified'}
41+
arch=${2:?'image arch must be specified'}
42+
43+
docker push circleci/"$repo":agent-"$arch"
44+
docker push circleci/"$repo":test-agent-"$arch"
45+
}
46+
47+
# This variable is used, but shellcheck can't tell.
48+
# shellcheck disable=SC2034
49+
help_publish_docker_manifest="Publish the multiarch manifest"
50+
publish-docker-manifest() {
51+
repo=${1:?'image repo name must be specified'}
52+
53+
docker manifest create circleci/runner-init:agent \
54+
--amend circleci/runner-init:agent-amd64 \
55+
--amend circleci/runner-init:agent-arm64
56+
57+
docker manifest push circleci/runner-init:agent
58+
59+
docker manifest create circleci/runner-init:test-agent \
60+
--amend circleci/runner-init:test-agent-amd64 \
61+
--amend circleci/runner-init:test-agent-arm64
62+
63+
docker manifest push circleci/runner-init:test-agent
64+
}
65+
1666
help-text-intro() {
1767
echo "
1868
DO

‎fake-agent/go.mod

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module github.com/circleci/circleci-fake-agent
2+
3+
go 1.21.0
4+
5+
require github.com/alecthomas/kong v0.9.0

‎fake-agent/go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
2+
github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
3+
github.com/alecthomas/kong v0.9.0 h1:G5diXxc85KvoV2f0ZRVuMsi45IrBgx9zDNGNj165aPA=
4+
github.com/alecthomas/kong v0.9.0/go.mod h1:Y47y5gKfHp1hDc7CH7OeXgLIpp+Q2m1Ni0L5s3bI8Os=
5+
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
6+
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
7+
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
8+
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=

‎fake-agent/main.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"io"
8+
"net/http"
9+
"net/url"
10+
"os"
11+
"os/exec"
12+
"os/signal"
13+
"strings"
14+
"syscall"
15+
"time"
16+
17+
"github.com/alecthomas/kong"
18+
)
19+
20+
func main() {
21+
// Example arguments: _internal agent-runner --verbose --agentAPITaskToken=a84dc93b7900c6ca3e74091785d4a883 --runnerAPIBaseURL=http://127.0.0.1:65244
22+
23+
opts := CommandRequest{}
24+
25+
kong.Parse(&opts, kong.Name("circleci-agent"))
26+
27+
// We need the fake task agent binary to be able to access the local test API from inside the cluster. If we are using kind (or another docker hosted k8s cluster)
28+
// for testing then using the domain below will forward requests on to the host machine's localhost interface
29+
driver := os.Getenv("DRIVER_MODE")
30+
dockerHostAddress := os.Getenv("DOCKER_HOST")
31+
if driver == "kubernetes" {
32+
url, err := url.Parse(opts.RunnerAPIBaseURL)
33+
mustNotError(err)
34+
url.Host = fmt.Sprintf("%s:%s", dockerHostAddress, url.Port())
35+
opts.RunnerAPIBaseURL = url.String()
36+
}
37+
38+
run(opts)
39+
}
40+
41+
func run(opts CommandRequest) {
42+
fmt.Println("fake circleci-agent 1f3k128h")
43+
44+
b, err := io.ReadAll(os.Stdin)
45+
mustNotError(err)
46+
opts.Stdin = string(b)
47+
48+
b, err = json.Marshal(opts)
49+
mustNotError(err)
50+
res, err := http.Post(opts.RunnerAPIBaseURL+"/fakes/agent/command", "application/json", bytes.NewReader(b))
51+
mustNotError(err)
52+
mustNotError(res.Body.Close())
53+
54+
terminate := make(chan os.Signal, 1)
55+
signal.Notify(terminate, syscall.SIGTERM, os.Interrupt)
56+
57+
shutdownTimer := time.NewTimer(time.Second * 4)
58+
for {
59+
intervalTimer := time.NewTimer(50 * time.Millisecond)
60+
select {
61+
case <-shutdownTimer.C:
62+
intervalTimer.Stop()
63+
fmt.Println("fake circleci-agent shutting down after timeout")
64+
return
65+
case <-intervalTimer.C:
66+
resp, err := http.Get(opts.RunnerAPIBaseURL + "/fakes/agent/terminated")
67+
if err != nil {
68+
// Exit if the test API has been closed
69+
return
70+
}
71+
mustNotError(resp.Body.Close())
72+
73+
if resp.StatusCode == http.StatusOK {
74+
shutdownTimer.Stop()
75+
fmt.Println("fake circleci-agent shutting down at the request of the test harness")
76+
resp, err := http.Post(opts.RunnerAPIBaseURL+"/fakes/agent/terminated", "",
77+
strings.NewReader(`{"by":"test-api"}`))
78+
mustNotError(err)
79+
mustNotError(resp.Body.Close())
80+
return
81+
}
82+
// Ignore the 413 as a thing. This is just a handy code to pick to trigger us to start misbehaving
83+
if resp.StatusCode == http.StatusRequestEntityTooLarge {
84+
fmt.Println("fake circleci-agent starting to misbehave")
85+
launchChildAndMisbehave(opts)
86+
panic("shouldn't get here in a test")
87+
}
88+
case <-terminate:
89+
intervalTimer.Stop()
90+
shutdownTimer.Stop()
91+
fmt.Println("fake circleci-agent shutting down after SIGTERM, notifying API")
92+
resp, err := http.Post(opts.RunnerAPIBaseURL+"/fakes/agent/terminated", "",
93+
strings.NewReader(`{"by":"launch-agent"}`))
94+
mustNotError(err)
95+
mustNotError(resp.Body.Close())
96+
return
97+
}
98+
}
99+
}
100+
101+
// Misbehaving mode to test PID group cleanup
102+
func launchChildAndMisbehave(opts CommandRequest) {
103+
signal.Ignore(os.Interrupt)
104+
105+
cmd := exec.Command("/bin/sleep", "20")
106+
err := cmd.Start()
107+
mustNotError(err)
108+
109+
res, err := http.Get(fmt.Sprintf("%s/fakes/agent/pids?taskagent=%d&taskagent-child=%d",
110+
opts.RunnerAPIBaseURL, os.Getpid(), cmd.Process.Pid))
111+
_ = res.Body.Close()
112+
mustNotError(err)
113+
114+
err = cmd.Wait()
115+
mustNotError(err)
116+
}
117+
118+
func mustNotError(err error) {
119+
if err != nil {
120+
panic(err)
121+
}
122+
}
123+
124+
type CommandRequest struct {
125+
RunnerAPIBaseURL string `name:"runnerAPIBaseURL" required:"true"`
126+
Allocation string `name:"allocation" required:"true"`
127+
Verbose bool `name:"verbose"`
128+
DisableSpinUpStep bool `name:"disableSpinUpStep"`
129+
DisableIsolatedSSHDir bool `name:"disableIsolatedSSHDir"`
130+
Args []string `arg:""`
131+
WorkDir string `name:"workDir"`
132+
Stdin string `kong:"-"`
133+
MaxRunTime time.Duration `name:"maxRunTime"`
134+
PidFile string `name:"pidfile"`
135+
}

‎runner-init/Dockerfile

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
FROM busybox:stable-musl as build
2+
3+
ARG ARCH=amd64
4+
5+
COPY ./bin/circleci-agent-${ARCH} /opt/circleci/circleci-agent
6+
RUN ln -s /opt/circleci/circleci-agent /opt/circleci/circleci
7+
8+
FROM scratch as temp
9+
10+
COPY --from=build /opt/circleci/circleci-agent /opt/circleci/circleci-agent
11+
COPY --from=build /opt/circleci/circleci /opt/circleci/circleci
12+
COPY --from=build /bin/cp /bin/cp
13+
14+
FROM scratch
15+
16+
COPY --from=temp / /
17+
18+
ENTRYPOINT ["/bin/cp"]

0 commit comments

Comments
 (0)