Skip to content

Commit 1480fcd

Browse files
npalmnpalm
authored
chor: Update ubuntu example with rootless docker and non privileged user (#433)
* Update ubuntu example: - stream log to clouwatch - re-use install and configure from module - add rootless docker - install in non privileges user account (runners) Update module - refactor user_data, splitting in seperate parts * Run pre-commit forced
1 parent bba7b90 commit 1480fcd

File tree

8 files changed

+227
-172
lines changed

8 files changed

+227
-172
lines changed

‎README.md

Lines changed: 60 additions & 60 deletions
Large diffs are not rendered by default.

‎examples/ubuntu/main.tf

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ module "runners" {
2727
webhook_secret = random_password.random.result
2828
}
2929

30-
webhook_lambda_zip = "lambdas-download/webhook.zip"
31-
runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip"
32-
runners_lambda_zip = "lambdas-download/runners.zip"
30+
# webhook_lambda_zip = "lambdas-download/webhook.zip"
31+
# runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip"
32+
# runners_lambda_zip = "lambdas-download/runners.zip"
3333

3434
enable_organization_runners = false
3535
runner_extra_labels = "ubuntu,example"
@@ -49,6 +49,17 @@ module "runners" {
4949
device_name = "/dev/sda1"
5050
}
5151

52+
runner_log_files = [
53+
{
54+
"file_path" : "/var/log/user-data.log",
55+
"log_stream_name" : "{instance_id}/user_data"
56+
},
57+
{
58+
"file_path" : "/home/runners/actions-runner/_diag/Runner_**.log",
59+
"log_stream_name" : "{instance_id}/runner"
60+
}
61+
]
62+
5263
# Uncommet idle config to have idle runners from 9 to 5 in time zone Amsterdam
5364
# idle_config = [{
5465
# cron = "* * 9-17 * * *"

‎examples/ubuntu/templates/user-data.sh

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,63 @@
1-
#!/bin/bash -e
1+
#!/bin/bash -x
22
exec > >(tee /var/log/user-data.log | logger -t user-data -s 2>/dev/console) 2>&1
33

4+
${pre_install}
5+
46
# Install AWS CLI
57
apt-get update
6-
DEBIAN_FRONTEND=noninteractive apt-get install -y awscli jq
8+
DEBIAN_FRONTEND=noninteractive apt-get install -y awscli jq curl wget git uidmap
9+
10+
USER_NAME=runners
11+
useradd -m -s /bin/bash $USER_NAME
12+
USER_ID=$(id -ru $USER_NAME)
13+
14+
# install and configure cloudwatch logging agent
15+
wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
16+
dpkg -i -E ./amazon-cloudwatch-agent.deb
17+
amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${ssm_key_cloudwatch_agent_config}
18+
19+
# configure systemd for running service in users accounts
20+
cat >/etc/systemd/[email protected] <<-EOF
21+
22+
[Unit]
23+
Description=User Manager for UID %i
24+
After=user-runtime-dir@%i.service
25+
Wants=user-runtime-dir@%i.service
26+
27+
[Service]
28+
LimitNOFILE=infinity
29+
LimitNPROC=infinity
30+
User=%i
31+
PAMName=systemd-user
32+
Type=notify
33+
34+
[Install]
35+
WantedBy=default.target
736
8-
# Install runner
9-
cd /home/ubuntu
10-
mkdir actions-runner && cd actions-runner
37+
EOF
1138

12-
aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz
13-
tar xzf ./actions-runner.tar.gz
14-
rm -rf actions-runner.tar.gz
39+
echo export XDG_RUNTIME_DIR=/run/user/$USER_ID >>/home/$USER_NAME/.profile
1540

16-
INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
17-
REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
41+
systemctl daemon-reload
42+
systemctl enable [email protected]
43+
systemctl start [email protected]
1844

19-
echo wait for configuration
20-
while [[ $(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") == null ]]; do
21-
echo Waiting for configuration ...
22-
sleep 1
23-
done
24-
CONFIG=$(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value")
25-
aws ssm delete-parameter --name ${environment}-$INSTANCE_ID --region $REGION
45+
curl -fsSL https://get.docker.com/rootless >>/opt/rootless.sh && chmod 755 /opt/rootless.sh
46+
su -l $USER_NAME -c /opt/rootless.sh
47+
echo export DOCKER_HOST=unix:///run/user/$USER_ID/docker.sock >>/home/$USER_NAME/.profile
48+
echo export PATH=/home/$USER_NAME/bin:$PATH >>/home/$USER_NAME/.profile
2649

27-
export RUNNER_ALLOW_RUNASROOT=1
50+
# Run docker service by default
51+
loginctl enable-linger $USER_NAME
52+
su -l $USER_NAME -c "systemctl --user enable docker"
2853

29-
sudo -u ubuntu mkdir /home/ubuntu/work
54+
${install_config_runner}
3055

31-
./bin/installdependencies.sh
32-
./config.sh --unattended --name $INSTANCE_ID --work "/home/ubuntu/work" $CONFIG
56+
# config runner for rootless docker
57+
cd /home/$USER_NAME/actions-runner/
58+
echo DOCKER_HOST=unix:///run/user/$USER_ID/docker.sock >>.env
59+
echo PATH=/home/$USER_NAME/bin:$PATH >>.env
3360

34-
chown -R ubuntu:ubuntu .
35-
./svc.sh install ubuntu
61+
${post_install}
3662

3763
./svc.sh start

‎main.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ module "runner_binaries" {
119119

120120
distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}"
121121

122-
runner_architecture = substr(var.instance_type, 0, 2) == "a1" || substr(var.instance_type, 1, 2) == "6g" ? "arm64" : "x64"
122+
runner_architecture = local.runner_architecture
123123
runner_allow_prerelease_binaries = var.runner_allow_prerelease_binaries
124124

125125
lambda_s3_bucket = var.lambda_s3_bucket

‎modules/runners/main.tf

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@ locals {
99
var.tags,
1010
)
1111

12-
name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
13-
name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"]
14-
role_path = var.role_path == null ? "/${var.environment}/" : var.role_path
15-
instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path
16-
lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip
17-
userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template
12+
name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
13+
name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"]
14+
role_path = var.role_path == null ? "/${var.environment}/" : var.role_path
15+
instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path
16+
lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip
17+
userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template
18+
userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl"
19+
userdata_install_config_runner = "${path.module}/templates/install-config-runner.sh"
1820
}
1921

2022
data "aws_ami" "runner" {
@@ -81,16 +83,24 @@ resource "aws_launch_template" "runner" {
8183
environment = var.environment
8284
pre_install = var.userdata_pre_install
8385
post_install = var.userdata_post_install
84-
s3_location_runner_distribution = var.s3_location_runner_binaries
85-
service_user = var.runner_as_root ? "root" : "ec2-user"
86-
runner_architecture = var.runner_architecture
8786
enable_cloudwatch_agent = var.enable_cloudwatch_agent
8887
ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : ""
88+
install_config_runner = local.install_config_runner
8989
}))
9090

9191
tags = local.tags
9292
}
9393

94+
locals {
95+
arm_patch = var.runner_architecture == "arm64" ? templatefile(local.userdata_arm_patch, {}) : ""
96+
install_config_runner = templatefile(local.userdata_install_config_runner, {
97+
environment = var.environment
98+
s3_location_runner_distribution = var.s3_location_runner_binaries
99+
run_as_root_user = var.runner_as_root ? "root" : ""
100+
arm_patch = local.arm_patch
101+
})
102+
}
103+
94104
resource "aws_security_group" "runner_sg" {
95105
name_prefix = "${var.environment}-github-actions-runner-sg"
96106
description = "Github Actions Runner security group"

‎modules/runners/templates/arm-runner-patch.tpl

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Patch for ARM64 (no ICU install by default)
2+
yum install -y patch
3+
patch -p1 <<ICU_PATCH
4+
diff -Naur a/bin/Runner.Listener.runtimeconfig.json b/bin/Runner.Listener.runtimeconfig.json
5+
--- a/bin/Runner.Listener.runtimeconfig.json 2020-07-01 02:21:09.000000000 +0000
6+
+++ b/bin/Runner.Listener.runtimeconfig.json 2020-07-28 00:02:38.748868613 +0000
7+
@@ -8,7 +8,8 @@
8+
}
9+
],
10+
"configProperties": {
11+
- "System.Runtime.TieredCompilation.QuickJit": true
12+
+ "System.Runtime.TieredCompilation.QuickJit": true,
13+
+ "System.Globalization.Invariant": true
14+
}
15+
}
16+
-}
17+
\ No newline at end of file
18+
+}
19+
diff -Naur a/bin/Runner.PluginHost.runtimeconfig.json b/bin/Runner.PluginHost.runtimeconfig.json
20+
--- a/bin/Runner.PluginHost.runtimeconfig.json 2020-07-01 02:21:22.000000000 +0000
21+
+++ b/bin/Runner.PluginHost.runtimeconfig.json 2020-07-28 00:02:59.358680003 +0000
22+
@@ -8,7 +8,8 @@
23+
}
24+
],
25+
"configProperties": {
26+
- "System.Runtime.TieredCompilation.QuickJit": true
27+
+ "System.Runtime.TieredCompilation.QuickJit": true,
28+
+ "System.Globalization.Invariant": true
29+
}
30+
}
31+
-}
32+
\ No newline at end of file
33+
+}
34+
diff -Naur a/bin/Runner.Worker.runtimeconfig.json b/bin/Runner.Worker.runtimeconfig.json
35+
--- a/bin/Runner.Worker.runtimeconfig.json 2020-07-01 02:21:16.000000000 +0000
36+
+++ b/bin/Runner.Worker.runtimeconfig.json 2020-07-28 00:02:19.159028531 +0000
37+
@@ -8,7 +8,8 @@
38+
}
39+
],
40+
"configProperties": {
41+
- "System.Runtime.TieredCompilation.QuickJit": true
42+
+ "System.Runtime.TieredCompilation.QuickJit": true,
43+
+ "System.Globalization.Invariant": true
44+
}
45+
}
46+
-}
47+
\ No newline at end of file
48+
+}
49+
ICU_PATCH

‎modules/runners/templates/install-config-runner.sh

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
cd /home/$USER_NAME
2+
mkdir actions-runner && cd actions-runner
3+
4+
aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz
5+
tar xzf ./actions-runner.tar.gz
6+
rm -rf actions-runner.tar.gz
7+
8+
${arm_patch}
9+
10+
INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
11+
REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
12+
13+
echo wait for configuration
14+
while [[ $(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") == null ]]; do
15+
echo Waiting for configuration ...
16+
sleep 1
17+
done
18+
CONFIG=$(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value")
19+
aws ssm delete-parameter --name ${environment}-$INSTANCE_ID --region $REGION
20+
21+
export RUNNER_ALLOW_RUNASROOT=1
22+
os_id=$(awk -F= '/^ID/{print $2}' /etc/os-release)
23+
if [[ "$os_id" =~ ^ubuntu.* ]]; then
24+
./bin/installdependencies.sh
25+
fi
26+
27+
./config.sh --unattended --name $INSTANCE_ID --work "_work" $CONFIG
28+
29+
chown -R $USER_NAME:$USER_NAME .
30+
OVERWRITE_SERVICE_USER=${run_as_root_user}
31+
SERVICE_USER=$${OVERWRITE_SERVICE_USER:-$USER_NAME}
32+
33+
./svc.sh install $SERVICE_USER

‎modules/runners/templates/user-data.sh

Lines changed: 2 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -17,82 +17,8 @@ usermod -a -G docker ec2-user
1717

1818
yum install -y curl jq git
1919

20-
# Install runner
21-
cd /home/ec2-user
22-
mkdir actions-runner && cd actions-runner
23-
24-
aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz
25-
tar xzf ./actions-runner.tar.gz
26-
rm -rf actions-runner.tar.gz
27-
28-
%{ if runner_architecture == "arm64" ~}
29-
# Patch for ARM64 (no ICU install by default)
30-
yum install -y patch
31-
patch -p1 <<ICU_PATCH
32-
diff -Naur a/bin/Runner.Listener.runtimeconfig.json b/bin/Runner.Listener.runtimeconfig.json
33-
--- a/bin/Runner.Listener.runtimeconfig.json 2020-07-01 02:21:09.000000000 +0000
34-
+++ b/bin/Runner.Listener.runtimeconfig.json 2020-07-28 00:02:38.748868613 +0000
35-
@@ -8,7 +8,8 @@
36-
}
37-
],
38-
"configProperties": {
39-
- "System.Runtime.TieredCompilation.QuickJit": true
40-
+ "System.Runtime.TieredCompilation.QuickJit": true,
41-
+ "System.Globalization.Invariant": true
42-
}
43-
}
44-
-}
45-
\ No newline at end of file
46-
+}
47-
diff -Naur a/bin/Runner.PluginHost.runtimeconfig.json b/bin/Runner.PluginHost.runtimeconfig.json
48-
--- a/bin/Runner.PluginHost.runtimeconfig.json 2020-07-01 02:21:22.000000000 +0000
49-
+++ b/bin/Runner.PluginHost.runtimeconfig.json 2020-07-28 00:02:59.358680003 +0000
50-
@@ -8,7 +8,8 @@
51-
}
52-
],
53-
"configProperties": {
54-
- "System.Runtime.TieredCompilation.QuickJit": true
55-
+ "System.Runtime.TieredCompilation.QuickJit": true,
56-
+ "System.Globalization.Invariant": true
57-
}
58-
}
59-
-}
60-
\ No newline at end of file
61-
+}
62-
diff -Naur a/bin/Runner.Worker.runtimeconfig.json b/bin/Runner.Worker.runtimeconfig.json
63-
--- a/bin/Runner.Worker.runtimeconfig.json 2020-07-01 02:21:16.000000000 +0000
64-
+++ b/bin/Runner.Worker.runtimeconfig.json 2020-07-28 00:02:19.159028531 +0000
65-
@@ -8,7 +8,8 @@
66-
}
67-
],
68-
"configProperties": {
69-
- "System.Runtime.TieredCompilation.QuickJit": true
70-
+ "System.Runtime.TieredCompilation.QuickJit": true,
71-
+ "System.Globalization.Invariant": true
72-
}
73-
}
74-
-}
75-
\ No newline at end of file
76-
+}
77-
ICU_PATCH
78-
%{ endif ~}
79-
80-
INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
81-
REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
82-
83-
echo wait for configuration
84-
while [[ $(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") == null ]]; do
85-
echo Waiting for configuration ...
86-
sleep 1
87-
done
88-
CONFIG=$(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value")
89-
aws ssm delete-parameter --name ${environment}-$INSTANCE_ID --region $REGION
90-
91-
export RUNNER_ALLOW_RUNASROOT=1
92-
./config.sh --unattended --name $INSTANCE_ID --work "_work" $CONFIG
93-
94-
chown -R ec2-user:ec2-user .
95-
./svc.sh install ${service_user}
20+
USER_NAME=ec2-user
21+
${install_config_runner}
9622

9723
${post_install}
9824

0 commit comments

Comments
 (0)