Skip to content

Commit 0cf690e

Browse files
samruddhikhandalesamruddhikhandale
authored
[docker in docker] Adds retries for docker daemon startup and cgroup nesting (#669)
* adds retry logic * adds stress test workflow * Adds cgroup retries * update workflow * move sleep cmd * update test * cleanup
1 parent a08434f commit 0cf690e

File tree

6 files changed

+139
-31
lines changed

6 files changed

+139
-31
lines changed

‎.github/workflows/docker-in-docker-stress-test.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: "Stress test - Docker in Docker"
2+
on:
3+
pull_request:
4+
paths:
5+
- 'src/docker-in-docker/**'
6+
workflow_dispatch:
7+
8+
jobs:
9+
test:
10+
strategy:
11+
matrix:
12+
test-pass: [ 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50 ]
13+
fail-fast: false
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v3
17+
18+
- name: "Install latest devcontainer CLI"
19+
run: npm install -g @devcontainers/cli
20+
21+
- name: "Generating tests for 'docker-in-docker' which validates if docker daemon is running"
22+
run: devcontainer features test --skip-scenarios -f docker-in-docker -i mcr.microsoft.com/devcontainers/base:ubuntu .
23+
24+
test-onCreate:
25+
strategy:
26+
matrix:
27+
test-pass: [ 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50 ]
28+
fail-fast: false
29+
runs-on: ubuntu-latest
30+
steps:
31+
- uses: actions/checkout@v3
32+
33+
- name: "Install latest devcontainer CLI"
34+
run: npm install -g @devcontainers/cli
35+
36+
- name: "Generating tests for 'docker-in-docker' which validates if docker daemon is available within 'onCreateCommand'"
37+
run: devcontainer features test -f docker-in-docker --skip-autogenerated --filter "docker_with_on_create_command"

‎src/docker-in-docker/devcontainer-feature.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"id": "docker-in-docker",
3-
"version": "2.3.1",
3+
"version": "2.4.0",
44
"name": "Docker (Docker-in-Docker)",
55
"documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker",
66
"description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.",

‎src/docker-in-docker/install.sh‎

Lines changed: 73 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
367367
find /run /var/run -iname 'docker*.pid' -delete || :
368368
find /run /var/run -iname 'container*.pid' -delete || :
369369
370-
## Dind wrapper script from docker team, adapted to a function
370+
# -- Start: dind wrapper script --
371371
# Maintained: https://github.com/moby/moby/blob/master/hack/dind
372372
373373
export container=docker
@@ -384,31 +384,52 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
384384
mount -t tmpfs none /tmp
385385
fi
386386
387-
# cgroup v2: enable nesting
388-
if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
389-
# move the processes from the root group to the /init group,
390-
# otherwise writing subtree_control fails with EBUSY.
391-
# An error during moving non-existent process (i.e., "cat") is ignored.
392-
mkdir -p /sys/fs/cgroup/init
393-
xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
394-
# enable controllers
395-
sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
396-
> /sys/fs/cgroup/cgroup.subtree_control
397-
fi
398-
## Dind wrapper over.
387+
set_cgroup_nesting()
388+
{
389+
# cgroup v2: enable nesting
390+
if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
391+
# move the processes from the root group to the /init group,
392+
# otherwise writing subtree_control fails with EBUSY.
393+
# An error during moving non-existent process (i.e., "cat") is ignored.
394+
mkdir -p /sys/fs/cgroup/init
395+
xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
396+
# enable controllers
397+
sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
398+
> /sys/fs/cgroup/cgroup.subtree_control
399+
fi
400+
}
401+
402+
# Set cgroup nesting, retrying if necessary
403+
retry_cgroup_nesting=0
404+
405+
until [ "${retry_cgroup_nesting}" -eq "5" ];
406+
do
407+
set +e
408+
set_cgroup_nesting
409+
410+
if [ $? -ne 0 ]; then
411+
echo "(*) cgroup v2: Failed to enable nesting, retrying..."
412+
else
413+
break
414+
fi
415+
416+
retry_cgroup_nesting=`expr $retry_cgroup_nesting + 1`
417+
set -e
418+
done
419+
420+
# -- End: dind wrapper script --
399421
400422
# Handle DNS
401423
set +e
402-
cat /etc/resolv.conf | grep -i 'internal.cloudapp.net'
403-
if [ $? -eq 0 ] && [ "${AZURE_DNS_AUTO_DETECTION}" = "true" ]
404-
then
405-
echo "Setting dockerd Azure DNS."
406-
CUSTOMDNS="--dns 168.63.129.16"
407-
else
408-
echo "Not setting dockerd DNS manually."
409-
CUSTOMDNS=""
410-
fi
411-
424+
cat /etc/resolv.conf | grep -i 'internal.cloudapp.net' > /dev/null 2>&1
425+
if [ $? -eq 0 ] && [ "${AZURE_DNS_AUTO_DETECTION}" = "true" ]
426+
then
427+
echo "Setting dockerd Azure DNS."
428+
CUSTOMDNS="--dns 168.63.129.16"
429+
else
430+
echo "Not setting dockerd DNS manually."
431+
CUSTOMDNS=""
432+
fi
412433
set -e
413434
414435
if [ -z "$DOCKER_DEFAULT_ADDRESS_POOL" ]
@@ -423,12 +444,35 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
423444
INNEREOF
424445
)"
425446
426-
# Start using sudo if not invoked as root
427-
if [ "$(id -u)" -ne 0 ]; then
428-
sudo /bin/sh -c "${dockerd_start}"
429-
else
430-
eval "${dockerd_start}"
431-
fi
447+
retry_docker_start_count=0
448+
docker_ok="false"
449+
450+
until [ "${docker_ok}" = "true" ] || [ "${retry_docker_start_count}" -eq "5" ];
451+
do
452+
# Start using sudo if not invoked as root
453+
if [ "$(id -u)" -ne 0 ]; then
454+
sudo /bin/sh -c "${dockerd_start}"
455+
else
456+
eval "${dockerd_start}"
457+
fi
458+
459+
retry_count=0
460+
until [ "${docker_ok}" = "true" ] || [ "${retry_count}" -eq "5" ];
461+
do
462+
sleep 1s
463+
set +e
464+
docker info > /dev/null 2>&1 && docker_ok="true"
465+
set -e
466+
467+
retry_count=`expr $retry_count + 1`
468+
done
469+
470+
if [ "${docker_ok}" != "true" ]; then
471+
echo "(*) Failed to start docker, retrying..."
472+
fi
473+
474+
retry_docker_start_count=`expr $retry_docker_start_count + 1`
475+
done
432476
433477
# Execute whatever commands were passed in (if any). This allows us
434478
# to set this script to ENTRYPOINT while still executing the default CMD.

‎test/docker-in-docker/docker_with_on_create_command.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
# Optional: Import test library
6+
source dev-container-features-test-lib
7+
8+
# Feature specific tests
9+
check "version" docker --version
10+
check "docker-ps" bash -c "docker ps"
11+
12+
check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
13+
check "log-exists" bash -c "ls /tmp/dockerd.log"
14+
check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
15+
check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
16+
17+
# Report result
18+
reportResults

‎test/docker-in-docker/scenarios.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,5 +67,14 @@
6767
}
6868
},
6969
"remoteUser": "node"
70+
},
71+
// DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow
72+
"docker_with_on_create_command": {
73+
"image": "mcr.microsoft.com/devcontainers/base:debian",
74+
"features": {
75+
"docker-in-docker": {}
76+
},
77+
"remoteUser": "vscode",
78+
"onCreateCommand": "docker ps && sleep 5s && docker ps"
7079
}
7180
}

‎test/docker-in-docker/test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ source dev-container-features-test-lib
88
# Feature specific tests
99
check "version" docker --version
1010
check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh"
11+
check "docker-ps" bash -c "docker ps"
1112
check "log-exists" bash -c "ls /tmp/dockerd.log"
1213
check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'"
1314
check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'"
14-
check "docker-ps" bash -c "docker ps"
1515

1616
# Report result
1717
reportResults

0 commit comments

Comments
 (0)