Skip to content

Commit a12943a

Browse files
matzfjuagargi
matzf
authored and
juagargi
committed
tools: specify project name in docker-compose files (scionproto#4396)
Simplify the usage of the various docker-compose configurations by including the project name in the configuration file. This has been supported for a while now in docker-compose v2. This allows to drop the `-p`/`--project-name` from all `docker-compose` incantations. Also, streamline the docker-compose files generated by the topogen scripts; remove the explicit `container_name` configurations and drop all the explicit `scion_` prefixes -- managing these prefixes is docker-compose's job. Shut up the warnings on "SCION_EXPERIMENTAL_... variable is not set. Defaulting to a blank string." by using the variable expansion syntax to explicitly default to a blank string. Also, drop the `docker compose` `--compatibility` flag. The compatibility flag affects what word separator is used in the container name and it has long been deprecated. We don't usually rely on specific container names, so this should be fine. In some exception cases where expecting specific container names seems more practical (containers for bazel-remote-cache and go-module-proxy) due to special casing in the CI scripts, container_name is set explicitly.
1 parent 0f78c34 commit a12943a

File tree

22 files changed

+86
-122
lines changed

22 files changed

+86
-122
lines changed

‎.buildkite/hooks/bazel-remote.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
version: "2.4"
2+
name: bazel_remote
23
services:
34
bazel-remote:
45
container_name: bazel-remote-cache

‎.buildkite/hooks/go-module-proxy.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
---
22
version: "2.4"
3+
name: athens
34
services:
45
go-module-proxy:
56
container_name: go-module-proxy

‎.buildkite/hooks/pre-command‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ echo "~~~ Starting bazel remote cache proxy"
4343
# Start bazel remote cache proxy for S3
4444
# Note that S3 keys are injected by buildkite, see
4545
# https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws
46-
docker compose --compatibility -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d
46+
docker compose -f .buildkite/hooks/bazel-remote.yml up -d
4747

4848
echo "~~~ Starting go module proxy"
49-
docker compose --compatibility -f .buildkite/hooks/go-module-proxy.yml -p athens up -d
49+
docker compose -f .buildkite/hooks/go-module-proxy.yml up -d

‎acceptance/cert_renewal/test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,9 @@ def _run(self):
7272
end2end.run_fg()
7373

7474
logger.info("==> Shutting down control servers and purging caches")
75-
for container in self.dc.list_containers("scion_sd.*"):
75+
for container in self.dc.list_containers("sd.*"):
7676
self.dc("rm", container)
77-
for container in self.dc.list_containers("scion_cs.*"):
77+
for container in self.dc.list_containers("cs.*"):
7878
self.dc.stop_container(container)
7979
for cs_config in cs_configs:
8080
files = list((pathlib.Path(self.artifacts) /

‎acceptance/common/docker.py‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,24 +35,20 @@
3535
from plumbum import cmd
3636

3737
SCION_DC_FILE = "gen/scion-dc.yml"
38-
DC_PROJECT = "scion"
3938
SCION_TESTING_DOCKER_ASSERTIONS_OFF = 'SCION_TESTING_DOCKER_ASSERTIONS_OFF'
4039

4140

4241
class Compose(object):
4342
def __init__(self,
44-
project: str = DC_PROJECT,
4543
compose_file: str = SCION_DC_FILE):
46-
self.project = project
4744
self.compose_file = compose_file
4845

4946
def __call__(self, *args, **kwargs) -> str:
5047
"""Runs docker compose with the given arguments"""
5148
# Note: not using plumbum here due to complications with encodings in the captured output
5249
try:
5350
res = subprocess.run(
54-
["docker", "compose", "--compatibility",
55-
"-f", self.compose_file, "-p", self.project, *args],
51+
["docker", "compose", "-f", self.compose_file, *args],
5652
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
5753
except subprocess.CalledProcessError as e:
5854
raise _CalledProcessErrorWithOutput(e) from None

‎acceptance/hidden_paths/test.py‎

Lines changed: 21 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,9 @@
55
import http.server
66
import threading
77

8-
from plumbum import cmd
9-
108
from acceptance.common import base
119
from acceptance.common import scion
10+
from tools.topology.scion_addr import ISD_AS
1211

1312

1413
class Test(base.TestTopogen):
@@ -108,12 +107,6 @@ def setup_start(self):
108107

109108
super().setup_start()
110109

111-
self._testers = {
112-
"2": "tester_1-ff00_0_2",
113-
"3": "tester_1-ff00_0_3",
114-
"4": "tester_1-ff00_0_4",
115-
"5": "tester_1-ff00_0_5",
116-
}
117110
self._ases = {
118111
"2": "1-ff00:0:2",
119112
"3": "1-ff00:0:3",
@@ -126,27 +119,30 @@ def _run(self):
126119
self._server.shutdown() # by now configuration must have been downloaded everywhere
127120

128121
# Group 3
129-
self._showpaths_bidirectional("2", "3", 0)
130-
self._showpaths_bidirectional("2", "5", 0)
131-
self._showpaths_bidirectional("3", "5", 0)
122+
self._showpaths_bidirectional("2", "3")
123+
self._showpaths_bidirectional("2", "5")
124+
self._showpaths_bidirectional("3", "5")
132125

133126
# Group 4
134-
self._showpaths_bidirectional("2", "4", 0)
135-
self._showpaths_bidirectional("2", "5", 0)
136-
self._showpaths_bidirectional("4", "5", 0)
127+
self._showpaths_bidirectional("2", "4")
128+
self._showpaths_bidirectional("2", "5")
129+
self._showpaths_bidirectional("4", "5")
137130

138131
# Group 3 X 4
139-
self._showpaths_bidirectional("3", "4", 1)
140-
141-
def _showpaths_bidirectional(self, source: str, destination: str, retcode: int):
142-
self._showpaths_run(source, destination, retcode)
143-
self._showpaths_run(destination, source, retcode)
144-
145-
def _showpaths_run(self, source_as: str, destination_as: str, retcode: int):
146-
print(cmd.docker("exec", "-t", self._testers[source_as], "scion",
147-
"sp", self._ases[destination_as],
148-
"--timeout", "2s",
149-
retcode=retcode))
132+
try:
133+
self._showpaths_bidirectional("3", "4")
134+
except Exception as e:
135+
print(e)
136+
else:
137+
raise AssertionError("Unexpected success; should not have paths 3 -> 4")
138+
139+
def _showpaths_bidirectional(self, source: str, destination: str):
140+
self._showpaths_run(source, destination)
141+
self._showpaths_run(destination, source)
142+
143+
def _showpaths_run(self, source_as: str, destination_as: str):
144+
print(self.execute_tester(ISD_AS(self._ases[source_as]),
145+
"scion", "sp", self._ases[destination_as], "--timeout", "2s"))
150146

151147

152148
def configuration_server(server):

‎acceptance/sig_short_exp_time/test‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,24 +55,24 @@ run_test() {(set -e
5555
docker image load -i acceptance/sig_short_exp_time/sig1.tar
5656
docker image load -i acceptance/sig_short_exp_time/sig2.tar
5757

58-
docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml up -d dispatcher1 dispatcher2 sig1 sig2 patha pathb
58+
docker compose -f acceptance/sig_short_exp_time/docker-compose.yml up -d dispatcher1 dispatcher2 sig1 sig2 patha pathb
5959

6060
# Set up forward route on network stack 1 and 2 through the sig tunnel
6161
# device. The route is a property of the network stack, and persists after
6262
# the container that added it has exited.
6363
#
6464
# If the route configuration fails, the test is not stopped.
65-
docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml run --name route1 --rm tester1 ip route add 242.254.200.2/32 dev sig || true
66-
docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml run --name route2 --rm tester2 ip route add 242.254.100.2/32 dev sig || true
65+
docker compose -f acceptance/sig_short_exp_time/docker-compose.yml run --name route1 --rm tester1 ip route add 242.254.200.2/32 dev sig || true
66+
docker compose -f acceptance/sig_short_exp_time/docker-compose.yml run --name route2 --rm tester2 ip route add 242.254.100.2/32 dev sig || true
6767

6868
echo "Start background ping, ping every 0.2 seconds"
69-
docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml run --name tester1 -d tester1 ping -i 0.2 242.254.200.2
69+
docker compose -f acceptance/sig_short_exp_time/docker-compose.yml run --name tester1 -d tester1 ping -i 0.2 242.254.200.2
7070

7171
echo "Waiting 10 seconds for path A to expire..."
7272
sleep 10
7373
echo "Path A expired, simulating it by shutting down path A proxy"
7474
# Traffic should have switched beforehand to path b, and no pings should be lost
75-
docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml stop patha
75+
docker compose -f acceptance/sig_short_exp_time/docker-compose.yml stop patha
7676
sleep 1
7777
docker kill -s SIGINT tester1
7878

@@ -104,9 +104,9 @@ OUTPUT_DIR=$TEST_UNDECLARED_OUTPUTS_DIR
104104
mkdir -p $OUTPUT_DIR/logs
105105

106106
for CNTR in sig1 sig2 dispatcher1 dispatcher2; do
107-
docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml logs "$CNTR" > "$OUTPUT_DIR/logs/$CNTR.log"
107+
docker compose -f acceptance/sig_short_exp_time/docker-compose.yml logs "$CNTR" > "$OUTPUT_DIR/logs/$CNTR.log"
108108
done
109109

110-
docker compose --compatibility -f acceptance/sig_short_exp_time/docker-compose.yml down -v
110+
docker compose -f acceptance/sig_short_exp_time/docker-compose.yml down -v
111111

112112
exit $RC

‎acceptance/topo_cs_reload/docker-compose.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,13 @@ networks:
88
- subnet: 242.253.100.0/24
99
services:
1010
topo_cs_reload_dispatcher:
11-
container_name: topo_cs_reload_dispatcher
1211
image: bazel/acceptance/topo_cs_reload:dispatcher
1312
networks:
1413
bridge1:
1514
ipv4_address: 242.253.100.2
1615
volumes:
1716
- vol_topo_cs_reload_disp:/run/shm/dispatcher:rw
1817
topo_cs_reload_control_srv:
19-
container_name: topo_cs_reload_control_srv
2018
image: bazel/acceptance/topo_cs_reload:control
2119
depends_on:
2220
- topo_cs_reload_dispatcher

‎acceptance/topo_cs_reload/reload_test.go‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -100,27 +100,23 @@ func setupTest(t *testing.T) testState {
100100
require.NoError(t, err)
101101
topoFile, err := bazel.Runfile(*topoLocation)
102102
require.NoError(t, err)
103-
s.mustExec(t, *genCryptoLocation, scionPKI,
104-
"crypto.tar", topoFile, cryptoLib)
103+
s.mustExec(t, *genCryptoLocation, scionPKI, "crypto.tar", topoFile, cryptoLib)
105104
s.mustExec(t, "tar", "-xf", "crypto.tar", "-C", tmpDir)
106105
// first load the docker images from bazel into the docker deamon, the
107106
// tars are in the same folder as this test runs in bazel.
108107
s.mustExec(t, "docker", "image", "load", "-i", "dispatcher.tar")
109108
s.mustExec(t, "docker", "image", "load", "-i", "control.tar")
110109
// now start the docker containers
111-
s.mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml",
112-
"up", "-d")
110+
s.mustExec(t, "docker", "compose", "-f", "docker-compose.yml", "up", "-d")
113111
// wait a bit to make sure the containers are ready.
114112
time.Sleep(time.Second / 2)
115113
t.Log("Test setup done")
116-
s.mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml",
117-
"ps")
114+
s.mustExec(t, "docker", "compose", "-f", "docker-compose.yml", "ps")
118115
return s
119116
}
120117

121118
func (s testState) teardownTest(t *testing.T) {
122-
defer s.mustExec(t, "docker", "compose", "--compatibility",
123-
"-f", "docker-compose.yml", "down", "-v")
119+
defer s.mustExec(t, "docker", "compose", "-f", "docker-compose.yml", "down", "-v")
124120

125121
outdir, exists := os.LookupEnv("TEST_UNDECLARED_OUTPUTS_DIR")
126122
require.True(t, exists, "TEST_UNDECLARED_OUTPUTS_DIR must be defined")
@@ -130,7 +126,7 @@ func (s testState) teardownTest(t *testing.T) {
130126
"topo_cs_reload_dispatcher": "disp.log",
131127
"topo_cs_reload_control_srv": "control.log",
132128
} {
133-
cmd := exec.Command("docker", "compose", "--compatibility",
129+
cmd := exec.Command("docker", "compose",
134130
"-f", "docker-compose.yml", "logs", "--no-color", service)
135131
logFileName := fmt.Sprintf("%s/logs/%s", outdir, file)
136132
logFile, err := os.Create(logFileName)
@@ -149,9 +145,9 @@ func (s testState) teardownTest(t *testing.T) {
149145
func (s testState) loadTopo(t *testing.T, name string) {
150146
t.Helper()
151147

152-
s.mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml",
148+
s.mustExec(t, "docker", "compose", "-f", "docker-compose.yml",
153149
"exec", "-T", "topo_cs_reload_control_srv", "mv", name, "/topology.json")
154-
s.mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml",
150+
s.mustExec(t, "docker", "compose", "-f", "docker-compose.yml",
155151
"kill", "-s", "SIGHUP", "topo_cs_reload_control_srv")
156152
}
157153

‎acceptance/topo_daemon_reload/reload_test.go‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,18 +71,17 @@ func setupTest(t *testing.T) {
7171
mustExec(t, "docker", "image", "load", "-i", "dispatcher.tar")
7272
mustExec(t, "docker", "image", "load", "-i", "daemon.tar")
7373
// now start the docker containers
74-
mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml",
74+
mustExec(t, "docker", "compose", "-f", "docker-compose.yml",
7575
"up", "-d", "topo_daemon_reload_dispatcher", "topo_daemon_reload_daemon")
7676
// wait a bit to make sure the containers are ready.
7777
time.Sleep(time.Second / 2)
7878
t.Log("Test setup done")
79-
mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml",
79+
mustExec(t, "docker", "compose", "-f", "docker-compose.yml",
8080
"ps")
8181
}
8282

8383
func teardownTest(t *testing.T) {
84-
defer mustExec(t, "docker", "compose", "--compatibility",
85-
"-f", "docker-compose.yml", "down", "-v")
84+
defer mustExec(t, "docker", "compose", "-f", "docker-compose.yml", "down", "-v")
8685

8786
outdir, exists := os.LookupEnv("TEST_UNDECLARED_OUTPUTS_DIR")
8887
require.True(t, exists, "TEST_UNDECLARED_OUTPUTS_DIR must be defined")
@@ -92,7 +91,7 @@ func teardownTest(t *testing.T) {
9291
"topo_daemon_reload_dispatcher": "disp.log",
9392
"topo_daemon_reload_daemon": "daemon.log",
9493
} {
95-
cmd := exec.Command("docker", "compose", "--compatibility",
94+
cmd := exec.Command("docker", "compose",
9695
"-f", "docker-compose.yml", "logs", "--no-color",
9796
service)
9897
logFileName := fmt.Sprintf("%s/logs/%s", outdir, file)
@@ -111,9 +110,9 @@ func teardownTest(t *testing.T) {
111110
func loadTopo(t *testing.T, name string) {
112111
t.Helper()
113112

114-
mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml",
113+
mustExec(t, "docker", "compose", "-f", "docker-compose.yml",
115114
"exec", "-T", "topo_daemon_reload_daemon", "mv", name, "/topology.json")
116-
mustExec(t, "docker", "compose", "--compatibility", "-f", "docker-compose.yml",
115+
mustExec(t, "docker", "compose", "-f", "docker-compose.yml",
117116
"kill", "-s", "SIGHUP", "topo_daemon_reload_daemon")
118117
}
119118

0 commit comments

Comments
 (0)