Skip to content

Commit 781a025

Browse files
tlobigtlobig
authored
Docker branch tagging and scheduler modularization (#56)
Co-authored-by: lobi_th <thomas.lobig@dlr.de> New feature in the build system: docker images are now tagged by the branch name to enable separate images to exist while working on different branches (implemented by https://github.com/akoerner1) Structural changes: further modularization on dependencies ADORe schedular changed into a module (implemented by https://github.com/n-mat)
1 parent 7e37327 commit 781a025

File tree

126 files changed

+1540
-1777
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

126 files changed

+1540
-1777
lines changed

‎.dockerignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,5 @@ catkin_workspace/
1616
**.vagrant
1717
**.catkin_workspace
1818
**.docker_image_cache**
19+
.docker_image_cache
20+
.log

‎.gitmodules‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,14 +61,6 @@
6161
ignore = untracked
6262
shallow = true
6363
branch = adore
64-
[submodule "libadore/make_gadgets"]
65-
path = libadore/make_gadgets
66-
url = git@github.com:DLR-TS/make_gadgets.git
67-
branch = master
68-
[submodule "adore_if_ros/make_gadgets"]
69-
path = adore_if_ros/make_gadgets
70-
url = git@github.com:DLR-TS/make_gadgets.git
71-
branch = master
7264
[submodule "adore_if_ros/cppcheck_docker"]
7365
path = adore_if_ros/cppcheck_docker
7466
url = git@github.com:DLR-TS/cppcheck_docker.git
@@ -93,3 +85,10 @@
9385
path = libadore/lizard_docker
9486
url = git@github.com:DLR-TS/lizard_docker.git
9587
branch = master
88+
[submodule "adore_if_ros/adore_scheduling"]
89+
path = adore_if_ros/adore_scheduling
90+
url = git@github.com:DLR-TS/adore_scheduling.git
91+
shallow = true
92+
[submodule "libadore/make_gadgets"]
93+
path = libadore/make_gadgets
94+
url = git@github.com:DLR-TS/make_gadgets.git

‎Jenkinsfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ pipeline {
1717
sh '''#!/usr/bin/env bash
1818
set -e
1919
export DOCKER_CONFIG=$(realpath "${DOCKER_CONFIG}") make
20-
make
20+
make build
2121
'''
2222
}
2323
}

‎Makefile‎

Lines changed: 79 additions & 224 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,71 @@
1-
include adore_if_ros/make_gadgets/Makefile
2-
include adore_if_ros/make_gadgets/docker/Makefile
3-
41
SHELL:=/bin/bash
52

63
.DEFAULT_GOAL := all
74

85
ROOT_DIR:=$(shell dirname "$(realpath $(firstword $(MAKEFILE_LIST)))")
9-
MAKEFLAGS += --no-print-directory
10-
116

12-
.EXPORT_ALL_VARIABLES:
13-
CATKIN_WORKSPACE_DIRECTORY=catkin_workspace
7+
include adore_if_ros_msg/make_gadgets/make_gadgets.mk
8+
include adore_if_ros_msg/make_gadgets/docker/docker-tools.mk
9+
include adore_if_ros_msg/make_gadgets/docker/docker-image-cacher.mk
10+
include adore-cli.mk
11+
#include adore_if_ros/adore_if_ros.mk
12+
#include adore_if_ros_msg/adore_if_ros_msg.mk
13+
include v2x_if_ros_msg/v2x_if_ros_msg.mk
14+
#include plotlabserver/plotlabserver.mk
15+
#include catkin_base.mk
1416

15-
DOCKER_IMAGE_EXCLUSION_LIST="adore_if_ros:latest adore_if_ros_msg:latest plotlablib:latest plotlabserver:latest plotlabserver_build:latest v2x_if_ros_msg:latest libzmq:latest csaps-cpp:latest adore-cli:latest carlasim/carla:0.9.12"
16-
DOCKER_IMAGE_INCLUSION_LIST="edrevo/dockerfile-plus:latest"
1717

18+
.EXPORT_ALL_VARIABLES:
19+
DOCKER_IMAGE_EXCLUSION_LIST="adore_if_ros:latest adore_if_ros_msg:latest plotlablib:latest plotlabserver:latest plotlabserver_build:latest v2x_if_ros_msg:latest libzmq:latest csaps-cpp:latest adore-cli:latest carlasim/carla:0.9.13"
20+
DOCKER_IMAGE_INCLUSION_LIST="edrevo/dockerfile-plus:latest ubuntu:20.04 ubuntu:focal"
1821
DOCKER_IMAGE_CACHE_DIRECTORY="${ROOT_DIR}/.docker_image_cache"
1922
DOCKER_IMAGE_SEARCH_PATH=${ROOT_DIR}
2023

24+
include apt_cacher_ng_docker/apt_cacher_ng_docker.mk
2125

22-
DOCKER_BUILDKIT?=1
23-
COMPOSE_DOCKER_CLI_BUILD?=1
24-
DOCKER_CONFIG:=$(shell realpath "${ROOT_DIR}")/apt_cacher_ng_docker
25-
26+
.PHONY: all
27+
all: help
2628

27-
DOCKER_GID := $(shell getent group | grep docker | cut -d":" -f3)
28-
USER := $(shell whoami)
29-
UID := $(shell id -u)
30-
GID := $(shell id -g)
29+
.PHONY: build
30+
build: _build ## Build ADORe
3131

32-
TEST_SCENARIOS?=baseline_test.launch baseline_test.launch
32+
.PHONY: build_adore_if_ros_msg
33+
build_adore_if_ros_msg:
34+
cd adore_if_ros_msg && \
35+
make build
3336

37+
.PHONY: build_adore_v2x
38+
build_adore_if_v2x:
39+
cd adore_if_v2x && \
40+
APT_CACHER_NG_DOCKER_MAKEFILE_PATH= make build
41+
42+
43+
.PHONY: _build
44+
_build: \
45+
docker_load \
46+
docker_host_context_check \
47+
docker_storage_inventory_prebuild \
48+
start_apt_cacher_ng \
49+
build_adore_if_ros_msg \
50+
build_v2x_if_ros_msg \
51+
build_adore_if_ros \
52+
build_adore_if_v2x \
53+
build_adore_if_ros_msg \
54+
docker_storage_inventory_postbuild \
55+
clean_up
56+
57+
.PHONY: clean
58+
clean: clean_adore-cli
59+
cd adore_if_ros && make clean
60+
cd libadore && make clean
61+
cd sumo_if_ros && make clean
62+
cd adore_if_ros_msg && make clean
63+
cd v2x_if_ros_msg && make clean
64+
cd adore_if_v2x && make clean
65+
cd plotlabserver && make clean
3466

35-
.PHONY: all
36-
all: \
37-
docker_group_check \
38-
root_check \
39-
clean \
40-
docker_conditional_load \
41-
docker_storage_inventory_prebuild \
42-
start_apt_cacher_ng \
43-
build_adore_if_ros_msg\
44-
build_adore_v2x_sim \
45-
build_adore_if_v2x \
46-
build_sumo_if_ros \
47-
build_plotlabserver \
48-
build_libadore \
49-
build_adore_if_ros \
50-
get_apt_cacher_ng_cache_statistics \
51-
docker_storage_inventory_postbuild \
52-
stop_apt_cacher_ng \
53-
docker_save_images \
67+
.PHONY: clean_up
68+
clean_up: stop_apt_cacher_ng docker_save_images docker_delete_all_none_tags
5469

5570
.PHONY: docker_storage_inventory_prebuild
5671
docker_storage_inventory_prebuild:
@@ -61,221 +76,61 @@ docker_storage_inventory_prebuild:
6176
docker_storage_inventory_postbuild:
6277
bash tools/docker_storage_inventory.sh --log-directory .log
6378

64-
.PHONY: build
65-
build: all
66-
67-
.PHONY: clean
68-
clean: delete_all_none_tags
69-
cd plotlabserver && make clean
70-
cd sumo_if_ros && make clean
71-
cd adore_if_ros_msg && make clean
72-
cd libadore && make clean
73-
cd adore_if_ros && make clean
74-
cd adore_if_v2x && make clean
75-
76-
.PHONY: start_apt_cacher_ng
77-
start_apt_cacher_ng: ## Start apt cacher ng service
78-
cd apt_cacher_ng_docker && \
79-
make up
80-
81-
.PHONY: stop_apt_cacher_ng
82-
stop_apt_cacher_ng: ## Stop apt cacher ng service
83-
cd apt_cacher_ng_docker && make down
84-
85-
.PHONY: get_apt_cacher_ng_cache_statistics
86-
get_apt_cacher_ng_cache_statistics: ## returns the cache statistics for apt cahcer ng
87-
@cd apt_cacher_ng_docker && \
88-
make get_cache_statistics
89-
90-
.PHONY: clean_apt_cacher_ng_cache
91-
clean_apt_cacher_ng_cache: ## Clean/delete apt cache
92-
@cd apt_cacher_ng_docker && \
93-
make clean
94-
95-
.PHONY: submodules_update
96-
submodules_update: # Updates git submodules
97-
git submodule update --init --recursive
98-
99-
.PHONY: build_adore_if_ros
100-
build_adore_if_ros: ## build adore_if_ros
101-
cd adore_if_ros && \
102-
make
103-
.PHONY: build_adore_if_ros_msg
104-
build_adore_if_ros_msg:
105-
cd adore_if_ros_msg && \
106-
make
107-
108-
.PHONY: build_plotlabserver
109-
build_plotlabserver: ## Build plotlabserver
110-
cd plotlabserver && \
111-
make build_fast
112-
113-
.PHONY: build_adore_if_v2x
114-
build_adore_if_v2x: ## Build adore_if_v2x
115-
cd adore_if_v2x && \
116-
make
117-
118-
.PHONY: build_adore_v2x_sim
119-
build_adore_v2x_sim: ## Build adore_v2x_sim
120-
cd adore_v2x_sim && \
121-
make
122-
123-
.PHONY: build_libadore
124-
build_libadore: start_apt_cacher_ng ## Build libadore
125-
cd libadore && \
126-
make
79+
.PHONY: docker_save_images
80+
docker_save_images:
81+
@nohup sh -c 'make docker_save' > /dev/null 2>&1 &
12782

128-
.PHONY: build_sumo_if_ros
129-
build_sumo_if_ros: ## Build sumo_if_ros
130-
cd sumo_if_ros && \
131-
make
83+
.PHONY: docker_load_images
84+
docker_load_images:
85+
@nohup make docker_load > /dev/null 2>&1 &
13286

13387
.PHONY: test
134-
test:
88+
test: ## Run ADORe unit tests
13589
mkdir -p .log && \
13690
cd libadore && \
13791
make test | tee "${ROOT_DIR}/.log/libadore_unit_test.log"; exit $$PIPESTATUS
13892

139-
.PHONY: lint_sumo_if_ros
140-
lint_sumo_if_ros:
141-
cd sumo_if_ros && make lint
142-
14393
.PHONY: lint
144-
lint: start_apt_cacher_ng## Run linting for all modules
94+
lint: ## Run linting for all modules
14595
mkdir -p .log
14696
find . -name "**lint_report.log" -exec rm -rf {} \;
14797
EXIT_STATUS=0; \
148-
(cd adore_if_ros && make lint) || EXIT_STATUS=$$? && \
14998
(cd sumo_if_ros && make lint) || EXIT_STATUS=$$? && \
15099
(cd libadore && make lint) || EXIT_STATUS=$$? && \
151-
find . -name "**lint_report.log" -print0 | xargs -0 -I {} mv {} .log/ && \
100+
(cd adore_if_ros && make lint) || EXIT_STATUS=$$? && \
101+
find sumo_if_ros -type f -name 'lint_report.log' -exec mv {} .log/sumo_if_ros_lint_report.log \; && \
102+
find libadore -type f -name 'lint_report.log' -exec mv {} .log/libadore_lint_report.log \; && \
103+
find adore_if_ros -type f -name 'lint_report.log' -exec mv {} .log/adore_if_ros_lint_report.log \; && \
152104
exit $$EXIT_STATUS
153105

154106
.PHONY: lizard
155-
lizard: start_apt_cacher_ng## Run lizard static analysis tool for all modules
107+
lizard: ## Run lizard static analysis tool for all modules
156108
mkdir -p .log
157109
find . -name "**lizard_report.**" -exec rm -rf {} \;
158-
EXIT_STATUS=0; \
110+
unset -f DOCKER_CONFIG && \
111+
EXIT_STATUS=0; \
112+
(cd sumo_if_ros && make lizard) || EXIT_STATUS=$$? && \
159113
(cd libadore && make lizard) || EXIT_STATUS=$$? \ && \
160114
(cd adore_if_ros && make lizard) || EXIT_STATUS=$$? && \
161-
(cd sumo_if_ros && make lizard) || EXIT_STATUS=$$? && \
162-
find . -name "**lizard_report.**" -print0 | xargs -0 -I {} mv {} .log/ && \
115+
find sumo_if_ros -type f -name 'lizard_report.log' -exec mv {} .log/sumo_if_ros_lizard_report.log \; && \
116+
find sumo_if_ros -type f -name 'lizard_report.xml' -exec mv {} .log/sumo_if_ros_lizard_report.xml \; && \
117+
find libadore -type f -name 'lizard_report.log' -exec mv {} .log/libadore_lizard_report.log \; && \
118+
find libadore -type f -name 'lizard_report.xml' -exec mv {} .log/libadore_lizard_report.xml \; && \
119+
find adore_if_ros -type f -name 'lizard_report.log' -exec mv {} .log/adore_if_ros_lizard_report.log \; && \
120+
find adore_if_ros -type f -name 'lizard_report.xml' -exec mv {} .log/adore_if_ros_if_ros_lizard_report.xml \; && \
163121
exit $$EXIT_STATUS
164122

165123
.PHONY: cppcheck
166-
cppcheck: start_apt_cacher_ng## Run cppcheck static checking tool for all modules.
124+
cppcheck: ## Run cppcheck static checking tool for all modules.
167125
mkdir -p .log
168126
find . -name "**cppcheck_report.log" -exec rm -rf {} \;
169127
EXIT_STATUS=0; \
128+
(cd sumo_if_ros && make cppcheck) || EXIT_STATUS=$$? && \
170129
(cd libadore && make cppcheck) || EXIT_STATUS=$$? && \
171130
(cd adore_if_ros && make cppcheck) || EXIT_STATUS=$$? && \
172-
(cd sumo_if_ros && make cppcheck) || EXIT_STATUS=$$? && \
173-
find . -name "**cppcheck_report.log" -print0 | xargs -0 -I {} mv {} .log/ && \
131+
find sumo_if_ros -type f -name 'cppcheck_report.log' -exec mv {} .log/sumo_if_ros_cppcheck_report.log \; && \
132+
find libadore -type f -name 'cppcheck_report.log' -exec mv {} .log/libadore_cppcheck_report.log \; && \
133+
find adore_if_ros -type f -name 'cppcheck_report.log' -exec mv {} .log/adore_if_ros_cppcheck_report.log \; && \
174134
exit $$EXIT_STATUS
175135

176-
.PHONY: clean_catkin_workspace
177-
clean_catkin_workspace:
178-
rm -rf "${CATKIN_WORKSPACE_DIRECTORY}"
179-
180-
.PHONY: build_catkin_base
181-
build_catkin_base: ## Build a docker image with base catkin tools installed with tag catkin_base:latest
182-
cd docker && \
183-
docker build \
184-
--network host \
185-
--build-arg UID=${UID} \
186-
--build-arg GID=${GID} \
187-
--file Dockerfile.catkin_base \
188-
--tag catkin_base .
189-
190-
.PHONY: create_catkin_workspace_docker
191-
create_catkin_workspace_docker: build_catkin_base
192-
docker run -it \
193-
--user "${UID}:${GID}" \
194-
--mount type=bind,source="${ROOT_DIR}",target="${ROOT_DIR}" \
195-
catkin_base \
196-
/bin/bash -c 'cd "${ROOT_DIR}" && HOME="${ROOT_DIR}" CATKIN_WORKSPACE_DIRECTORY="${CATKIN_WORKSPACE_DIRECTORY}" bash tools/create_catkin_workspace.sh 2>&1 | tee -a .log/create_catkin_workspace.log'
197-
198-
.PHONY: create_catkin_workspace
199-
create_catkin_workspace: clean_catkin_workspace## Creates a catkin workspace @ adore/catkin_workspace. Can be called within the adore-cli or on the host.
200-
@if [ -f "/.dockerenv" ]; then\
201-
bash tools/create_catkin_workspace.sh;\
202-
exit 0;\
203-
else\
204-
make create_catkin_workspace_docker;\
205-
exit 0;\
206-
fi;
207-
208-
.PHONY: build_adore-cli_fast
209-
build_adore-cli_fast: # build adore-cli if it does not already exist in the docker repository. If it does exist this is a noop.
210-
@[ -n "$$(docker images -q adore-cli:latest)" ] || \
211-
make build_adore-cli
212-
213-
.PHONY: build_adore-cli
214-
build_adore-cli: start_apt_cacher_ng build_catkin_base build_plotlabserver ## Builds the ADORe CLI docker context/image
215-
docker compose build adore-cli \
216-
--build-arg UID=${UID} \
217-
--build-arg GID=${GID} \
218-
--build-arg DOCKER_GID=${DOCKER_GID}
219-
docker compose build adore-cli-x11-display \
220-
--build-arg UID=${UID} \
221-
--build-arg GID=${GID} \
222-
--build-arg DOCKER_GID=${DOCKER_GID}
223-
make stop_apt_cacher_ng
224-
225-
.PHONY: run_ci_scenarios
226-
run_ci_scenarios:
227-
bash tools/run_ci_scenarios.txt
228-
229-
230-
.PHONY: adore-cli_setup
231-
adore-cli_setup: build_adore-cli_fast
232-
@echo "Running adore-cli setup..."
233-
@mkdir -p .log/.ros/bag_files
234-
@mkdir -p plotlabserver/.log
235-
@cd .log && ln -sf ../plotlabserver/.log plotlabserver
236-
@touch .zsh_history
237-
@touch .zsh_history.new
238-
cd plotlabserver && \
239-
make down
240-
241-
.PHONY: adore-cli_teardown
242-
adore-cli_teardown:
243-
@echo "Running adore-cli teardown..."
244-
@docker compose down && xhost - 1> /dev/null
245-
@docker compose rm -f
246-
247-
.PHONY: adore-cli_start
248-
adore-cli_start:
249-
@xhost + && \
250-
docker compose up adore-cli-x11-display --force-recreate -V -d; \
251-
xhost -
252-
253-
.PHONY: adore-cli_start_headless
254-
adore-cli_start_headless:
255-
DISPLAY_MODE=headless make adore-cli_start
256-
257-
.PHONY: adore-cli_attach
258-
adore-cli_attach:
259-
docker exec -it --user adore-cli adore-cli /bin/zsh -c "bash tools/adore-cli.sh" || true
260-
261-
.PHONY: adore-cli_scenarios_run
262-
adore-cli_scenarios_run:
263-
docker exec -it --user adore-cli adore-cli /bin/zsh -c "bash tools/run_test_scenarios.sh" || true
264-
265-
.PHONY: adore-cli
266-
adore-cli: adore-cli_setup adore-cli_start adore-cli_attach adore-cli_teardown ## Start an adore-cli context
267-
268-
.PHONY: run_test_scenarios
269-
run_test_scenarios: adore-cli_setup adore-cli_start_headless adore-cli_scenarios_run adore-cli_teardown
270-
271-
.PHONY: run_scenarios
272-
run_scenarios: adore-cli_setup adore-cli_start adore-cli_scenarios_run adore-cli_teardown
273-
274-
.PHONY: docker_save_images
275-
docker_save_images:
276-
@nohup make docker_save > /dev/null 2>&1 &
277-
278-
.PHONY: clean_all_cache
279-
clean_all_cache:
280-
echo todo
281136

0 commit comments

Comments
 (0)