Skip to content

Commit cfba462

Browse files
author
Robin VAN DE MERGHEL
committed
feat: Add legacy adaptor in CI
1 parent c53246f commit cfba462

File tree

4 files changed

+69
-9
lines changed

4 files changed

+69
-9
lines changed

integration_tests.py

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,15 @@
7171
}
7272
LOG_PATTERN = re.compile(r"^[\d\-]{10} [\d:]{8} UTC [^\s]+ ([A-Z]+):")
7373

74+
# Note for MacOSX
75+
# You can run
76+
# ```bash
77+
# OSX=true ./integration_tests.py ...
78+
# ```
79+
# To support docker-compose
80+
IS_OSX = os.getenv("OSX")
81+
DOCKER_COMPOSE_CMD = "docker-compose" if IS_OSX else "docker"
82+
7483

7584
class NaturalOrderGroup(typer.core.TyperGroup):
7685
"""Group for showing subcommands in the correct order"""
@@ -154,9 +163,10 @@ def create(
154163
run_server_tests: bool = True,
155164
run_client_tests: bool = True,
156165
run_pilot_tests: bool = True,
166+
legacy_adapted_services: Optional[list[str]] = typer.Option(None, help="Services that have legacy adaptors."),
157167
):
158168
"""Start a local instance of the integration tests"""
159-
prepare_environment(flags, editable, extra_module, diracx_dist_dir, release_var)
169+
prepare_environment(flags, editable, extra_module, diracx_dist_dir, release_var, legacy_adapted_services)
160170
install_server()
161171
install_client()
162172
install_pilot()
@@ -194,7 +204,7 @@ def destroy():
194204
with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn:
195205
os.execvpe(
196206
"docker",
197-
["docker", "compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"],
207+
[DOCKER_COMPOSE_CMD, "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"],
198208
_make_env({}),
199209
)
200210

@@ -206,6 +216,7 @@ def prepare_environment(
206216
extra_module: Optional[list[str]] = None,
207217
diracx_dist_dir: Optional[str] = None,
208218
release_var: Optional[str] = None,
219+
legacy_adapted_services: Optional[list[str]] = typer.Option(None, help="Services that have legacy adaptors."),
209220
):
210221
"""Prepare the local environment for installing DIRAC."""
211222
if extra_module is None:
@@ -226,7 +237,13 @@ def prepare_environment(
226237
flags = {}
227238
else:
228239
flags = dict(f.split("=", 1) for f in flags)
229-
docker_compose_env = _make_env(flags)
240+
241+
legacy_adapted_services_str = ""
242+
if legacy_adapted_services:
243+
legacy_adapted_services_str = " ".join(legacy_adapted_services)
244+
245+
docker_compose_env = _make_env(flags, legacy_adapted_services_str)
246+
230247
server_flags = {}
231248
client_flags = {}
232249
pilot_flags = {}
@@ -253,7 +270,7 @@ def prepare_environment(
253270
typer.secho("Running docker compose to create containers", fg=c.GREEN)
254271
with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn:
255272
subprocess.run(
256-
["docker", "compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client", "dirac-pilot"]
273+
[DOCKER_COMPOSE_CMD, "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client", "dirac-pilot"]
257274
+ extra_services,
258275
check=True,
259276
env=docker_compose_env,
@@ -360,7 +377,7 @@ def prepare_environment(
360377
subStderr = open(docker_compose_fn_final / "stderr", "w")
361378

362379
subprocess.Popen(
363-
["docker", "compose", "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"],
380+
[DOCKER_COMPOSE_CMD, "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"],
364381
env=docker_compose_env,
365382
stdin=None,
366383
stdout=subStdout,
@@ -619,7 +636,7 @@ def _gen_docker_compose(modules, *, diracx_dist_dir=None):
619636
def _check_containers_running(*, is_up=True):
620637
with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn:
621638
running_containers = subprocess.run(
622-
["docker", "compose", "-f", docker_compose_fn, "ps", "-q", "-a"],
639+
[DOCKER_COMPOSE_CMD, "-f", docker_compose_fn, "ps", "-q", "-a"],
623640
stdout=subprocess.PIPE,
624641
env=_make_env({}),
625642
# docker compose ps has a non-zero exit code when no containers are running
@@ -689,7 +706,12 @@ def _find_dirac_release():
689706
return version_branch
690707

691708

692-
def _make_env(flags):
709+
def _make_env(flags, legacy_adapted_services=""):
710+
# Important note: when you add an env variable here to pass it to
711+
# a docker compose, make sure that in docker-compose.yml, you added an
712+
# `environment` field for your service.
713+
#
714+
# If you don't, the variable won't be passed to the container.
693715
env = os.environ.copy()
694716
env["DIRAC_UID"] = str(os.getuid())
695717
env["DIRAC_GID"] = str(os.getgid())
@@ -705,6 +727,7 @@ def _make_env(flags):
705727
if "CVMFS_DIR" not in env or not Path(env["CVMFS_DIR"]).is_dir():
706728
typer.secho(f"CVMFS_DIR environment value: {env.get('CVMFS_DIR', 'NOT SET')}", fg=c.YELLOW)
707729
env["CVMFS_DIR"] = "/tmp"
730+
env["LEGACY_ADAPTED_SERVICES"] = legacy_adapted_services
708731
return env
709732

710733

tests/CI/docker-compose.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ volumes:
1010

1111
services:
1212
mysql:
13+
platform: linux/amd64
1314
image: ${MYSQL_VER}
1415
container_name: mysql
1516
environment:
@@ -24,6 +25,7 @@ services:
2425
pull_policy: always
2526

2627
opensearch:
28+
platform: linux/amd64
2729
image: ${ES_VER}
2830
container_name: opensearch
2931
hostname: opensearch
@@ -39,6 +41,7 @@ services:
3941
pull_policy: always
4042

4143
iam-login-service:
44+
platform: linux/amd64
4245
image: ${IAM_VER}
4346
container_name: iam-login-service
4447
hostname: iam-login-service
@@ -58,6 +61,7 @@ services:
5861
pull_policy: always
5962

6063
iam-init-keystore:
64+
platform: linux/amd64
6165
image: alpine:latest
6266
container_name: iam-init-keystore
6367
volumes:
@@ -83,6 +87,7 @@ services:
8387

8488
# Mock of an S3 storage
8589
s3-direct:
90+
platform: linux/amd64
8691
# Fix the version until https://github.com/adobe/S3Mock/issues/2321
8792
# is resolved
8893
image: adobe/s3mock:3.12.0
@@ -97,6 +102,7 @@ services:
97102
pull_policy: always
98103

99104
diracx-wait-for-db:
105+
platform: linux/amd64
100106
image: ${MYSQL_VER}
101107
container_name: diracx-wait-for-db
102108
depends_on:
@@ -105,6 +111,7 @@ services:
105111
command: /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/check_db_initialized.sh
106112

107113
dirac-init-certificates:
114+
platform: linux/amd64
108115
image: ghcr.io/diracgrid/management/certificates-generation:latest
109116
container_name: dirac-init-certificates
110117
volumes:
@@ -114,6 +121,7 @@ services:
114121
pull_policy: always
115122

116123
dirac-server:
124+
platform: linux/amd64
117125
image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac
118126
container_name: server
119127
hostname: server
@@ -143,11 +151,13 @@ services:
143151
environment:
144152
- DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo
145153
- DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE=file:///keystore/jwks.json
154+
- LEGACY_ADAPTED_SERVICES
146155
command: ["sleep", "infinity"] # This is necessary because of the issue described in https://github.com/moby/moby/issues/42275. What is added here is a hack/workaround.
147156
pull_policy: always
148157

149158

150159
dirac-client:
160+
platform: linux/amd64
151161
image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac
152162
container_name: client
153163
hostname: client
@@ -162,6 +172,7 @@ services:
162172
pull_policy: always
163173

164174
dirac-pilot:
175+
platform: linux/amd64
165176
image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac
166177
container_name: pilot
167178
hostname: pilot
@@ -187,6 +198,7 @@ services:
187198

188199

189200
diracx-chmod:
201+
platform: linux/amd64
190202
image: ghcr.io/diracgrid/diracx/secret-generation:latest
191203
container_name: diracx-chmod
192204
volumes:
@@ -201,6 +213,7 @@ services:
201213

202214

203215
diracx-init-keystore:
216+
platform: linux/amd64
204217
image: ghcr.io/diracgrid/diracx/services:dev
205218
container_name: diracx-init-keystore
206219
depends_on:
@@ -215,6 +228,7 @@ services:
215228
pull_policy: always
216229

217230
diracx-init-cs:
231+
platform: linux/amd64
218232
image: ghcr.io/diracgrid/diracx/client:dev
219233
container_name: diracx-init-cs
220234
depends_on:
@@ -233,6 +247,7 @@ services:
233247
pull_policy: always
234248

235249
diracx-init-db:
250+
platform: linux/amd64
236251
image: ghcr.io/diracgrid/diracx/services:dev
237252
container_name: diracx-init-db
238253
depends_on:
@@ -245,6 +260,7 @@ services:
245260
pull_policy: always
246261

247262
diracx:
263+
platform: linux/amd64
248264
image: ghcr.io/diracgrid/diracx/services:dev
249265
container_name: diracx
250266
environment:

tests/Jenkins/dirac-cfg-setup-diracx.py

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ def parse_args():
1212
parser.add_argument("--disable-vo", nargs="+", help="Disable a VO", default=[])
1313
parser.add_argument("--url", help="URL of the DiracX services")
1414
parser.add_argument("--credentials-dir", help="Directory where hostcert.pem/hostkey.pem can be found")
15+
parser.add_argument("--legacy-adapted-services", nargs="+", help="Services that have legacy adaptors.", default=[])
1516
args = parser.parse_args()
1617

1718
DIRAC.initialize(
@@ -21,10 +22,10 @@ def parse_args():
2122
)
2223
)
2324

24-
main(args.url, args.disable_vo)
25+
main(args.url, args.disable_vo, args.legacy_adapted_services)
2526

2627

27-
def main(url: str, disabled_vos: list[str]):
28+
def main(url: str, disabled_vos: list[str], legacy_adapted_services: list[str]):
2829
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
2930

3031
csAPI = CSAPI()
@@ -72,6 +73,21 @@ def main(url: str, disabled_vos: list[str]):
7273
csSyncCFG = CFG().loadFromDict(csSync)
7374
returnValueOrRaise(csAPI.mergeCFGUnderSection("/DiracX/", csSyncCFG))
7475

76+
# Add service with legacy adaptors.
77+
legacy = {"LegacyClientEnabled": {}}
78+
for service in legacy_adapted_services:
79+
print(f"Adding legacy adaptor for service {service}")
80+
# Service name such as: system/name
81+
system, name = service.split("/")
82+
83+
if not system in legacy["LegacyClientEnabled"]:
84+
legacy["LegacyClientEnabled"][system] = {}
85+
86+
legacy["LegacyClientEnabled"][system][name] = "yes"
87+
88+
legacyCFG = CFG().loadFromDict(legacy)
89+
returnValueOrRaise(csAPI.mergeCFGUnderSection("/DiracX/", legacyCFG))
90+
7591
returnValueOrRaise(csAPI.commit())
7692

7793

tests/Jenkins/dirac_ci.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,11 @@ installSite() {
167167
else
168168
diracxSetupArgs+=("--disable-vo" "vo")
169169
fi
170+
if [[ -n "${LEGACY_ADAPTED_SERVICES:-}" ]]; then
171+
# Add legacy adaptor
172+
# Format : --legacy-adapted-services serviceA serviceB ...
173+
diracxSetupArgs+=("--legacy-adapted-services" ${LEGACY_ADAPTED_SERVICES})
174+
fi
170175
if ! python "${TESTCODE}/DIRAC/tests/Jenkins/dirac-cfg-setup-diracx.py" "${diracxSetupArgs[@]}"; then
171176
echo "ERROR: dirac-cfg-setup-diracx.py failed" >&2
172177
exit 1

0 commit comments

Comments
 (0)