Native multiarch support with backwards compatibility to osrf images

ros_buildfarm/argument.py

@@ -453,6 +453,12 @@ def add_argument_skip_tests(parser):
         help='Skip execution of tests as part of the package build process.')
 
 
+def add_argument_docker_image_prefix(parser):
+    parser.add_argument(
+        '--docker-image-prefix',
+        help='Prefix for the docker image (e.g. "osrf/" or "arm64v8/")')
+
+
 def check_len_action(minargs, maxargs):
     class CheckLength(argparse.Action):
 

ros_buildfarm/release_job.py

@@ -403,7 +403,8 @@ def configure_release_job(
         is_disabled=False, other_build_files_same_platform=None,
         groovy_script=None,
         filter_arches=None,
-        dry_run=False):
+        dry_run=False,
+        docker_image_prefix=None):
     """
     Configure a Jenkins release job.
 
@@ -565,7 +566,8 @@ def configure_release_job(
             config, build_file, os_name, os_code_name, arch,
             pkg_name, repo_name, repo.release_repository,
             cached_pkgs=cached_pkgs, upstream_job_names=upstream_job_names,
-            is_disabled=is_disabled)
+            is_disabled=is_disabled,
+            docker_image_prefix=docker_image_prefix)
         # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
         if isinstance(jenkins, object) and jenkins is not False:
             configure_job(jenkins, job_name, job_config, dry_run=dry_run)
@@ -687,7 +689,8 @@ def _get_binarydeb_job_config(
         config, build_file, os_name, os_code_name, arch,
         pkg_name, repo_name, release_repository,
         cached_pkgs=None, upstream_job_names=None,
-        is_disabled=False):
+        is_disabled=False,
+        docker_image_prefix=None):
     package_format = package_format_mapping[os_name]
     template_name = 'release/%s/binarypkg_job.xml.em' % package_format
 
@@ -765,6 +768,8 @@ def _get_binarydeb_job_config(
         'credential_id': build_file.upload_credential_id,
 
         'shared_ccache': build_file.shared_ccache,
+
+        'docker_image_prefix': docker_image_prefix,
     }
     job_config = expand_template(template_name, job_data)
     return job_config

ros_buildfarm/scripts/release/create_binarydeb_task_generator.py

@@ -24,6 +24,7 @@
 from ros_buildfarm.argument import \
     add_argument_distribution_repository_key_files
 from ros_buildfarm.argument import add_argument_distribution_repository_urls
+from ros_buildfarm.argument import add_argument_docker_image_prefix
 from ros_buildfarm.argument import add_argument_dockerfile_dir
 from ros_buildfarm.argument import add_argument_env_vars
 from ros_buildfarm.argument import add_argument_os_code_name
@@ -56,6 +57,7 @@ def main(argv=sys.argv[1:]):
     add_argument_dockerfile_dir(parser)
     add_argument_env_vars(parser)
     add_argument_skip_tests(parser)
+    add_argument_docker_image_prefix(parser)
     args = parser.parse_args(argv)
 
     debian_package_name = get_os_package_name(
@@ -112,6 +114,7 @@ def main(argv=sys.argv[1:]):
         'rosdistro_name': args.rosdistro_name,
         'package_name': args.package_name,
         'binarypkg_dir': args.binarypkg_dir,
+        'docker_image_prefix': args.docker_image_prefix,
     }
     create_dockerfile(
         'release/deb/binarypkg_task.Dockerfile.em', data, args.dockerfile_dir)

ros_buildfarm/scripts/release/generate_release_script.py

@@ -19,6 +19,7 @@
 from ros_buildfarm.argument import add_argument_arch
 from ros_buildfarm.argument import add_argument_build_name
 from ros_buildfarm.argument import add_argument_config_url
+from ros_buildfarm.argument import add_argument_docker_image_prefix
 from ros_buildfarm.argument import add_argument_os_code_name
 from ros_buildfarm.argument import add_argument_os_name
 from ros_buildfarm.argument import add_argument_package_name
@@ -41,6 +42,7 @@ def main(argv=sys.argv[1:]):
     add_argument_os_name(parser)
     add_argument_os_code_name(parser)
     add_argument_arch(parser)
+    add_argument_docker_image_prefix(parser)
     parser.add_argument(
         '--skip-binary',
         action='store_true',
@@ -97,7 +99,8 @@ def beforeInclude(self, *args, **kwargs):
         args.config_url, args.rosdistro_name, args.release_build_name,
         args.package_name, args.os_name, args.os_code_name,
         jenkins=False, views=[], generate_import_package_job=False,
-        generate_sync_packages_jobs=False, filter_arches=args.arch)
+        generate_sync_packages_jobs=False, filter_arches=args.arch,
+        docker_image_prefix=args.docker_image_prefix)
 
     templates.template_hooks = None
 

ros_buildfarm/scripts/release/run_binarydeb_job.py

@@ -22,6 +22,7 @@
 from ros_buildfarm.argument import \
     add_argument_distribution_repository_key_files
 from ros_buildfarm.argument import add_argument_distribution_repository_urls
+from ros_buildfarm.argument import add_argument_docker_image_prefix
 from ros_buildfarm.argument import add_argument_dockerfile_dir
 from ros_buildfarm.argument import add_argument_env_vars
 from ros_buildfarm.argument import add_argument_os_code_name
@@ -55,6 +56,7 @@ def main(argv=sys.argv[1:]):
     add_argument_append_timestamp(parser)
     add_argument_env_vars(parser)
     add_argument_skip_tests(parser)
+    add_argument_docker_image_prefix(parser)
     args = parser.parse_args(argv)
 
     data = copy.deepcopy(args.__dict__)

ros_buildfarm/templates/ci/ci_job.xml.em

@@ -220,7 +220,7 @@ parameters = [
         'echo "# BEGIN SECTION: Build Dockerfile - generating CI tasks"',
         'cd $WORKSPACE/docker_generating_dockers',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_task_generation.%s .' % (rosdistro_name or 'global'),
+        'docker build --force-rm --platform=linux/%s -t $DOCKER_IMAGE_PREFIX.ci_task_generation.%s .' % (arch, rosdistro_name or 'global'),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - generating CI tasks"',
@@ -260,7 +260,7 @@ parameters = [
         '# build and run create_workspace Dockerfile',
         'cd $WORKSPACE/docker_create_workspace',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s .' % (rosdistro_name or 'global'),
+        'docker build --force-rm --platform=linux/%s -t $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s .' % (arch, rosdistro_name or 'global'),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - create workspace"',
@@ -314,7 +314,7 @@ parameters = [
         '# build and run build and install Dockerfile',
         'cd $WORKSPACE/docker_build_and_install',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s .' % (rosdistro_name or 'global'),
+        'docker build --force-rm --platform=linux/%s -t $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s .' % (arch, rosdistro_name or 'global'),
         'echo "# END SECTION"',
         '',
     ] + ([
@@ -398,7 +398,7 @@ parameters = [
         '# build and run build and test Dockerfile',
         'cd $WORKSPACE/docker_build_and_test',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s .' % (rosdistro_name or 'global'),
+        'docker build --force-rm --platform=linux/%s -t $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s .' % (arch, rosdistro_name or 'global'),
         'echo "# END SECTION"',
         '',
     ] + ([

ros_buildfarm/templates/devel/devel_job.xml.em

@@ -159,7 +159,7 @@ if pull_request:
         'echo "# BEGIN SECTION: Build Dockerfile - generating devel tasks"',
         'cd $WORKSPACE/docker_generating_dockers',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t devel_task_generation.%s_%s .' % (rosdistro_name, source_repo_spec.name.lower()),
+        'docker build --force-rm --platform=linux/%s -t devel_task_generation.%s_%s .' % (arch, rosdistro_name, source_repo_spec.name.lower()),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - generating devel tasks"',
@@ -199,7 +199,7 @@ if pull_request:
         '# build and run build_and_install Dockerfile',
         'cd $WORKSPACE/docker_build_and_install',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t devel_build_and_install.%s_%s .' % (rosdistro_name, source_repo_spec.name.lower()),
+        'docker build --force-rm --platform=linux/%s -t devel_build_and_install.%s_%s .' % (arch, rosdistro_name, source_repo_spec.name.lower()),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - build and install"',
@@ -234,7 +234,7 @@ if pull_request:
         '# build and run build_and_test Dockerfile',
         'cd $WORKSPACE/docker_build_and_test',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t devel_build_and_test.%s_%s .' % (rosdistro_name, source_repo_spec.name.lower()),
+        'docker build --force-rm --platform=linux/%s -t devel_build_and_test.%s_%s .' % (arch, rosdistro_name, source_repo_spec.name.lower()),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - build and test"',

ros_buildfarm/templates/doc/doc_independent_job.xml.em

@@ -117,7 +117,7 @@ else:
         'echo "# BEGIN SECTION: Build Dockerfile - doc independent"',
         'cd $WORKSPACE/docker_doc_independent',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t doc_independent .',
+        'docker build --force-rm --platform=linux/amd64 -t doc_independent .',
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - doc independent"',

ros_buildfarm/templates/doc/doc_job.xml.em

@@ -178,7 +178,7 @@ else:
         'echo "# BEGIN SECTION: Build Dockerfile - generating doc task"',
         'cd $WORKSPACE/docker_generating_docker',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t doc_task_generation.%s_%s .' % (rosdistro_name, doc_repo_spec.name.lower()),
+        'docker build --force-rm --platform=linux/%s -t doc_task_generation.%s_%s .' % (arch, rosdistro_name, doc_repo_spec.name.lower()),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - generating doc task"',
@@ -218,7 +218,7 @@ else:
         '# build and run build_and_install Dockerfile',
         'cd $WORKSPACE/docker_doc',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t doc.%s_%s .' % (rosdistro_name, doc_repo_spec.name.lower()),
+        'docker build --force-rm --platform=linux/%s -t doc.%s_%s .' % (arch, rosdistro_name, doc_repo_spec.name.lower()),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - doc"',

ros_buildfarm/templates/doc/doc_metadata_job.xml.em

@@ -91,7 +91,7 @@
         'echo "# BEGIN SECTION: Build Dockerfile - doc metadata"',
         'cd $WORKSPACE/docker_doc_metadata',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t doc_metadata.%s .' % rosdistro_name,
+        'docker build --force-rm --platform=linux/amd64 -t doc_metadata.%s .' % rosdistro_name,
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - doc metadata"',

ros_buildfarm/templates/doc/rosdoc2_job.xml.em

@@ -136,7 +136,7 @@ else:
         'echo "# BEGIN SECTION: Build Dockerfile - generating doc task"',
         'cd $WORKSPACE/docker_generating_docker',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t rosdoc2_task_generation.%s_%s .' % (rosdistro_name, doc_repo_spec.name.lower()),
+        'docker build --force-rm --platform=linux/%s -t rosdoc2_task_generation.%s_%s .' % (arch, rosdistro_name, doc_repo_spec.name.lower()),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - generating doc task"',
@@ -173,7 +173,7 @@ else:
         '# build and run build_and_install Dockerfile',
         'cd $WORKSPACE/docker_doc',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t rosdoc2.%s_%s .' % (rosdistro_name, doc_repo_spec.name.lower()),
+        'docker build --force-rm --platform=linux/%s -t rosdoc2.%s_%s .' % (arch, rosdistro_name, doc_repo_spec.name.lower()),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - doc"',

ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em

@@ -5,6 +5,7 @@
     os_name=os_name,
     os_code_name=os_code_name,
     arch=arch,
+    docker_image_prefix=vars().get('docker_image_prefix'),
 ))@
 
 VOLUME ["/var/cache/apt/archives"]
@@ -108,6 +109,7 @@ cmds.append(
     ' --binarypkg-dir ' + binarypkg_dir +
     ' --env-vars ' + ' '.join(build_environment_variables) +
     ' --dockerfile-dir ' + dockerfile_dir +
-    (' --skip-tests' if skip_tests else ''))
+    (' --skip-tests' if skip_tests else '') +
+    (' --docker-image-prefix ' + docker_image_prefix if vars().get('docker_image_prefix') else ''))
 }@
 CMD ["@(' && '.join(cmds))"]

ros_buildfarm/templates/release/deb/binarypkg_job.xml.em

@@ -75,6 +75,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
     os_name=os_name,
     os_code_name=os_code_name,
     arch=arch,
+    docker_image_prefix=vars().get('docker_image_prefix'),
 ))@
 @(SNIPPET(
     'builder_shell_clone-ros-buildfarm',
@@ -113,13 +114,14 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         ' --dockerfile-dir $WORKSPACE/docker_generating_docker' +
         ' --env-vars ' + ' '.join(build_environment_variables) +
         (' --append-timestamp' if append_timestamp else '') +
-        (' --skip-tests' if skip_tests else ''),
+        (' --skip-tests' if skip_tests else '') +
+        (' --docker-image-prefix ' + docker_image_prefix if vars().get('docker_image_prefix') else ''),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Build Dockerfile - binarydeb task"',
         'cd $WORKSPACE/docker_generating_docker',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --network=host --force-rm -t binarydeb_task_generation.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
+        'docker build --network=host --force-rm --platform=linux/%s -t binarydeb_task_generation.%s_%s_%s_%s_%s .' % (arch, rosdistro_name, os_name, os_code_name, arch, pkg_name),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - binarydeb task"',
@@ -160,7 +162,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         '# build and run build_binarydeb Dockerfile',
         'cd $WORKSPACE/docker_build_binarydeb',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --network=host --force-rm -t binarydeb_build.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
+        'docker build --network=host --force-rm --platform=linux/%s -t binarydeb_build.%s_%s_%s_%s_%s .' % (arch, rosdistro_name, os_name, os_code_name, arch, pkg_name),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - build binarydeb"',

ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em

@@ -6,6 +6,7 @@
     os_name=os_name,
     os_code_name=os_code_name,
     arch=arch,
+    docker_image_prefix=docker_image_prefix,
 ))@
 
 VOLUME ["/var/cache/apt/archives"]

ros_buildfarm/templates/release/deb/sourcepkg_job.xml.em

@@ -104,7 +104,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'echo "# BEGIN SECTION: Build Dockerfile - generate sourcedeb"',
         'cd $WORKSPACE/docker_sourcedeb',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --network=host --force-rm -t sourcedeb.%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, pkg_name),
+        'docker build --network=host --force-rm --platform=linux/%s -t sourcedeb.%s_%s_%s_%s .' % (arch, rosdistro_name, os_name, os_code_name, pkg_name),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - generate sourcedeb"',

ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em

@@ -121,7 +121,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'echo "# BEGIN SECTION: Build Dockerfile - build binaryrpm"',
         'cd $WORKSPACE/docker_binaryrpm',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t binaryrpm.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
+        'docker build --force-rm --platform=linux/%s -t binaryrpm.%s_%s_%s_%s_%s .' % (arch, rosdistro_name, os_name, os_code_name, arch, pkg_name),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - build binaryrpm"',

ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em

@@ -104,7 +104,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'echo "# BEGIN SECTION: Build Dockerfile - generate sourcerpm"',
         'cd $WORKSPACE/docker_sourcerpm',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t sourcerpm.%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, pkg_name),
+        'docker build --force-rm --platform=linux/%s -t sourcerpm.%s_%s_%s_%s .' % (arch, rosdistro_name, os_name, os_code_name, pkg_name),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - generate sourcerpm"',

ros_buildfarm/templates/snippet/builder_check-docker.xml.em

@@ -1,9 +1,18 @@
 @{
 # same logic as in from_base_image.Dockerfile.em
-if arch in ['i386', 'armhf', 'arm64']:
-    base_image = 'osrf/%s_%s:%s' % (os_name, arch, os_code_name)
-else:
-    base_image = '%s:%s' % (os_name, os_code_name)
+base_image = '%s:%s' % (
+    vars().get('docker_image_prefix') or (
+        'osrf/%s_%s' % (os_name, arch)
+        if arch in ('i386', 'armhf', 'arm64') and (
+            os_code_name in (
+                'artful', 'bionic', 'cosmic', 'disco', 'focal',
+                'jammy', 'noble', 'wily', 'xenial', 'yakkety',
+                'zesty')
+        )
+        else os_name
+    ),
+    os_code_name,
+)
 }@
 @(SNIPPET(
     'builder_shell',

ros_buildfarm/templates/snippet/from_base_image.Dockerfile.em

@@ -1,13 +1,22 @@
-@[if 'base_image' in locals()]@
-FROM @base_image
-@[else]@
-@# same logic as in builder_check-docker.xml.em
-@[  if arch in ['i386', 'armhf', 'arm64']]@
-FROM osrf/@(os_name)_@arch:@os_code_name
-@[  else]@
-FROM @os_name:@os_code_name
-@[  end if]@
+@[if 'base_image' not in locals()]@
+@{
+base_image = '%s:%s' % (
+  vars().get('docker_image_prefix') or (
+      'osrf/%s_%s' % (os_name, arch)
+      if arch in ('i386', 'armhf', 'arm64') and (
+            os_code_name in (
+                'artful', 'bionic', 'cosmic', 'disco', 'focal',
+                'jammy', 'noble', 'wily', 'xenial', 'yakkety',
+                'zesty')
+      )
+      else os_name
+  ),
+  os_code_name,
+)
+}
 @[end if]@
+@# same logic as in builder_check-docker.xml.em
+FROM @base_image
 @[if vars().get('maintainer_name')]@
 LABEL maintainer "@maintainer_name"
 @[end if]@