From ff6c95f6cf4da944366f55dd6edda6ec6bee48ec Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 2 Sep 2024 11:27:24 +0100 Subject: [PATCH 1/8] Add sync_group variable --- .github/workflows/package-sync-nightly.yml | 43 ++++++- .github/workflows/package-sync.yml | 11 +- ansible/filter_plugins/filters.py | 25 ++-- .../inventory/group_vars/all/package-repos | 113 +++++++++++++++++- ansible/sync-matrix-build.yml | 0 ansible/validate-deb-repos.yml | 18 ++- 6 files changed, 192 insertions(+), 18 deletions(-) create mode 100644 ansible/sync-matrix-build.yml diff --git a/.github/workflows/package-sync-nightly.yml b/.github/workflows/package-sync-nightly.yml index 37279dad..3b68529a 100644 --- a/.github/workflows/package-sync-nightly.yml +++ b/.github/workflows/package-sync-nightly.yml @@ -6,17 +6,54 @@ on: - cron: '17 23 * * *' env: ANSIBLE_FORCE_COLOR: True + ANSIBLE_VAULT_PASSWORD_FILE: ${{ github.workspace }}/vault-pass jobs: + sync-matrix-build: + name: Build package matrix of package repo sync jobs + runs-on: arc-release-train-runner + outputs: + matrix: ${{ steps.matrix-build.outputs.matrix }} + steps: + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt update + sudo apt install wget + + - name: Install YQ + uses: dcarbone/install-yq-action@v1.1.1 + + - name: Create sync matrix + id: matrix-build + # Use YQ to make a list of all unique sync groups, and format as a GH + # actions matrix + run: | + groups=$(yq -o=json -I=0 \ + '[.rpm_package_repos[] | .sync_group | select(.) ] | unique' \ + ansible/inventory/group_vars/all/package-repos) + echo -n "matrix={\"sync_group\": " >> $GITHUB_OUTPUT + echo -n $groups >> $GITHUB_OUTPUT + echo "}" >> $GITHUB_OUTPUT + echo "" + echo "Package sync matrix:" + echo -n "matrix={\"sync_group\": " + echo -n $groups + echo "}" + sync-matrix-run: name: Sync + needs: + - sync-matrix-build strategy: - matrix: - filter: ['rocky','(centos|rhel|epel|^docker|opensearch|grafana|rabbitmq|^treasuredata|elasticsearch)', 'jammy', 'focal', 'ubuntu_cloud_archive'] + matrix: ${{ fromJson(needs.sync-matrix-build.outputs.matrix) }} max-parallel: 1 fail-fast: False uses: stackhpc/stackhpc-release-train/.github/workflows/package-sync.yml@main with: sync_ark: True sync_test: False - filter: ${{ matrix.filter }} + package_sync_group: ${{ matrix.sync_group }} secrets: inherit diff --git a/.github/workflows/package-sync.yml b/.github/workflows/package-sync.yml index a534ebcc..4e7f2f9c 100644 --- a/.github/workflows/package-sync.yml +++ b/.github/workflows/package-sync.yml @@ -11,10 +11,11 @@ on: description: Sync package repositories in Test Pulp type: boolean required: true - filter: - description: Space-separated list of regular expressions matching short_name of repositories to sync + package_sync_group: + description: String matching sync_group of repositories to sync type: string - required: true + required: false + default: "" workflow_dispatch: inputs: sync_ark: @@ -62,11 +63,13 @@ jobs: ansible/dev-pulp-repo-sync.yml \ ansible/dev-pulp-repo-publication-cleanup.yml \ ansible/dev-pulp-repo-publish.yml \ + -e package_sync_group="'$PACKAGE_SYNC_GROUP'" \ -e deb_package_repo_filter="'$FILTER'" \ -e rpm_package_repo_filter="'$FILTER'" retry_wait_seconds: 3600 env: FILTER: ${{ inputs.filter }} + PACKAGE_SYNC_GROUP: ${{ inputs.package_sync_group }} package-sync-test: name: Sync package repositories in test @@ -95,8 +98,10 @@ jobs: ansible/test-pulp-repo-sync.yml \ ansible/test-pulp-repo-publication-cleanup.yml \ ansible/test-pulp-repo-publish.yml \ + -e package_sync_group="'$PACKAGE_SYNC_GROUP'" \ -e deb_package_repo_filter="'$FILTER'" \ -e rpm_package_repo_filter="'$FILTER'" retry_wait_seconds: 3600 env: FILTER: ${{ inputs.filter }} + PACKAGE_SYNC_GROUP: ${{ inputs.package_sync_group }} diff --git a/ansible/filter_plugins/filters.py b/ansible/filter_plugins/filters.py index 1a3c3024..f3045d4e 100644 --- a/ansible/filter_plugins/filters.py +++ b/ansible/filter_plugins/filters.py @@ -15,18 +15,27 @@ import re -def select_repos(repos, filter_string): - """Select repositories that match a filter string. +def select_repos(repos, filter_string, package_sync_group): + """Select repositories that match a filter string and package sync group. The filter string is a whitespace-separated list of regular expressions matching repository short names. + + The package sync group is a string matching a repository sync group. """ - if not filter_string: - return repos - regexes = filter_string.split() - patterns = re.compile(r"|".join(regexes).join('()')) - return [repo for repo in repos - if "short_name" in repo and re.search(patterns, repo["short_name"])] + if filter_string: + regexes = filter_string.split() + patterns = re.compile(r"|".join(regexes).join('()')) + filtered_repos = [repo for repo in repos + if "short_name" in repo and re.search(patterns, repo["short_name"])] + else: + filtered_repos = repos + + if package_sync_group: + return [repo for repo in filtered_repos + if "sync_group" in repo and repo["sync_group"] == package_sync_group] + else: + return filtered_repos def select_images(images, filter_string): diff --git a/ansible/inventory/group_vars/all/package-repos b/ansible/inventory/group_vars/all/package-repos index c3384f1c..91d1c116 100644 --- a/ansible/inventory/group_vars/all/package-repos +++ b/ansible/inventory/group_vars/all/package-repos @@ -1,4 +1,11 @@ --- +############################################################################### +# Universal + +# String to select a single package sync group to sync. Empty string will match +# all groups. +package_sync_group: "" + ############################################################################### # Deb @@ -16,6 +23,7 @@ # mode: Publication mode. # base_path: Base path prefix for distributions. # short_name: Name used internally for variable names. +# sync_group: Repos with the same group will be synced at the same time # distribution_name: Name prefix for distributions. Version will be appended. # sync: Whether to sync the repository with a remote. Optional, default is true. # publish: Whether to publish and distribute the repository. Optional, default is true. @@ -35,6 +43,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu/focal/ short_name: ubuntu_focal + sync_group: ubuntu_focal distribution_name: ubuntu-focal- # https://wiki.ubuntu.com/SecurityTeam/FAQ suggests that security.ubuntu.com @@ -50,6 +59,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu/focal-security/ short_name: ubuntu_focal_security + sync_group: ubuntu_focal distribution_name: ubuntu-focal-security- # Ubuntu Cloud Archive (UCA) @@ -63,6 +73,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu-cloud-archive/ short_name: ubuntu_cloud_archive + sync_group: ubuntu_cloud_archive distribution_name: ubuntu-cloud-archive- # Base Ubuntu Jammy 22.04 repositories @@ -80,6 +91,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu/jammy/ short_name: ubuntu_jammy + sync_group: ubuntu_jammy distribution_name: ubuntu-jammy- # https://wiki.ubuntu.com/SecurityTeam/FAQ suggests that security.ubuntu.com @@ -95,6 +107,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu/jammy-security/ short_name: ubuntu_jammy_security + sync_group: ubuntu_jammy distribution_name: ubuntu-jammy-security- # Third-party repositories @@ -111,6 +124,7 @@ deb_package_repos: mode: verbatim base_path: docker-ce/ubuntu-focal/ short_name: docker_ce_ubuntu_focal + sync_group: docker distribution_name: docker-ce-for-ubuntu-focal- - name: Docker CE for Ubuntu Jammy @@ -123,13 +137,14 @@ deb_package_repos: mode: verbatim base_path: docker-ce/ubuntu-jammy/ short_name: docker_ce_ubuntu_jammy + sync_group: docker distribution_name: docker-ce-for-ubuntu-jammy- # Default filter string for Deb package repositories. deb_package_repo_filter: "" # List of package repositories after applying filter. -deb_package_repos_filtered: "{{ deb_package_repos | select_repos(deb_package_repo_filter) }}" +deb_package_repos_filtered: "{{ deb_package_repos | select_repos(deb_package_repo_filter, package_sync_group) }}" ############################################################################### # RPM @@ -151,24 +166,28 @@ rpm_package_repos: url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=AppStream&infra=genclo base_path: centos/8-stream/AppStream/x86_64/os/ short_name: centos_stream_8_appstream + sync_group: centos distribution_name: centos-stream-8-appstream- sync: false - name: CentOS Stream 8 - BaseOS url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=BaseOS&infra=genclo base_path: centos/8-stream/BaseOS/x86_64/os/ short_name: centos_stream_8_baseos + sync_group: centos distribution_name: centos-stream-8-baseos- sync: false - name: CentOS Stream 8 - Extras url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=extras&infra=genclo base_path: centos/8-stream/extras/x86_64/os/ short_name: centos_stream_8_extras + sync_group: centos distribution_name: centos-stream-8-extras- sync: false - name: CentOS Stream 8 - Extras common packages url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=extras-extras-common&infra=genclo base_path: centos/8-stream/extras/x86_64/extras-common/ short_name: centos_stream_8_extras_common + sync_group: centos distribution_name: centos-stream-8-extras-common- sync: false @@ -177,84 +196,98 @@ rpm_package_repos: url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=virt-advancedvirt-common base_path: centos/8-stream/virt/x86_64/advancedvirt-common/ short_name: centos_stream_8_advanced_virtualization + sync_group: centos distribution_name: centos-stream-8-advancedvirt- sync: false - name: CentOS Stream 8 - Ceph Nautilus url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=storage-ceph-nautilus base_path: centos/8-stream/storage/x86_64/ceph-nautilus/ short_name: centos_stream_8_storage_ceph_nautilus + sync_group: centos distribution_name: centos-stream-8-storage-ceph-nautilus- sync: false - name: CentOS Stream 8 - Ceph Pacific url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=storage-ceph-pacific base_path: centos/8-stream/storage/x86_64/ceph-pacific/ short_name: centos_stream_8_storage_ceph_pacific + sync_group: centos distribution_name: centos-stream-8-storage-ceph-pacific- sync: false - name: CentOS Stream 8 - Ceph Quincy url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=storage-ceph-quincy base_path: centos/8-stream/storage/x86_64/ceph-quincy/ short_name: centos_stream_8_storage_ceph_quincy + sync_group: centos distribution_name: centos-stream-8-storage-ceph-quincy- sync: false - name: CentOS Stream 8 - NFV Extras url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=nfv-network-extras base_path: centos/8-stream/nfv/x86_64/network-extras/ short_name: centos_stream_8_nfv_extras + sync_group: centos distribution_name: centos-stream-8-nfv-extras- sync: false - name: CentOS Stream 8 - NFV OpenvSwitch url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=nfv-openvswitch-2 base_path: centos/8-stream/nfv/x86_64/openvswitch-2/ short_name: centos_stream_8_nfv_openvswitch + sync_group: centos distribution_name: centos-stream-8-nfv-openvswitch- sync: false - name: CentOS Stream 8 - OpenStack Wallaby url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=cloud-openstack-wallaby base_path: centos/8-stream/cloud/x86_64/openstack-wallaby/ short_name: centos_stream_8_openstack_wallaby + sync_group: centos distribution_name: centos-stream-8-openstack-wallaby- sync: false - name: CentOS Stream 8 - OpenStack Xena url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=cloud-openstack-xena base_path: centos/8-stream/cloud/x86_64/openstack-xena/ short_name: centos_stream_8_openstack_xena + sync_group: centos distribution_name: centos-stream-8-openstack-xena- sync: false - name: CentOS Stream 8 - OpenStack Yoga url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=cloud-openstack-yoga base_path: centos/8-stream/cloud/x86_64/openstack-yoga/ short_name: centos_stream_8_openstack_yoga + sync_group: centos distribution_name: centos-stream-8-openstack-yoga- sync: false - name: CentOS Stream 8 - OpsTools - collectd url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=opstools-collectd-5 base_path: centos/8-stream/opstools/x86_64/collectd-5/ short_name: centos_stream_8_opstools + sync_group: centos distribution_name: centos-stream-8-opstools- sync: false - name: CentOS Stream 8 - PowerTools url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=PowerTools&infra=genclo base_path: centos/8-stream/PowerTools/x86_64/os/ short_name: centos_stream_8_powertools + sync_group: centos distribution_name: centos-stream-8-powertools- sync: false - name: CentOS Stream 8 - HighAvailability url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=HighAvailability&infra=genclo base_path: centos/8-stream/HighAvailability/x86_64/os/ short_name: centos_stream_8_highavailability + sync_group: centos distribution_name: centos-stream-8-highavailability- sync: false - name: CentOS Stream 8 - Kmods Main url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=kmods-packages-main&infra=genclo base_path: centos/8-stream/kmods/x86_64/packages-main/ short_name: centos_stream_8_kmods_main + sync_group: centos distribution_name: centos-stream-8-kmods-main- sync: false - name: CentOS Stream 8 - Kmods Rebuild url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=kmods-packages-rebuild&infra=genclo base_path: centos/8-stream/kmods/x86_64/packages-rebuild/ short_name: centos_stream_8_kmods_rebuild + sync_group: centos distribution_name: centos-stream-8-kmods-rebuild- sync: false @@ -273,6 +306,7 @@ rpm_package_repos: - name: CentOS 8 - Ceph Nautilus base_path: centos/8/storage/x86_64/ceph-nautilus/ short_name: centos_8_storage_ceph_nautilus + sync_group: centos distribution_name: centos-8-storage-ceph-nautilus- sync: false - name: CentOS 8 - Ceph Octopus @@ -282,6 +316,7 @@ rpm_package_repos: - name: CentOS 8 - OpsTools - collectd base_path: centos/8/opstools/x86_64/collectd-5/ short_name: centos_8_opstools + sync_group: centos distribution_name: centos-8-opstools- sync: false @@ -310,11 +345,13 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: epel/8/Everything/x86_64/ short_name: epel + sync_group: epel distribution_name: extra-packages-for-enterprise-linux-8-x86_64- - name: Extra Packages for Enterprise Linux Modular 8 - x86_64 url: https://mirrors.fedoraproject.org/mirrorlist?repo=epel-modular-8&arch=x86_64&country=NL&infra=stock&content=centos base_path: epel/8/Modular/x86_64/ short_name: epel_modular + sync_group: epel distribution_name: extra-packages-for-enterprise-linux-modular-8-x86_64- # Third-party repositories @@ -322,6 +359,7 @@ rpm_package_repos: url: https://download.docker.com/linux/centos/8/x86_64/stable base_path: docker-ce/centos/8/x86_64/stable/ short_name: docker + sync_group: docker distribution_name: docker-ce-for-centos-8- # Note(piotrp): With repository size of 39G - for now we'll want this on_demand - name: ELK repository for 7.x packages @@ -329,6 +367,7 @@ rpm_package_repos: policy: on_demand base_path: elasticsearch/oss-7.x/ short_name: elasticsearch_logstash_kibana_7_x + sync_group: third_party distribution_name: elasticsearch-logstash-kibana-7.x- # Note(piotrp): With repository size of 16G - for now we'll want this on_demand - name: Grafana @@ -336,16 +375,19 @@ rpm_package_repos: policy: on_demand base_path: grafana/oss/rpm/ short_name: grafana + sync_group: third_party distribution_name: grafana- - name: MariaDB 10.5 url: https://dlm.mariadb.com/repo/mariadb-server/10.5/yum/rhel/8/x86_64 base_path: mariadb-10.5/yum/centos8-amd64/ short_name: mariadb_10_5_centos8 + sync_group: third_party distribution_name: mariadb-10.5-centos8 - name: MariaDB 10.6 url: https://dlm.mariadb.com/repo/mariadb-server/10.6/yum/rhel/8/x86_64 base_path: mariadb-10.6/yum/centos8-amd64/ short_name: mariadb_10_6_centos8 + sync_group: third_party distribution_name: mariadb-10.6-centos8 # RabbitMQ - Erlang for Redhat family, version 8/9 agnostic - name: RabbitMQ - Erlang @@ -355,6 +397,7 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: rabbitmq/erlang/el/8/x86_64/ short_name: rabbitmq_erlang + sync_group: third_party distribution_name: rabbitmq-erlang- # RabbitMQ for Redhat family, version 8/9 agnostic - name: RabbitMQ - Server @@ -364,11 +407,13 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: rabbitmq/rabbitmq-server/el/8/x86_64/ short_name: rabbitmq_server + sync_group: third_party distribution_name: rabbitmq-server- - name: TreasureData 4 url: http://packages.treasuredata.com/4/redhat/8/x86_64 base_path: treasuredata/4/redhat/8/x86_64/ short_name: treasuredata_4 + sync_group: third_party distribution_name: treasuredata-4- # Base Rocky Linux 8.5 repositories @@ -376,18 +421,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.5&arch=x86_64&country=NL base_path: rocky/8.5/AppStream/x86_64/os/ short_name: rocky_8_5_appstream + sync_group: third_party distribution_name: rocky-8.5-appstream- sync: false - name: Rocky Linux 8.5 - BaseOS url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.5&arch=x86_64&country=NL base_path: rocky/8.5/BaseOS/x86_64/os/ short_name: rocky_8_5_baseos + sync_group: third_party distribution_name: rocky-8.5-baseos- sync: false - name: Rocky Linux 8.5 - Extras url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-extras-8.5&arch=x86_64&country=NL base_path: rocky/8.5/extras/x86_64/os/ short_name: rocky_8_5_extras + sync_group: third_party distribution_name: rocky-8.5-extras- sync: false @@ -397,12 +445,14 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-nfv-8.5&arch=x86_64&country=NL base_path: rocky/8.5/nfv/x86_64/os/ short_name: rocky_8_5_nfv + sync_group: rocky_8 distribution_name: rocky-8.5-nfv- sync: false - name: Rocky Linux 8.5 - PowerTools url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-PowerTools-8.5&arch=x86_64&country=NL base_path: rocky/8.5/PowerTools/x86_64/os/ short_name: rocky_8_5_powertools + sync_group: rocky_8 distribution_name: rocky-8.5-powertools- sync: false @@ -411,18 +461,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.6&arch=x86_64&country=NL base_path: rocky/8.6/AppStream/x86_64/os/ short_name: rocky_8_6_appstream + sync_group: rocky_8 distribution_name: rocky-8.6-appstream- sync: false - name: Rocky Linux 8.6 - BaseOS url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.6&arch=x86_64&country=NL base_path: rocky/8.6/BaseOS/x86_64/os/ short_name: rocky_8_6_baseos + sync_group: rocky_8 distribution_name: rocky-8.6-baseos- sync: false - name: Rocky Linux 8.6 - Extras url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-extras-8.6&arch=x86_64&country=NL base_path: rocky/8.6/extras/x86_64/os/ short_name: rocky_8_6_extras + sync_group: rocky_8 distribution_name: rocky-8.6-extras- sync: false @@ -432,18 +485,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-NFV-8.6&arch=x86_64&country=NL base_path: rocky/8.6/nfv/x86_64/os/ short_name: rocky_8_6_nfv + sync_group: rocky_8 distribution_name: rocky-8.6-nfv- sync: false - name: Rocky Linux 8.6 - PowerTools url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-PowerTools-8.6&arch=x86_64&country=NL base_path: rocky/8.6/PowerTools/x86_64/os/ short_name: rocky_8_6_powertools + sync_group: rocky_8 distribution_name: rocky-8.6-powertools- sync: false - name: Rocky Linux 8.6 - HighAvailability url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-8.6&arch=x86_64&country=NL base_path: rocky/8.6/HighAvailability/x86_64/os/ short_name: rocky_8_6_highavailability + sync_group: rocky_8 distribution_name: rocky-8.6-highavailability- sync: false @@ -452,18 +508,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.7&arch=x86_64&country=NL base_path: rocky/8.7/AppStream/x86_64/os/ short_name: rocky_8_7_appstream + sync_group: rocky_8 distribution_name: rocky-8.7-appstream- sync: false - name: Rocky Linux 8.7 - BaseOS url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.7&arch=x86_64&country=NL base_path: rocky/8.7/BaseOS/x86_64/os/ short_name: rocky_8_7_baseos + sync_group: rocky_8 distribution_name: rocky-8.7-baseos- sync: false - name: Rocky Linux 8.7 - Extras url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-extras-8.7&arch=x86_64&country=NL base_path: rocky/8.7/extras/x86_64/os/ short_name: rocky_8_7_extras + sync_group: rocky_8 distribution_name: rocky-8.7-extras- sync: false @@ -473,18 +532,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-NFV-8.7&arch=x86_64&country=NL base_path: rocky/8.7/nfv/x86_64/os/ short_name: rocky_8_7_nfv + sync_group: rocky_8 distribution_name: rocky-8.7-nfv- sync: false - name: Rocky Linux 8.7 - PowerTools url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-PowerTools-8.7&arch=x86_64&country=NL base_path: rocky/8.7/PowerTools/x86_64/os/ short_name: rocky_8_7_powertools + sync_group: rocky_8 distribution_name: rocky-8.7-powertools- sync: false - name: Rocky Linux 8.7 - HighAvailability url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-8.7&arch=x86_64&country=NL base_path: rocky/8.7/HighAvailability/x86_64/os/ short_name: rocky_8_7_highavailability + sync_group: rocky_8 distribution_name: rocky-8.7-highavailability- sync: false @@ -493,18 +555,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.8&arch=x86_64&country=NL base_path: rocky/8.8/AppStream/x86_64/os/ short_name: rocky_8_8_appstream + sync_group: rocky_8 distribution_name: rocky-8.8-appstream- sync: false - name: Rocky Linux 8.8 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.8&arch=x86_64&country=NL base_path: rocky/8.8/BaseOS/x86_64/os/ short_name: rocky_8_8_baseos + sync_group: rocky_8 distribution_name: rocky-8.8-baseos- sync: false - name: Rocky Linux 8.8 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-8.8&arch=x86_64&country=NL base_path: rocky/8.8/extras/x86_64/os/ short_name: rocky_8_8_extras + sync_group: rocky_8 distribution_name: rocky-8.8-extras- sync: false @@ -514,18 +579,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-NFV-8.8&arch=x86_64&country=NL base_path: rocky/8.8/nfv/x86_64/os/ short_name: rocky_8_8_nfv + sync_group: rocky_8 distribution_name: rocky-8.8-nfv- sync: false - name: Rocky Linux 8.8 - PowerTools url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-PowerTools-8.8&arch=x86_64&country=NL base_path: rocky/8.8/PowerTools/x86_64/os/ short_name: rocky_8_8_powertools + sync_group: rocky_8 distribution_name: rocky-8.8-powertools- sync: false - name: Rocky Linux 8.8 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-8.8&arch=x86_64&country=NL base_path: rocky/8.8/HighAvailability/x86_64/os/ short_name: rocky_8_8_highavailability + sync_group: rocky_8 distribution_name: rocky-8.8-highavailability- sync: false @@ -534,6 +602,7 @@ rpm_package_repos: url: https://linux.mellanox.com/public/repo/mlnx_ofed/5.6-1.0.3.3/rhel8.6/x86_64/ base_path: mlnx_ofed/5.6-1.0.3.3/rhel8.6/x86_64/ short_name: mlnx_ofed_5_6_1_0_3_3_rhel8_6 + sync_group: third_party distribution_name: mlnx_ofed-5.6-1.0.3.3-rhel8.6- sync: false @@ -542,6 +611,7 @@ rpm_package_repos: url: https://linux.mellanox.com/public/repo/mlnx_ofed/5.7-1.0.2.0/rhel8.6/x86_64/ base_path: mlnx_ofed/5.7-1.0.2.0/rhel8.6/x86_64/ short_name: mlnx_ofed_5_7_1_0_2_0_rhel8_6 + sync_group: third_party distribution_name: mlnx_ofed-5.7-1.0.2.0-rhel8.6- sync: false @@ -550,18 +620,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-9.1&arch=x86_64&country=NL base_path: rocky/9.1/AppStream/x86_64/os/ short_name: rocky_9_1_appstream + sync_group: rocky_9 distribution_name: rocky-9.1-appstream- sync: false - name: Rocky Linux 9.1 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-9.1&arch=x86_64&country=NL base_path: rocky/9.1/BaseOS/x86_64/os/ short_name: rocky_9_1_baseos + sync_group: rocky_9 distribution_name: rocky-9.1-baseos- sync: false - name: Rocky Linux 9.1 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-9.1&arch=x86_64&country=NL base_path: rocky/9.1/extras/x86_64/os/ short_name: rocky_9_1_extras + sync_group: rocky_9 distribution_name: rocky-9.1-extras- sync: false @@ -571,12 +644,14 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-CRB-9.1&arch=x86_64&country=NL base_path: rocky/9.1/CRB/x86_64/os/ short_name: rocky_9_1_crb + sync_group: rocky_9 distribution_name: rocky-9.1-crb- sync: false - name: Rocky Linux 9.1 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-9.1&arch=x86_64&country=NL base_path: rocky/9.1/highavailability/x86_64/os/ short_name: rocky_9_1_highavailability + sync_group: rocky_9 distribution_name: rocky-9.1-highavailability- sync: false @@ -585,18 +660,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-9.2&arch=x86_64&country=NL base_path: rocky/9.2/AppStream/x86_64/os/ short_name: rocky_9_2_appstream + sync_group: rocky_9 distribution_name: rocky-9.2-appstream- sync: false - name: Rocky Linux 9.2 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-9.2&arch=x86_64&country=NL base_path: rocky/9.2/BaseOS/x86_64/os/ short_name: rocky_9_2_baseos + sync_group: rocky_9 distribution_name: rocky-9.2-baseos- sync: false - name: Rocky Linux 9.2 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-9.2&arch=x86_64&country=NL base_path: rocky/9.2/extras/x86_64/os/ short_name: rocky_9_2_extras + sync_group: rocky_9 distribution_name: rocky-9.2-extras- sync: false @@ -606,12 +684,14 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-CRB-9.2&arch=x86_64&country=NL base_path: rocky/9.2/CRB/x86_64/os/ short_name: rocky_9_2_crb + sync_group: rocky_9 distribution_name: rocky-9.2-crb- sync: false - name: Rocky Linux 9.2 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-9.2&arch=x86_64&country=NL base_path: rocky/9.2/highavailability/x86_64/os/ short_name: rocky_9_2_highavailability + sync_group: rocky_9 distribution_name: rocky-9.2-highavailability- sync: false @@ -620,18 +700,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-9.3&arch=x86_64&country=NL base_path: rocky/9.3/AppStream/x86_64/os/ short_name: rocky_9_3_appstream + sync_group: rocky_9 distribution_name: rocky-9.3-appstream- sync: false - name: Rocky Linux 9.3 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-9.3&arch=x86_64&country=NL base_path: rocky/9.3/BaseOS/x86_64/os/ short_name: rocky_9_3_baseos + sync_group: rocky_9 distribution_name: rocky-9.3-baseos- sync: false - name: Rocky Linux 9.3 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-9.3&arch=x86_64&country=NL base_path: rocky/9.3/extras/x86_64/os/ short_name: rocky_9_3_extras + sync_group: rocky_9 distribution_name: rocky-9.3-extras- sync: false @@ -641,12 +724,14 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-CRB-9.3&arch=x86_64&country=NL base_path: rocky/9.3/CRB/x86_64/os/ short_name: rocky_9_3_crb + sync_group: rocky_9 distribution_name: rocky-9.3-crb- sync: false - name: Rocky Linux 9.3 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-9.3&arch=x86_64&country=NL base_path: rocky/9.3/highavailability/x86_64/os/ short_name: rocky_9_3_highavailability + sync_group: rocky_9 distribution_name: rocky-9.3-highavailability- sync: false @@ -655,16 +740,19 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-9.4&arch=x86_64&country=NL base_path: rocky/9.4/AppStream/x86_64/os/ short_name: rocky_9_4_appstream + sync_group: rocky_9 distribution_name: rocky-9.4-appstream- - name: Rocky Linux 9.4 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-9.4&arch=x86_64&country=NL base_path: rocky/9.4/BaseOS/x86_64/os/ short_name: rocky_9_4_baseos + sync_group: rocky_9_baseos distribution_name: rocky-9.4-baseos- - name: Rocky Linux 9.4 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-9.4&arch=x86_64&country=NL base_path: rocky/9.4/extras/x86_64/os/ short_name: rocky_9_4_extras + sync_group: rocky_9 distribution_name: rocky-9.4-extras- # Additional Rocky Linux 9.4 repositories @@ -673,16 +761,19 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-CRB-9.4&arch=x86_64&country=NL base_path: rocky/9.4/CRB/x86_64/os/ short_name: rocky_9_4_crb + sync_group: rocky_9 distribution_name: rocky-9.4-crb- - name: Rocky Linux 9.4 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-9.4&arch=x86_64&country=NL base_path: rocky/9.4/highavailability/x86_64/os/ short_name: rocky_9_4_highavailability + sync_group: rocky_9 distribution_name: rocky-9.4-highavailability- - name: Rocky Linux 9 - SIG Security Common url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-sig-security-common-9&arch=x86_64&country=NL base_path: rocky/sig/9/security/x86_64/security-common/ short_name: rocky_9_sig_security_common + sync_group: rocky_9 distribution_name: rocky-9-sig-security-common- # Additional CentOS Stream 9 repositories @@ -691,30 +782,35 @@ rpm_package_repos: url: https://mirror.stream.centos.org/SIGs/9-stream/nfv/x86_64/openvswitch-2/ base_path: centos/9-stream/nfv/x86_64/openvswitch-2/ short_name: centos_stream_9_nfv_openvswitch + sync_group: centos distribution_name: centos-stream-9-nfv-openvswitch- # Opstools for CentOS Stream 9 - name: CentOS Stream 9 - opstools url: https://mirror.stream.centos.org/SIGs/9-stream/opstools/x86_64/collectd-5/ base_path: centos/9-stream/opstools/x86_64/collectd-5/ short_name: centos_stream_9_opstools + sync_group: centos distribution_name: centos-stream-9-opstools- # Ceph Pacific for CentOS Stream 9 - name: CentOS Stream 9 - Ceph Pacific url: https://mirror.stream.centos.org/SIGs/9-stream/storage/x86_64/ceph-pacific/ base_path: centos/9-stream/storage/x86_64/ceph-pacific/ short_name: centos_stream_9_storage_ceph_pacific + sync_group: centos distribution_name: centos-stream-9-storage-ceph-pacific- # Ceph Quincy for CentOS Stream 9 - name: CentOS Stream 9 - Ceph Quincy url: https://mirror.stream.centos.org/SIGs/9-stream/storage/x86_64/ceph-quincy/ base_path: centos/9-stream/storage/x86_64/ceph-quincy/ short_name: centos_stream_9_storage_ceph_quincy + sync_group: centos distribution_name: centos-stream-9-storage-ceph-quincy- # Ceph Reef for CentOS Stream 9 - name: CentOS Stream 9 - Ceph Reef url: https://mirror.stream.centos.org/SIGs/9-stream/storage/x86_64/ceph-reef/ base_path: centos/9-stream/storage/x86_64/ceph-reef/ short_name: centos_stream_9_storage_ceph_reef + sync_group: centos distribution_name: centos-stream-9-storage-ceph-reef- # EPEL 9 repository - name: Extra Packages for Enterprise Linux 9 @@ -722,6 +818,7 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: epel/9/Everything/x86_64/ short_name: epel_9 + sync_group: epel distribution_name: extra-packages-for-enterprise-linux-9-x86_64- # ELRepo 9 repository - name: ELRepo.org Community Enterprise Linux Repository - el9 @@ -729,30 +826,35 @@ rpm_package_repos: url: https://lon.mirror.rackspace.com/elrepo/elrepo/el9/x86_64/ base_path: elrepo/elrepo/el9/x86_64/ short_name: elrepo_9 + sync_group: epel distribution_name: elrepo-el9-x86_64- # Docker CE for CentOS Stream 9 - name: Docker CE - CentOS Stream 9 url: https://download.docker.com/linux/centos/9/x86_64/stable base_path: docker-ce/centos/9/x86_64/stable/ short_name: centos_stream_9_docker + sync_group: centos distribution_name: docker-ce-for-centos-9- # Openstack Yoga for CentOS Stream 9 - name: CentOS Stream 9 - OpenStack Yoga url: https://mirror.stream.centos.org/SIGs/9-stream/cloud/x86_64/openstack-yoga/ base_path: centos/9-stream/cloud/x86_64/openstack-yoga/ short_name: centos_stream_9_openstack_yoga + sync_group: centos distribution_name: centos-stream-9-openstack-yoga- # Openstack Zed for CentOS Stream 9 - name: CentOS Stream 9 - OpenStack Zed url: https://mirror.stream.centos.org/SIGs/9-stream/cloud/x86_64/openstack-zed/ base_path: centos/9-stream/cloud/x86_64/openstack-zed/ short_name: centos_stream_9_openstack_zed + sync_group: centos distribution_name: centos-stream-9-openstack-zed- # OpenStack Antelope for CentOS Stream 9 - name: CentOS Stream 9 - OpenStack Antelope url: https://mirror.stream.centos.org/SIGs/9-stream/cloud/x86_64/openstack-antelope/ base_path: centos/9-stream/cloud/x86_64/openstack-antelope/ short_name: centos_stream_9_openstack_antelope + sync_group: centos distribution_name: centos-stream-9-openstack-antelope- # Additional RHEL 9 repositories @@ -761,46 +863,53 @@ rpm_package_repos: url: http://packages.treasuredata.com/4/redhat/9/x86_64 base_path: treasuredata/4/redhat/9/x86_64/ short_name: rhel_9_treasuredata_4 + sync_group: third_party distribution_name: rhel_9_treasuredata-4- # MariaDB 10.6 for RHEL 9 - name: MariaDB 10.6 - RHEL 9 url: https://dlm.mariadb.com/repo/mariadb-server/10.6/yum/rhel/9/x86_64 base_path: mariadb-10.6/yum/rhel/9/x86_64/ short_name: rhel_9_mariadb_10_6 + sync_group: third_party distribution_name: mariadb-10.6-rhel-9- # InfluxDB for RHEL 9 - name: InfluxDB - RHEL 9 url: https://repos.influxdata.com/rhel/9/x86_64/stable/ base_path: influxdb/rhel/9/x86_64/stable/ short_name: rhel_9_influxdb + sync_group: third_party distribution_name: influxdb-rhel-9- # MLNX OFED 5.9-0.5.6.0 RHEL 9.1 repository - name: Mellanox Technologies mlnx_ofed 5.9-0.5.6.0 url: https://linux.mellanox.com/public/repo/mlnx_ofed/5.9-0.5.6.0/rhel9.1/x86_64/ base_path: mlnx_ofed/5.9-0.5.6.0/rhel9.1/x86_64/ short_name: mlnx_ofed_5_9_0_5_6_0_rhel9_1 + sync_group: third_party distribution_name: mlnx_ofed-5.9-0.5.6.0-rhel9.1- # MLNX OFED 23.07-0.5.1.2 RHEL 9.2 repository - name: Mellanox Technologies mlnx_ofed 23.07-0.5.1.2 url: https://linux.mellanox.com/public/repo/mlnx_ofed/23.07-0.5.1.2/rhel9.2/x86_64/ base_path: mlnx_ofed/23.07-0.5.1.2/rhel9.2/x86_64/ short_name: mlnx_ofed_23_07_0_5_1_2_rhel9_2 + sync_group: third_party distribution_name: mlnx_ofed-23.07-0.5.1.2-rhel9.2- # OpenSearch 2.x - name: OpenSearch 2.x url: https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/yum base_path: opensearch/2.x/yum/ short_name: opensearch_2_x + sync_group: third_party distribution_name: opensearch-2.x-yum- # OpenSearch dashboards - name: OpenSearch Dashboards 2.x url: https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/yum base_path: opensearch-dashboards/2.x/yum/ short_name: opensearch_dashboards_2_x + sync_group: third_party distribution_name: opensearch_dashboards-2.x-yum- # Default filter string for RPM package repositories. rpm_package_repo_filter: "" # List of package repositories after applying filter. -rpm_package_repos_filtered: "{{ rpm_package_repos | select_repos(rpm_package_repo_filter) }}" +rpm_package_repos_filtered: "{{ rpm_package_repos | select_repos(rpm_package_repo_filter, package_sync_group) }}" diff --git a/ansible/sync-matrix-build.yml b/ansible/sync-matrix-build.yml new file mode 100644 index 00000000..e69de29b diff --git a/ansible/validate-deb-repos.yml b/ansible/validate-deb-repos.yml index 88d5924c..feb7d3d8 100644 --- a/ansible/validate-deb-repos.yml +++ b/ansible/validate-deb-repos.yml @@ -71,10 +71,24 @@ vars: deb_package_repo_filter: docker_ce_ubuntu_jammy ubuntu_jammy_security$ - - name: Assert that dev package repository list is defined + - name: Assert that Deb package repository list can be filtered to one group assert: that: - - dev_pulp_repository_deb_repos is defined + - deb_package_repos_filtered | length == 2 + - deb_package_repos_filtered[0].short_name == 'docker_ce_ubuntu_focal' + - deb_package_repos_filtered[1].short_name == 'docker_ce_ubuntu_jammy' + vars: + package_sync_group: docker + + - name: Assert that Deb package repository list can be filtered and use a sync group together + assert: + that: + - deb_package_repos_filtered | length == 1 + - deb_package_repos_filtered[0].short_name == 'docker_ce_ubuntu_jammy' + vars: + package_sync_group: docker + deb_package_repo_filter: jammy + - name: Assert that dev package publication list is defined assert: From 7f4ebcf4f50531b89f6fc966dce3aa7663979c01 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 2 Sep 2024 14:14:46 +0100 Subject: [PATCH 2/8] Increase package sync timeout Some individual repos have started taking >6h to sync. This Commit bumps the timeout so they don't expire. --- .github/workflows/package-sync.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/package-sync.yml b/.github/workflows/package-sync.yml index 4e7f2f9c..a6d05c44 100644 --- a/.github/workflows/package-sync.yml +++ b/.github/workflows/package-sync.yml @@ -40,7 +40,7 @@ jobs: package-sync-ark: name: Sync package repositories in Ark runs-on: arc-release-train-runner - timeout-minutes: 480 + timeout-minutes: 720 if: inputs.sync_ark steps: @@ -56,7 +56,7 @@ jobs: - name: Sync and publish package repositories in Ark uses: nick-fields/retry@v3 with: - timeout_minutes: 360 + timeout_minutes: 480 max_attempts: 2 command: | ansible-playbook -i ansible/inventory \ @@ -75,7 +75,7 @@ jobs: name: Sync package repositories in test runs-on: arc-release-train-runner needs: package-sync-ark - timeout-minutes: 480 + timeout-minutes: 720 if: inputs.sync_test steps: - name: Checkout @@ -90,7 +90,7 @@ jobs: - name: Sync and publish package repositories in test uses: nick-fields/retry@v3 with: - timeout_minutes: 360 + timeout_minutes: 480 max_attempts: 2 command: | ansible-playbook -i ansible/inventory \ From 5bccb7d19397d2291d17f3ad30da5473834a5a81 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 2 Sep 2024 15:27:01 +0100 Subject: [PATCH 3/8] Add Ubuntu to nightly package repo sync matrix --- .github/workflows/package-sync-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-sync-nightly.yml b/.github/workflows/package-sync-nightly.yml index 3b68529a..aa855a8b 100644 --- a/.github/workflows/package-sync-nightly.yml +++ b/.github/workflows/package-sync-nightly.yml @@ -32,7 +32,7 @@ jobs: # actions matrix run: | groups=$(yq -o=json -I=0 \ - '[.rpm_package_repos[] | .sync_group | select(.) ] | unique' \ + '([.rpm_package_repos[] | .sync_group | select(.) ]) + ([.deb_package_repos[] | .sync_group | select(.) ]) | unique' \ ansible/inventory/group_vars/all/package-repos) echo -n "matrix={\"sync_group\": " >> $GITHUB_OUTPUT echo -n $groups >> $GITHUB_OUTPUT From 776eb957af9af27d002c7cd03643ce14a7b1da38 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 2 Sep 2024 16:03:03 +0100 Subject: [PATCH 4/8] Remove stray file --- ansible/sync-matrix-build.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ansible/sync-matrix-build.yml diff --git a/ansible/sync-matrix-build.yml b/ansible/sync-matrix-build.yml deleted file mode 100644 index e69de29b..00000000 From 224fc955de03f0bbbe17c2cc7800487dfe5fdedf Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 5 Sep 2024 12:00:57 +0100 Subject: [PATCH 5/8] Post-review fixes Added new tests to check sync groups are defined for all synced repos Updated sync group on various repos --- .github/workflows/package-sync-nightly.yml | 1 - .../inventory/group_vars/all/package-repos | 12 ++++++---- ansible/validate-deb-repos.yml | 9 ++++++++ ansible/validate-rpm-repos.yml | 23 +++++++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/package-sync-nightly.yml b/.github/workflows/package-sync-nightly.yml index aa855a8b..853a6420 100644 --- a/.github/workflows/package-sync-nightly.yml +++ b/.github/workflows/package-sync-nightly.yml @@ -6,7 +6,6 @@ on: - cron: '17 23 * * *' env: ANSIBLE_FORCE_COLOR: True - ANSIBLE_VAULT_PASSWORD_FILE: ${{ github.workspace }}/vault-pass jobs: sync-matrix-build: name: Build package matrix of package repo sync jobs diff --git a/ansible/inventory/group_vars/all/package-repos b/ansible/inventory/group_vars/all/package-repos index 135a3ece..8bed9c8e 100644 --- a/ansible/inventory/group_vars/all/package-repos +++ b/ansible/inventory/group_vars/all/package-repos @@ -421,21 +421,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.5&arch=x86_64&country=NL base_path: rocky/8.5/AppStream/x86_64/os/ short_name: rocky_8_5_appstream - sync_group: third_party + sync_group: rocky_8 distribution_name: rocky-8.5-appstream- sync: false - name: Rocky Linux 8.5 - BaseOS url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.5&arch=x86_64&country=NL base_path: rocky/8.5/BaseOS/x86_64/os/ short_name: rocky_8_5_baseos - sync_group: third_party + sync_group: rocky_8 distribution_name: rocky-8.5-baseos- sync: false - name: Rocky Linux 8.5 - Extras url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-extras-8.5&arch=x86_64&country=NL base_path: rocky/8.5/extras/x86_64/os/ short_name: rocky_8_5_extras - sync_group: third_party + sync_group: rocky_8 distribution_name: rocky-8.5-extras- sync: false @@ -833,7 +833,7 @@ rpm_package_repos: url: https://download.docker.com/linux/centos/9/x86_64/stable base_path: docker-ce/centos/9/x86_64/stable/ short_name: centos_stream_9_docker - sync_group: centos + sync_group: docker distribution_name: docker-ce-for-centos-9- # Openstack Yoga for CentOS Stream 9 - name: CentOS Stream 9 - OpenStack Yoga @@ -861,6 +861,7 @@ rpm_package_repos: url: https://mirror.stream.centos.org/SIGs/9-stream/cloud/x86_64/openstack-caracal/ base_path: centos/9-stream/cloud/x86_64/openstack-caracal/ short_name: centos_stream_9_openstack_caracal + sync_group: centos distribution_name: centos-stream-9-openstack-caracal- # Additional RHEL 9 repositories @@ -872,6 +873,7 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: rabbitmq/erlang/el/9/x86_64/ short_name: rhel9_rabbitmq_erlang + sync_group: third_party distribution_name: rhel9-rabbitmq-erlang- # RabbitMQ for Redhat family, version 9 - name: RabbitMQ - Server - RHEL 9 @@ -881,6 +883,7 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: rabbitmq/rabbitmq-server/el/9/x86_64/ short_name: rhel9_rabbitmq_server + sync_group: third_party distribution_name: rhel9-rabbitmq-server- # TreasureData 4 for RHEL 9 - name: TreasureData 4 - RHEL 9 @@ -908,6 +911,7 @@ rpm_package_repos: url: https://dlm.mariadb.com/repo/mariadb-server/10.11/yum/rhel/9/x86_64 base_path: mariadb-10.11/yum/rhel/9/x86_64/ short_name: rhel_9_mariadb_10_11 + sync_group: third_party distribution_name: mariadb-10.11-rhel-9- # InfluxDB for RHEL 9 - name: InfluxDB - RHEL 9 diff --git a/ansible/validate-deb-repos.yml b/ansible/validate-deb-repos.yml index feb7d3d8..f66eaedc 100644 --- a/ansible/validate-deb-repos.yml +++ b/ansible/validate-deb-repos.yml @@ -28,6 +28,11 @@ that: - synced_deb_package_repos | rejectattr('url', 'defined') | list | length == 0 + - name: Assert that synced Deb package repositories have sync groups + assert: + that: + - synced_deb_package_repos | rejectattr('sync_group', 'defined') | list | length == 0 + - name: Assert that published Deb package repositories have short names assert: that: @@ -89,6 +94,10 @@ package_sync_group: docker deb_package_repo_filter: jammy + - name: Assert that dev package repository list is defined + assert: + that: + - dev_pulp_repository_deb_repos is defined - name: Assert that dev package publication list is defined assert: diff --git a/ansible/validate-rpm-repos.yml b/ansible/validate-rpm-repos.yml index 77ee40cd..77ce3555 100644 --- a/ansible/validate-rpm-repos.yml +++ b/ansible/validate-rpm-repos.yml @@ -28,6 +28,11 @@ that: - synced_rpm_package_repos | rejectattr('url', 'defined') | list | length == 0 + - name: Assert that synced RPM package repositories have sync groups + assert: + that: + - synced_rpm_package_repos | rejectattr('sync_group', 'defined') | list | length == 0 + - name: Assert that published RPM package repositories have short names assert: that: @@ -72,6 +77,24 @@ vars: rpm_package_repo_filter: docker centos_stream_8_baseos centos_stream_9_docker + - name: Assert that RPM package repository list can be filtered to one group + assert: + that: + - rpm_package_repos_filtered | length == 2 + - rpm_package_repos_filtered[0].short_name == 'docker' + - rpm_package_repos_filtered[1].short_name == 'centos_stream_9_docker' + vars: + package_sync_group: docker + + - name: Assert that RPM package repository list can be filtered and use a sync group together + assert: + that: + - rpm_package_repos_filtered | length == 1 + - rpm_package_repos_filtered[0].short_name == 'centos_stream_9_docker' + vars: + package_sync_group: docker + rpm_package_repo_filter: stream_9 + - name: Assert that dev package repository list is defined assert: that: From 6c3a2f7382512cf0a9a71307d20fe8cf53984582 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 5 Sep 2024 12:03:26 +0100 Subject: [PATCH 6/8] Revert "Increase package sync timeout" Increasing the timeout did not help syncs This reverts commit 7f4ebcf4f50531b89f6fc966dce3aa7663979c01. --- .github/workflows/package-sync.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/package-sync.yml b/.github/workflows/package-sync.yml index a6d05c44..4e7f2f9c 100644 --- a/.github/workflows/package-sync.yml +++ b/.github/workflows/package-sync.yml @@ -40,7 +40,7 @@ jobs: package-sync-ark: name: Sync package repositories in Ark runs-on: arc-release-train-runner - timeout-minutes: 720 + timeout-minutes: 480 if: inputs.sync_ark steps: @@ -56,7 +56,7 @@ jobs: - name: Sync and publish package repositories in Ark uses: nick-fields/retry@v3 with: - timeout_minutes: 480 + timeout_minutes: 360 max_attempts: 2 command: | ansible-playbook -i ansible/inventory \ @@ -75,7 +75,7 @@ jobs: name: Sync package repositories in test runs-on: arc-release-train-runner needs: package-sync-ark - timeout-minutes: 720 + timeout-minutes: 480 if: inputs.sync_test steps: - name: Checkout @@ -90,7 +90,7 @@ jobs: - name: Sync and publish package repositories in test uses: nick-fields/retry@v3 with: - timeout_minutes: 480 + timeout_minutes: 360 max_attempts: 2 command: | ansible-playbook -i ansible/inventory \ From 4f6e434ad5e129a24a18edc1799fc9825b08bbf4 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Fri, 6 Sep 2024 09:16:06 +0100 Subject: [PATCH 7/8] Move debug print to separate job --- .github/workflows/package-sync-nightly.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/package-sync-nightly.yml b/.github/workflows/package-sync-nightly.yml index 853a6420..bcf39858 100644 --- a/.github/workflows/package-sync-nightly.yml +++ b/.github/workflows/package-sync-nightly.yml @@ -36,11 +36,11 @@ jobs: echo -n "matrix={\"sync_group\": " >> $GITHUB_OUTPUT echo -n $groups >> $GITHUB_OUTPUT echo "}" >> $GITHUB_OUTPUT - echo "" + + - name: Print sync matrix + run: | echo "Package sync matrix:" - echo -n "matrix={\"sync_group\": " - echo -n $groups - echo "}" + echo -n "${{ steps.matrix-build.outputs.matrix }}" sync-matrix-run: name: Sync From e28c3ca78025b6c17d68db183f297d1d20b6b3d6 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Fri, 6 Sep 2024 15:14:26 +0100 Subject: [PATCH 8/8] Use non-broken ubuntu package mirrors --- ansible/inventory/group_vars/all/package-repos | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/inventory/group_vars/all/package-repos b/ansible/inventory/group_vars/all/package-repos index 8bed9c8e..f1e858fe 100644 --- a/ansible/inventory/group_vars/all/package-repos +++ b/ansible/inventory/group_vars/all/package-repos @@ -30,7 +30,7 @@ package_sync_group: "" deb_package_repos: # Base Ubuntu Focal 20.04 repositories - name: Ubuntu focal - url: http://nova.clouds.archive.ubuntu.com/ubuntu/ + url: http://archive.ubuntu.com/ubuntu policy: immediate architectures: amd64 components: main restricted universe multiverse @@ -78,7 +78,7 @@ deb_package_repos: # Base Ubuntu Jammy 22.04 repositories - name: Ubuntu jammy - url: http://nova.clouds.archive.ubuntu.com/ubuntu/ + url: http://archive.ubuntu.com/ubuntu policy: immediate architectures: amd64 components: main restricted universe multiverse