diff --git a/.github/workflows/package-sync-nightly.yml b/.github/workflows/package-sync-nightly.yml index 37279dad..bcf39858 100644 --- a/.github/workflows/package-sync-nightly.yml +++ b/.github/workflows/package-sync-nightly.yml @@ -7,16 +7,52 @@ on: env: ANSIBLE_FORCE_COLOR: True jobs: + sync-matrix-build: + name: Build package matrix of package repo sync jobs + runs-on: arc-release-train-runner + outputs: + matrix: ${{ steps.matrix-build.outputs.matrix }} + steps: + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt update + sudo apt install wget + + - name: Install YQ + uses: dcarbone/install-yq-action@v1.1.1 + + - name: Create sync matrix + id: matrix-build + # Use YQ to make a list of all unique sync groups, and format as a GH + # actions matrix + run: | + groups=$(yq -o=json -I=0 \ + '([.rpm_package_repos[] | .sync_group | select(.) ]) + ([.deb_package_repos[] | .sync_group | select(.) ]) | unique' \ + ansible/inventory/group_vars/all/package-repos) + echo -n "matrix={\"sync_group\": " >> $GITHUB_OUTPUT + echo -n $groups >> $GITHUB_OUTPUT + echo "}" >> $GITHUB_OUTPUT + + - name: Print sync matrix + run: | + echo "Package sync matrix:" + echo -n "${{ steps.matrix-build.outputs.matrix }}" + sync-matrix-run: name: Sync + needs: + - sync-matrix-build strategy: - matrix: - filter: ['rocky','(centos|rhel|epel|^docker|opensearch|grafana|rabbitmq|^treasuredata|elasticsearch)', 'jammy', 'focal', 'ubuntu_cloud_archive'] + matrix: ${{ fromJson(needs.sync-matrix-build.outputs.matrix) }} max-parallel: 1 fail-fast: False uses: stackhpc/stackhpc-release-train/.github/workflows/package-sync.yml@main with: sync_ark: True sync_test: False - filter: ${{ matrix.filter }} + package_sync_group: ${{ matrix.sync_group }} secrets: inherit diff --git a/.github/workflows/package-sync.yml b/.github/workflows/package-sync.yml index a534ebcc..4e7f2f9c 100644 --- a/.github/workflows/package-sync.yml +++ b/.github/workflows/package-sync.yml @@ -11,10 +11,11 @@ on: description: Sync package repositories in Test Pulp type: boolean required: true - filter: - description: Space-separated list of regular expressions matching short_name of repositories to sync + package_sync_group: + description: String matching sync_group of repositories to sync type: string - required: true + required: false + default: "" workflow_dispatch: inputs: sync_ark: @@ -62,11 +63,13 @@ jobs: ansible/dev-pulp-repo-sync.yml \ ansible/dev-pulp-repo-publication-cleanup.yml \ ansible/dev-pulp-repo-publish.yml \ + -e package_sync_group="'$PACKAGE_SYNC_GROUP'" \ -e deb_package_repo_filter="'$FILTER'" \ -e rpm_package_repo_filter="'$FILTER'" retry_wait_seconds: 3600 env: FILTER: ${{ inputs.filter }} + PACKAGE_SYNC_GROUP: ${{ inputs.package_sync_group }} package-sync-test: name: Sync package repositories in test @@ -95,8 +98,10 @@ jobs: ansible/test-pulp-repo-sync.yml \ ansible/test-pulp-repo-publication-cleanup.yml \ ansible/test-pulp-repo-publish.yml \ + -e package_sync_group="'$PACKAGE_SYNC_GROUP'" \ -e deb_package_repo_filter="'$FILTER'" \ -e rpm_package_repo_filter="'$FILTER'" retry_wait_seconds: 3600 env: FILTER: ${{ inputs.filter }} + PACKAGE_SYNC_GROUP: ${{ inputs.package_sync_group }} diff --git a/ansible/filter_plugins/filters.py b/ansible/filter_plugins/filters.py index 1a3c3024..f3045d4e 100644 --- a/ansible/filter_plugins/filters.py +++ b/ansible/filter_plugins/filters.py @@ -15,18 +15,27 @@ import re -def select_repos(repos, filter_string): - """Select repositories that match a filter string. +def select_repos(repos, filter_string, package_sync_group): + """Select repositories that match a filter string and package sync group. The filter string is a whitespace-separated list of regular expressions matching repository short names. + + The package sync group is a string matching a repository sync group. """ - if not filter_string: - return repos - regexes = filter_string.split() - patterns = re.compile(r"|".join(regexes).join('()')) - return [repo for repo in repos - if "short_name" in repo and re.search(patterns, repo["short_name"])] + if filter_string: + regexes = filter_string.split() + patterns = re.compile(r"|".join(regexes).join('()')) + filtered_repos = [repo for repo in repos + if "short_name" in repo and re.search(patterns, repo["short_name"])] + else: + filtered_repos = repos + + if package_sync_group: + return [repo for repo in filtered_repos + if "sync_group" in repo and repo["sync_group"] == package_sync_group] + else: + return filtered_repos def select_images(images, filter_string): diff --git a/ansible/inventory/group_vars/all/package-repos b/ansible/inventory/group_vars/all/package-repos index 5f86886c..f1e858fe 100644 --- a/ansible/inventory/group_vars/all/package-repos +++ b/ansible/inventory/group_vars/all/package-repos @@ -1,4 +1,11 @@ --- +############################################################################### +# Universal + +# String to select a single package sync group to sync. Empty string will match +# all groups. +package_sync_group: "" + ############################################################################### # Deb @@ -16,13 +23,14 @@ # mode: Publication mode. # base_path: Base path prefix for distributions. # short_name: Name used internally for variable names. +# sync_group: Repos with the same group will be synced at the same time # distribution_name: Name prefix for distributions. Version will be appended. # sync: Whether to sync the repository with a remote. Optional, default is true. # publish: Whether to publish and distribute the repository. Optional, default is true. deb_package_repos: # Base Ubuntu Focal 20.04 repositories - name: Ubuntu focal - url: http://nova.clouds.archive.ubuntu.com/ubuntu/ + url: http://archive.ubuntu.com/ubuntu policy: immediate architectures: amd64 components: main restricted universe multiverse @@ -35,6 +43,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu/focal/ short_name: ubuntu_focal + sync_group: ubuntu_focal distribution_name: ubuntu-focal- # https://wiki.ubuntu.com/SecurityTeam/FAQ suggests that security.ubuntu.com @@ -50,6 +59,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu/focal-security/ short_name: ubuntu_focal_security + sync_group: ubuntu_focal distribution_name: ubuntu-focal-security- # Ubuntu Cloud Archive (UCA) @@ -63,11 +73,12 @@ deb_package_repos: mode: verbatim base_path: ubuntu-cloud-archive/ short_name: ubuntu_cloud_archive + sync_group: ubuntu_cloud_archive distribution_name: ubuntu-cloud-archive- # Base Ubuntu Jammy 22.04 repositories - name: Ubuntu jammy - url: http://nova.clouds.archive.ubuntu.com/ubuntu/ + url: http://archive.ubuntu.com/ubuntu policy: immediate architectures: amd64 components: main restricted universe multiverse @@ -80,6 +91,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu/jammy/ short_name: ubuntu_jammy + sync_group: ubuntu_jammy distribution_name: ubuntu-jammy- # https://wiki.ubuntu.com/SecurityTeam/FAQ suggests that security.ubuntu.com @@ -95,6 +107,7 @@ deb_package_repos: mode: verbatim base_path: ubuntu/jammy-security/ short_name: ubuntu_jammy_security + sync_group: ubuntu_jammy distribution_name: ubuntu-jammy-security- # Third-party repositories @@ -111,6 +124,7 @@ deb_package_repos: mode: verbatim base_path: docker-ce/ubuntu-focal/ short_name: docker_ce_ubuntu_focal + sync_group: docker distribution_name: docker-ce-for-ubuntu-focal- - name: Docker CE for Ubuntu Jammy @@ -123,13 +137,14 @@ deb_package_repos: mode: verbatim base_path: docker-ce/ubuntu-jammy/ short_name: docker_ce_ubuntu_jammy + sync_group: docker distribution_name: docker-ce-for-ubuntu-jammy- # Default filter string for Deb package repositories. deb_package_repo_filter: "" # List of package repositories after applying filter. -deb_package_repos_filtered: "{{ deb_package_repos | select_repos(deb_package_repo_filter) }}" +deb_package_repos_filtered: "{{ deb_package_repos | select_repos(deb_package_repo_filter, package_sync_group) }}" ############################################################################### # RPM @@ -151,24 +166,28 @@ rpm_package_repos: url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=AppStream&infra=genclo base_path: centos/8-stream/AppStream/x86_64/os/ short_name: centos_stream_8_appstream + sync_group: centos distribution_name: centos-stream-8-appstream- sync: false - name: CentOS Stream 8 - BaseOS url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=BaseOS&infra=genclo base_path: centos/8-stream/BaseOS/x86_64/os/ short_name: centos_stream_8_baseos + sync_group: centos distribution_name: centos-stream-8-baseos- sync: false - name: CentOS Stream 8 - Extras url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=extras&infra=genclo base_path: centos/8-stream/extras/x86_64/os/ short_name: centos_stream_8_extras + sync_group: centos distribution_name: centos-stream-8-extras- sync: false - name: CentOS Stream 8 - Extras common packages url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=extras-extras-common&infra=genclo base_path: centos/8-stream/extras/x86_64/extras-common/ short_name: centos_stream_8_extras_common + sync_group: centos distribution_name: centos-stream-8-extras-common- sync: false @@ -177,84 +196,98 @@ rpm_package_repos: url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=virt-advancedvirt-common base_path: centos/8-stream/virt/x86_64/advancedvirt-common/ short_name: centos_stream_8_advanced_virtualization + sync_group: centos distribution_name: centos-stream-8-advancedvirt- sync: false - name: CentOS Stream 8 - Ceph Nautilus url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=storage-ceph-nautilus base_path: centos/8-stream/storage/x86_64/ceph-nautilus/ short_name: centos_stream_8_storage_ceph_nautilus + sync_group: centos distribution_name: centos-stream-8-storage-ceph-nautilus- sync: false - name: CentOS Stream 8 - Ceph Pacific url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=storage-ceph-pacific base_path: centos/8-stream/storage/x86_64/ceph-pacific/ short_name: centos_stream_8_storage_ceph_pacific + sync_group: centos distribution_name: centos-stream-8-storage-ceph-pacific- sync: false - name: CentOS Stream 8 - Ceph Quincy url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=storage-ceph-quincy base_path: centos/8-stream/storage/x86_64/ceph-quincy/ short_name: centos_stream_8_storage_ceph_quincy + sync_group: centos distribution_name: centos-stream-8-storage-ceph-quincy- sync: false - name: CentOS Stream 8 - NFV Extras url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=nfv-network-extras base_path: centos/8-stream/nfv/x86_64/network-extras/ short_name: centos_stream_8_nfv_extras + sync_group: centos distribution_name: centos-stream-8-nfv-extras- sync: false - name: CentOS Stream 8 - NFV OpenvSwitch url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=nfv-openvswitch-2 base_path: centos/8-stream/nfv/x86_64/openvswitch-2/ short_name: centos_stream_8_nfv_openvswitch + sync_group: centos distribution_name: centos-stream-8-nfv-openvswitch- sync: false - name: CentOS Stream 8 - OpenStack Wallaby url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=cloud-openstack-wallaby base_path: centos/8-stream/cloud/x86_64/openstack-wallaby/ short_name: centos_stream_8_openstack_wallaby + sync_group: centos distribution_name: centos-stream-8-openstack-wallaby- sync: false - name: CentOS Stream 8 - OpenStack Xena url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=cloud-openstack-xena base_path: centos/8-stream/cloud/x86_64/openstack-xena/ short_name: centos_stream_8_openstack_xena + sync_group: centos distribution_name: centos-stream-8-openstack-xena- sync: false - name: CentOS Stream 8 - OpenStack Yoga url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=cloud-openstack-yoga base_path: centos/8-stream/cloud/x86_64/openstack-yoga/ short_name: centos_stream_8_openstack_yoga + sync_group: centos distribution_name: centos-stream-8-openstack-yoga- sync: false - name: CentOS Stream 8 - OpsTools - collectd url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=opstools-collectd-5 base_path: centos/8-stream/opstools/x86_64/collectd-5/ short_name: centos_stream_8_opstools + sync_group: centos distribution_name: centos-stream-8-opstools- sync: false - name: CentOS Stream 8 - PowerTools url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=PowerTools&infra=genclo base_path: centos/8-stream/PowerTools/x86_64/os/ short_name: centos_stream_8_powertools + sync_group: centos distribution_name: centos-stream-8-powertools- sync: false - name: CentOS Stream 8 - HighAvailability url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=HighAvailability&infra=genclo base_path: centos/8-stream/HighAvailability/x86_64/os/ short_name: centos_stream_8_highavailability + sync_group: centos distribution_name: centos-stream-8-highavailability- sync: false - name: CentOS Stream 8 - Kmods Main url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=kmods-packages-main&infra=genclo base_path: centos/8-stream/kmods/x86_64/packages-main/ short_name: centos_stream_8_kmods_main + sync_group: centos distribution_name: centos-stream-8-kmods-main- sync: false - name: CentOS Stream 8 - Kmods Rebuild url: http://mirrorlist.centos.org/?release=8-stream&arch=x86_64&country=NL&repo=kmods-packages-rebuild&infra=genclo base_path: centos/8-stream/kmods/x86_64/packages-rebuild/ short_name: centos_stream_8_kmods_rebuild + sync_group: centos distribution_name: centos-stream-8-kmods-rebuild- sync: false @@ -273,6 +306,7 @@ rpm_package_repos: - name: CentOS 8 - Ceph Nautilus base_path: centos/8/storage/x86_64/ceph-nautilus/ short_name: centos_8_storage_ceph_nautilus + sync_group: centos distribution_name: centos-8-storage-ceph-nautilus- sync: false - name: CentOS 8 - Ceph Octopus @@ -282,6 +316,7 @@ rpm_package_repos: - name: CentOS 8 - OpsTools - collectd base_path: centos/8/opstools/x86_64/collectd-5/ short_name: centos_8_opstools + sync_group: centos distribution_name: centos-8-opstools- sync: false @@ -310,11 +345,13 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: epel/8/Everything/x86_64/ short_name: epel + sync_group: epel distribution_name: extra-packages-for-enterprise-linux-8-x86_64- - name: Extra Packages for Enterprise Linux Modular 8 - x86_64 url: https://mirrors.fedoraproject.org/mirrorlist?repo=epel-modular-8&arch=x86_64&country=NL&infra=stock&content=centos base_path: epel/8/Modular/x86_64/ short_name: epel_modular + sync_group: epel distribution_name: extra-packages-for-enterprise-linux-modular-8-x86_64- # Third-party repositories @@ -322,6 +359,7 @@ rpm_package_repos: url: https://download.docker.com/linux/centos/8/x86_64/stable base_path: docker-ce/centos/8/x86_64/stable/ short_name: docker + sync_group: docker distribution_name: docker-ce-for-centos-8- # Note(piotrp): With repository size of 39G - for now we'll want this on_demand - name: ELK repository for 7.x packages @@ -329,6 +367,7 @@ rpm_package_repos: policy: on_demand base_path: elasticsearch/oss-7.x/ short_name: elasticsearch_logstash_kibana_7_x + sync_group: third_party distribution_name: elasticsearch-logstash-kibana-7.x- # Note(piotrp): With repository size of 16G - for now we'll want this on_demand - name: Grafana @@ -336,16 +375,19 @@ rpm_package_repos: policy: on_demand base_path: grafana/oss/rpm/ short_name: grafana + sync_group: third_party distribution_name: grafana- - name: MariaDB 10.5 url: https://dlm.mariadb.com/repo/mariadb-server/10.5/yum/rhel/8/x86_64 base_path: mariadb-10.5/yum/centos8-amd64/ short_name: mariadb_10_5_centos8 + sync_group: third_party distribution_name: mariadb-10.5-centos8 - name: MariaDB 10.6 url: https://dlm.mariadb.com/repo/mariadb-server/10.6/yum/rhel/8/x86_64 base_path: mariadb-10.6/yum/centos8-amd64/ short_name: mariadb_10_6_centos8 + sync_group: third_party distribution_name: mariadb-10.6-centos8 # RabbitMQ - Erlang for Redhat family, version 8 - name: RabbitMQ - Erlang @@ -355,6 +397,7 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: rabbitmq/erlang/el/8/x86_64/ short_name: rabbitmq_erlang + sync_group: third_party distribution_name: rabbitmq-erlang- # RabbitMQ for Redhat family, version 8 - name: RabbitMQ - Server @@ -364,11 +407,13 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: rabbitmq/rabbitmq-server/el/8/x86_64/ short_name: rabbitmq_server + sync_group: third_party distribution_name: rabbitmq-server- - name: TreasureData 4 url: http://packages.treasuredata.com/4/redhat/8/x86_64 base_path: treasuredata/4/redhat/8/x86_64/ short_name: treasuredata_4 + sync_group: third_party distribution_name: treasuredata-4- # Base Rocky Linux 8.5 repositories @@ -376,18 +421,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.5&arch=x86_64&country=NL base_path: rocky/8.5/AppStream/x86_64/os/ short_name: rocky_8_5_appstream + sync_group: rocky_8 distribution_name: rocky-8.5-appstream- sync: false - name: Rocky Linux 8.5 - BaseOS url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.5&arch=x86_64&country=NL base_path: rocky/8.5/BaseOS/x86_64/os/ short_name: rocky_8_5_baseos + sync_group: rocky_8 distribution_name: rocky-8.5-baseos- sync: false - name: Rocky Linux 8.5 - Extras url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-extras-8.5&arch=x86_64&country=NL base_path: rocky/8.5/extras/x86_64/os/ short_name: rocky_8_5_extras + sync_group: rocky_8 distribution_name: rocky-8.5-extras- sync: false @@ -397,12 +445,14 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-nfv-8.5&arch=x86_64&country=NL base_path: rocky/8.5/nfv/x86_64/os/ short_name: rocky_8_5_nfv + sync_group: rocky_8 distribution_name: rocky-8.5-nfv- sync: false - name: Rocky Linux 8.5 - PowerTools url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-PowerTools-8.5&arch=x86_64&country=NL base_path: rocky/8.5/PowerTools/x86_64/os/ short_name: rocky_8_5_powertools + sync_group: rocky_8 distribution_name: rocky-8.5-powertools- sync: false @@ -411,18 +461,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.6&arch=x86_64&country=NL base_path: rocky/8.6/AppStream/x86_64/os/ short_name: rocky_8_6_appstream + sync_group: rocky_8 distribution_name: rocky-8.6-appstream- sync: false - name: Rocky Linux 8.6 - BaseOS url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.6&arch=x86_64&country=NL base_path: rocky/8.6/BaseOS/x86_64/os/ short_name: rocky_8_6_baseos + sync_group: rocky_8 distribution_name: rocky-8.6-baseos- sync: false - name: Rocky Linux 8.6 - Extras url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-extras-8.6&arch=x86_64&country=NL base_path: rocky/8.6/extras/x86_64/os/ short_name: rocky_8_6_extras + sync_group: rocky_8 distribution_name: rocky-8.6-extras- sync: false @@ -432,18 +485,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-NFV-8.6&arch=x86_64&country=NL base_path: rocky/8.6/nfv/x86_64/os/ short_name: rocky_8_6_nfv + sync_group: rocky_8 distribution_name: rocky-8.6-nfv- sync: false - name: Rocky Linux 8.6 - PowerTools url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-PowerTools-8.6&arch=x86_64&country=NL base_path: rocky/8.6/PowerTools/x86_64/os/ short_name: rocky_8_6_powertools + sync_group: rocky_8 distribution_name: rocky-8.6-powertools- sync: false - name: Rocky Linux 8.6 - HighAvailability url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-8.6&arch=x86_64&country=NL base_path: rocky/8.6/HighAvailability/x86_64/os/ short_name: rocky_8_6_highavailability + sync_group: rocky_8 distribution_name: rocky-8.6-highavailability- sync: false @@ -452,18 +508,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.7&arch=x86_64&country=NL base_path: rocky/8.7/AppStream/x86_64/os/ short_name: rocky_8_7_appstream + sync_group: rocky_8 distribution_name: rocky-8.7-appstream- sync: false - name: Rocky Linux 8.7 - BaseOS url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.7&arch=x86_64&country=NL base_path: rocky/8.7/BaseOS/x86_64/os/ short_name: rocky_8_7_baseos + sync_group: rocky_8 distribution_name: rocky-8.7-baseos- sync: false - name: Rocky Linux 8.7 - Extras url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-extras-8.7&arch=x86_64&country=NL base_path: rocky/8.7/extras/x86_64/os/ short_name: rocky_8_7_extras + sync_group: rocky_8 distribution_name: rocky-8.7-extras- sync: false @@ -473,18 +532,21 @@ rpm_package_repos: url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-NFV-8.7&arch=x86_64&country=NL base_path: rocky/8.7/nfv/x86_64/os/ short_name: rocky_8_7_nfv + sync_group: rocky_8 distribution_name: rocky-8.7-nfv- sync: false - name: Rocky Linux 8.7 - PowerTools url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-PowerTools-8.7&arch=x86_64&country=NL base_path: rocky/8.7/PowerTools/x86_64/os/ short_name: rocky_8_7_powertools + sync_group: rocky_8 distribution_name: rocky-8.7-powertools- sync: false - name: Rocky Linux 8.7 - HighAvailability url: https://mirror.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-8.7&arch=x86_64&country=NL base_path: rocky/8.7/HighAvailability/x86_64/os/ short_name: rocky_8_7_highavailability + sync_group: rocky_8 distribution_name: rocky-8.7-highavailability- sync: false @@ -493,18 +555,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-8.8&arch=x86_64&country=NL base_path: rocky/8.8/AppStream/x86_64/os/ short_name: rocky_8_8_appstream + sync_group: rocky_8 distribution_name: rocky-8.8-appstream- sync: false - name: Rocky Linux 8.8 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-8.8&arch=x86_64&country=NL base_path: rocky/8.8/BaseOS/x86_64/os/ short_name: rocky_8_8_baseos + sync_group: rocky_8 distribution_name: rocky-8.8-baseos- sync: false - name: Rocky Linux 8.8 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-8.8&arch=x86_64&country=NL base_path: rocky/8.8/extras/x86_64/os/ short_name: rocky_8_8_extras + sync_group: rocky_8 distribution_name: rocky-8.8-extras- sync: false @@ -514,18 +579,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-NFV-8.8&arch=x86_64&country=NL base_path: rocky/8.8/nfv/x86_64/os/ short_name: rocky_8_8_nfv + sync_group: rocky_8 distribution_name: rocky-8.8-nfv- sync: false - name: Rocky Linux 8.8 - PowerTools url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-PowerTools-8.8&arch=x86_64&country=NL base_path: rocky/8.8/PowerTools/x86_64/os/ short_name: rocky_8_8_powertools + sync_group: rocky_8 distribution_name: rocky-8.8-powertools- sync: false - name: Rocky Linux 8.8 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-8.8&arch=x86_64&country=NL base_path: rocky/8.8/HighAvailability/x86_64/os/ short_name: rocky_8_8_highavailability + sync_group: rocky_8 distribution_name: rocky-8.8-highavailability- sync: false @@ -534,6 +602,7 @@ rpm_package_repos: url: https://linux.mellanox.com/public/repo/mlnx_ofed/5.6-1.0.3.3/rhel8.6/x86_64/ base_path: mlnx_ofed/5.6-1.0.3.3/rhel8.6/x86_64/ short_name: mlnx_ofed_5_6_1_0_3_3_rhel8_6 + sync_group: third_party distribution_name: mlnx_ofed-5.6-1.0.3.3-rhel8.6- sync: false @@ -542,6 +611,7 @@ rpm_package_repos: url: https://linux.mellanox.com/public/repo/mlnx_ofed/5.7-1.0.2.0/rhel8.6/x86_64/ base_path: mlnx_ofed/5.7-1.0.2.0/rhel8.6/x86_64/ short_name: mlnx_ofed_5_7_1_0_2_0_rhel8_6 + sync_group: third_party distribution_name: mlnx_ofed-5.7-1.0.2.0-rhel8.6- sync: false @@ -550,18 +620,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-9.1&arch=x86_64&country=NL base_path: rocky/9.1/AppStream/x86_64/os/ short_name: rocky_9_1_appstream + sync_group: rocky_9 distribution_name: rocky-9.1-appstream- sync: false - name: Rocky Linux 9.1 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-9.1&arch=x86_64&country=NL base_path: rocky/9.1/BaseOS/x86_64/os/ short_name: rocky_9_1_baseos + sync_group: rocky_9 distribution_name: rocky-9.1-baseos- sync: false - name: Rocky Linux 9.1 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-9.1&arch=x86_64&country=NL base_path: rocky/9.1/extras/x86_64/os/ short_name: rocky_9_1_extras + sync_group: rocky_9 distribution_name: rocky-9.1-extras- sync: false @@ -571,12 +644,14 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-CRB-9.1&arch=x86_64&country=NL base_path: rocky/9.1/CRB/x86_64/os/ short_name: rocky_9_1_crb + sync_group: rocky_9 distribution_name: rocky-9.1-crb- sync: false - name: Rocky Linux 9.1 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-9.1&arch=x86_64&country=NL base_path: rocky/9.1/highavailability/x86_64/os/ short_name: rocky_9_1_highavailability + sync_group: rocky_9 distribution_name: rocky-9.1-highavailability- sync: false @@ -585,18 +660,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-9.2&arch=x86_64&country=NL base_path: rocky/9.2/AppStream/x86_64/os/ short_name: rocky_9_2_appstream + sync_group: rocky_9 distribution_name: rocky-9.2-appstream- sync: false - name: Rocky Linux 9.2 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-9.2&arch=x86_64&country=NL base_path: rocky/9.2/BaseOS/x86_64/os/ short_name: rocky_9_2_baseos + sync_group: rocky_9 distribution_name: rocky-9.2-baseos- sync: false - name: Rocky Linux 9.2 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-9.2&arch=x86_64&country=NL base_path: rocky/9.2/extras/x86_64/os/ short_name: rocky_9_2_extras + sync_group: rocky_9 distribution_name: rocky-9.2-extras- sync: false @@ -606,12 +684,14 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-CRB-9.2&arch=x86_64&country=NL base_path: rocky/9.2/CRB/x86_64/os/ short_name: rocky_9_2_crb + sync_group: rocky_9 distribution_name: rocky-9.2-crb- sync: false - name: Rocky Linux 9.2 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-9.2&arch=x86_64&country=NL base_path: rocky/9.2/highavailability/x86_64/os/ short_name: rocky_9_2_highavailability + sync_group: rocky_9 distribution_name: rocky-9.2-highavailability- sync: false @@ -620,18 +700,21 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-9.3&arch=x86_64&country=NL base_path: rocky/9.3/AppStream/x86_64/os/ short_name: rocky_9_3_appstream + sync_group: rocky_9 distribution_name: rocky-9.3-appstream- sync: false - name: Rocky Linux 9.3 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-9.3&arch=x86_64&country=NL base_path: rocky/9.3/BaseOS/x86_64/os/ short_name: rocky_9_3_baseos + sync_group: rocky_9 distribution_name: rocky-9.3-baseos- sync: false - name: Rocky Linux 9.3 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-9.3&arch=x86_64&country=NL base_path: rocky/9.3/extras/x86_64/os/ short_name: rocky_9_3_extras + sync_group: rocky_9 distribution_name: rocky-9.3-extras- sync: false @@ -641,12 +724,14 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-CRB-9.3&arch=x86_64&country=NL base_path: rocky/9.3/CRB/x86_64/os/ short_name: rocky_9_3_crb + sync_group: rocky_9 distribution_name: rocky-9.3-crb- sync: false - name: Rocky Linux 9.3 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-9.3&arch=x86_64&country=NL base_path: rocky/9.3/highavailability/x86_64/os/ short_name: rocky_9_3_highavailability + sync_group: rocky_9 distribution_name: rocky-9.3-highavailability- sync: false @@ -655,16 +740,19 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-AppStream-9.4&arch=x86_64&country=NL base_path: rocky/9.4/AppStream/x86_64/os/ short_name: rocky_9_4_appstream + sync_group: rocky_9 distribution_name: rocky-9.4-appstream- - name: Rocky Linux 9.4 - BaseOS url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-BaseOS-9.4&arch=x86_64&country=NL base_path: rocky/9.4/BaseOS/x86_64/os/ short_name: rocky_9_4_baseos + sync_group: rocky_9_baseos distribution_name: rocky-9.4-baseos- - name: Rocky Linux 9.4 - Extras url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-extras-9.4&arch=x86_64&country=NL base_path: rocky/9.4/extras/x86_64/os/ short_name: rocky_9_4_extras + sync_group: rocky_9 distribution_name: rocky-9.4-extras- # Additional Rocky Linux 9.4 repositories @@ -673,16 +761,19 @@ rpm_package_repos: url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-CRB-9.4&arch=x86_64&country=NL base_path: rocky/9.4/CRB/x86_64/os/ short_name: rocky_9_4_crb + sync_group: rocky_9 distribution_name: rocky-9.4-crb- - name: Rocky Linux 9.4 - HighAvailability url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-HighAvailability-9.4&arch=x86_64&country=NL base_path: rocky/9.4/highavailability/x86_64/os/ short_name: rocky_9_4_highavailability + sync_group: rocky_9 distribution_name: rocky-9.4-highavailability- - name: Rocky Linux 9 - SIG Security Common url: https://mirrors.rockylinux.org/mirrorlist?repo=rocky-sig-security-common-9&arch=x86_64&country=NL base_path: rocky/sig/9/security/x86_64/security-common/ short_name: rocky_9_sig_security_common + sync_group: rocky_9 distribution_name: rocky-9-sig-security-common- # Additional CentOS Stream 9 repositories @@ -691,30 +782,35 @@ rpm_package_repos: url: https://mirror.stream.centos.org/SIGs/9-stream/nfv/x86_64/openvswitch-2/ base_path: centos/9-stream/nfv/x86_64/openvswitch-2/ short_name: centos_stream_9_nfv_openvswitch + sync_group: centos distribution_name: centos-stream-9-nfv-openvswitch- # Opstools for CentOS Stream 9 - name: CentOS Stream 9 - opstools url: https://mirror.stream.centos.org/SIGs/9-stream/opstools/x86_64/collectd-5/ base_path: centos/9-stream/opstools/x86_64/collectd-5/ short_name: centos_stream_9_opstools + sync_group: centos distribution_name: centos-stream-9-opstools- # Ceph Pacific for CentOS Stream 9 - name: CentOS Stream 9 - Ceph Pacific url: https://mirror.stream.centos.org/SIGs/9-stream/storage/x86_64/ceph-pacific/ base_path: centos/9-stream/storage/x86_64/ceph-pacific/ short_name: centos_stream_9_storage_ceph_pacific + sync_group: centos distribution_name: centos-stream-9-storage-ceph-pacific- # Ceph Quincy for CentOS Stream 9 - name: CentOS Stream 9 - Ceph Quincy url: https://mirror.stream.centos.org/SIGs/9-stream/storage/x86_64/ceph-quincy/ base_path: centos/9-stream/storage/x86_64/ceph-quincy/ short_name: centos_stream_9_storage_ceph_quincy + sync_group: centos distribution_name: centos-stream-9-storage-ceph-quincy- # Ceph Reef for CentOS Stream 9 - name: CentOS Stream 9 - Ceph Reef url: https://mirror.stream.centos.org/SIGs/9-stream/storage/x86_64/ceph-reef/ base_path: centos/9-stream/storage/x86_64/ceph-reef/ short_name: centos_stream_9_storage_ceph_reef + sync_group: centos distribution_name: centos-stream-9-storage-ceph-reef- # EPEL 9 repository - name: Extra Packages for Enterprise Linux 9 @@ -722,6 +818,7 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: epel/9/Everything/x86_64/ short_name: epel_9 + sync_group: epel distribution_name: extra-packages-for-enterprise-linux-9-x86_64- # ELRepo 9 repository - name: ELRepo.org Community Enterprise Linux Repository - el9 @@ -729,36 +826,42 @@ rpm_package_repos: url: https://lon.mirror.rackspace.com/elrepo/elrepo/el9/x86_64/ base_path: elrepo/elrepo/el9/x86_64/ short_name: elrepo_9 + sync_group: epel distribution_name: elrepo-el9-x86_64- # Docker CE for CentOS Stream 9 - name: Docker CE - CentOS Stream 9 url: https://download.docker.com/linux/centos/9/x86_64/stable base_path: docker-ce/centos/9/x86_64/stable/ short_name: centos_stream_9_docker + sync_group: docker distribution_name: docker-ce-for-centos-9- # Openstack Yoga for CentOS Stream 9 - name: CentOS Stream 9 - OpenStack Yoga url: https://mirror.stream.centos.org/SIGs/9-stream/cloud/x86_64/openstack-yoga/ base_path: centos/9-stream/cloud/x86_64/openstack-yoga/ short_name: centos_stream_9_openstack_yoga + sync_group: centos distribution_name: centos-stream-9-openstack-yoga- # Openstack Zed for CentOS Stream 9 - name: CentOS Stream 9 - OpenStack Zed url: https://mirror.stream.centos.org/SIGs/9-stream/cloud/x86_64/openstack-zed/ base_path: centos/9-stream/cloud/x86_64/openstack-zed/ short_name: centos_stream_9_openstack_zed + sync_group: centos distribution_name: centos-stream-9-openstack-zed- # OpenStack Antelope for CentOS Stream 9 - name: CentOS Stream 9 - OpenStack Antelope url: https://mirror.stream.centos.org/SIGs/9-stream/cloud/x86_64/openstack-antelope/ base_path: centos/9-stream/cloud/x86_64/openstack-antelope/ short_name: centos_stream_9_openstack_antelope + sync_group: centos distribution_name: centos-stream-9-openstack-antelope- # OpenStack Caracal for CentOS Stream 9 - name: CentOS Stream 9 - OpenStack Caracal url: https://mirror.stream.centos.org/SIGs/9-stream/cloud/x86_64/openstack-caracal/ base_path: centos/9-stream/cloud/x86_64/openstack-caracal/ short_name: centos_stream_9_openstack_caracal + sync_group: centos distribution_name: centos-stream-9-openstack-caracal- # Additional RHEL 9 repositories @@ -770,6 +873,7 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: rabbitmq/erlang/el/9/x86_64/ short_name: rhel9_rabbitmq_erlang + sync_group: third_party distribution_name: rhel9-rabbitmq-erlang- # RabbitMQ for Redhat family, version 9 - name: RabbitMQ - Server - RHEL 9 @@ -779,64 +883,74 @@ rpm_package_repos: sync_policy: mirror_content_only base_path: rabbitmq/rabbitmq-server/el/9/x86_64/ short_name: rhel9_rabbitmq_server + sync_group: third_party distribution_name: rhel9-rabbitmq-server- # TreasureData 4 for RHEL 9 - name: TreasureData 4 - RHEL 9 url: http://packages.treasuredata.com/4/redhat/9/x86_64 base_path: treasuredata/4/redhat/9/x86_64/ short_name: rhel_9_treasuredata_4 + sync_group: third_party distribution_name: rhel_9_treasuredata-4- # TreasureData 5 for RHEL 9 - name: TreasureData 5 - RHEL 9 url: https://packages.treasuredata.com/lts/5/redhat/9/x86_64 base_path: treasuredata/lts/5/redhat/9/x86_64/ short_name: rhel_9_treasuredata_5 + sync_group: third_party distribution_name: rhel_9_treasuredata-5- # MariaDB 10.6 for RHEL 9 - name: MariaDB 10.6 - RHEL 9 url: https://dlm.mariadb.com/repo/mariadb-server/10.6/yum/rhel/9/x86_64 base_path: mariadb-10.6/yum/rhel/9/x86_64/ short_name: rhel_9_mariadb_10_6 + sync_group: third_party distribution_name: mariadb-10.6-rhel-9- # MariaDB 10.11 for RHEL 9 - name: MariaDB 10.11 - RHEL 9 url: https://dlm.mariadb.com/repo/mariadb-server/10.11/yum/rhel/9/x86_64 base_path: mariadb-10.11/yum/rhel/9/x86_64/ short_name: rhel_9_mariadb_10_11 + sync_group: third_party distribution_name: mariadb-10.11-rhel-9- # InfluxDB for RHEL 9 - name: InfluxDB - RHEL 9 url: https://repos.influxdata.com/rhel/9/x86_64/stable/ base_path: influxdb/rhel/9/x86_64/stable/ short_name: rhel_9_influxdb + sync_group: third_party distribution_name: influxdb-rhel-9- # MLNX OFED 5.9-0.5.6.0 RHEL 9.1 repository - name: Mellanox Technologies mlnx_ofed 5.9-0.5.6.0 url: https://linux.mellanox.com/public/repo/mlnx_ofed/5.9-0.5.6.0/rhel9.1/x86_64/ base_path: mlnx_ofed/5.9-0.5.6.0/rhel9.1/x86_64/ short_name: mlnx_ofed_5_9_0_5_6_0_rhel9_1 + sync_group: third_party distribution_name: mlnx_ofed-5.9-0.5.6.0-rhel9.1- # MLNX OFED 23.07-0.5.1.2 RHEL 9.2 repository - name: Mellanox Technologies mlnx_ofed 23.07-0.5.1.2 url: https://linux.mellanox.com/public/repo/mlnx_ofed/23.07-0.5.1.2/rhel9.2/x86_64/ base_path: mlnx_ofed/23.07-0.5.1.2/rhel9.2/x86_64/ short_name: mlnx_ofed_23_07_0_5_1_2_rhel9_2 + sync_group: third_party distribution_name: mlnx_ofed-23.07-0.5.1.2-rhel9.2- # OpenSearch 2.x - name: OpenSearch 2.x url: https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/yum base_path: opensearch/2.x/yum/ short_name: opensearch_2_x + sync_group: third_party distribution_name: opensearch-2.x-yum- # OpenSearch dashboards - name: OpenSearch Dashboards 2.x url: https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/yum base_path: opensearch-dashboards/2.x/yum/ short_name: opensearch_dashboards_2_x + sync_group: third_party distribution_name: opensearch_dashboards-2.x-yum- # Default filter string for RPM package repositories. rpm_package_repo_filter: "" # List of package repositories after applying filter. -rpm_package_repos_filtered: "{{ rpm_package_repos | select_repos(rpm_package_repo_filter) }}" +rpm_package_repos_filtered: "{{ rpm_package_repos | select_repos(rpm_package_repo_filter, package_sync_group) }}" diff --git a/ansible/validate-deb-repos.yml b/ansible/validate-deb-repos.yml index 88d5924c..f66eaedc 100644 --- a/ansible/validate-deb-repos.yml +++ b/ansible/validate-deb-repos.yml @@ -28,6 +28,11 @@ that: - synced_deb_package_repos | rejectattr('url', 'defined') | list | length == 0 + - name: Assert that synced Deb package repositories have sync groups + assert: + that: + - synced_deb_package_repos | rejectattr('sync_group', 'defined') | list | length == 0 + - name: Assert that published Deb package repositories have short names assert: that: @@ -71,6 +76,24 @@ vars: deb_package_repo_filter: docker_ce_ubuntu_jammy ubuntu_jammy_security$ + - name: Assert that Deb package repository list can be filtered to one group + assert: + that: + - deb_package_repos_filtered | length == 2 + - deb_package_repos_filtered[0].short_name == 'docker_ce_ubuntu_focal' + - deb_package_repos_filtered[1].short_name == 'docker_ce_ubuntu_jammy' + vars: + package_sync_group: docker + + - name: Assert that Deb package repository list can be filtered and use a sync group together + assert: + that: + - deb_package_repos_filtered | length == 1 + - deb_package_repos_filtered[0].short_name == 'docker_ce_ubuntu_jammy' + vars: + package_sync_group: docker + deb_package_repo_filter: jammy + - name: Assert that dev package repository list is defined assert: that: diff --git a/ansible/validate-rpm-repos.yml b/ansible/validate-rpm-repos.yml index 77ee40cd..77ce3555 100644 --- a/ansible/validate-rpm-repos.yml +++ b/ansible/validate-rpm-repos.yml @@ -28,6 +28,11 @@ that: - synced_rpm_package_repos | rejectattr('url', 'defined') | list | length == 0 + - name: Assert that synced RPM package repositories have sync groups + assert: + that: + - synced_rpm_package_repos | rejectattr('sync_group', 'defined') | list | length == 0 + - name: Assert that published RPM package repositories have short names assert: that: @@ -72,6 +77,24 @@ vars: rpm_package_repo_filter: docker centos_stream_8_baseos centos_stream_9_docker + - name: Assert that RPM package repository list can be filtered to one group + assert: + that: + - rpm_package_repos_filtered | length == 2 + - rpm_package_repos_filtered[0].short_name == 'docker' + - rpm_package_repos_filtered[1].short_name == 'centos_stream_9_docker' + vars: + package_sync_group: docker + + - name: Assert that RPM package repository list can be filtered and use a sync group together + assert: + that: + - rpm_package_repos_filtered | length == 1 + - rpm_package_repos_filtered[0].short_name == 'centos_stream_9_docker' + vars: + package_sync_group: docker + rpm_package_repo_filter: stream_9 + - name: Assert that dev package repository list is defined assert: that: