diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml index da933c91d..331035001 100644 --- a/.github/workflows/fatimage.yml +++ b/.github/workflows/fatimage.yml @@ -33,6 +33,7 @@ jobs: OS_CLOUD: openstack CI_CLOUD: ${{ github.event.inputs.ci_cloud }} ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }} + LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/nightlybuild.yml b/.github/workflows/nightlybuild.yml index 596b85a05..2485cd2df 100644 --- a/.github/workflows/nightlybuild.yml +++ b/.github/workflows/nightlybuild.yml @@ -35,6 +35,7 @@ jobs: OS_CLOUD: openstack CI_CLOUD: ${{ github.event.inputs.ci_cloud || vars.CI_CLOUD }} ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }} + LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }} steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index f61bf8df4..f66441915 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ It requires an OpenStack cloud, and an Ansible "deploy host" with access to that Before starting ensure that: - You have root access on the deploy host. - You can create instances using a Rocky 9 GenericCloud image (or an image based on that). - - **NB**: In general it is recommended to use the [latest released image](https://github.com/stackhpc/ansible-slurm-appliance/releases) which already contains the required packages. This is built and tested in StackHPC's CI. However the appliance will install the necessary packages if a GenericCloud image is used. + - **NB**: In general it is recommended to use the [latest released image](https://github.com/stackhpc/ansible-slurm-appliance/releases) which already contains the required packages. This is built and tested in StackHPC's CI. - You have a SSH keypair defined in OpenStack, with the private key available on the deploy host. - Created instances have access to internet (note proxies can be setup through the appliance if necessary). - Created instances have accurate/synchronised time (for VM instances this is usually provided by the hypervisor; if not or for bare metal instances it may be necessary to configure a time service via the appliance). diff --git a/ansible/.gitignore b/ansible/.gitignore index 3fef64ecc..6c4f32017 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -66,5 +66,7 @@ roles/* !roles/lustre/** !roles/dnf_repos/ !roles/dnf_repos/** +!roles/pulp_site/ +!roles/pulp_site/** !roles/doca/ !roles/doca/** diff --git a/ansible/adhoc/deploy-pulp.yml b/ansible/adhoc/deploy-pulp.yml new file mode 100644 index 000000000..2858d032b --- /dev/null +++ b/ansible/adhoc/deploy-pulp.yml @@ -0,0 +1,26 @@ +# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=" + +- name: Add temporary pulp server host + hosts: localhost + tasks: + - ansible.builtin.add_host: + name: "{{ pulp_server }}" + group: "_pulp_host" + +- name: Install pulp on server and add to config + become: yes + hosts: _pulp_host + tasks: + - name: Install pulp + ansible.builtin.include_role: + name: pulp_site + tasks_from: install.yml + public: true + + - name: Print Pulp endpoint + become: no + debug: + msg: | + Server configured, override 'appliances_pulp_url' with + appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}" + in your environments diff --git a/ansible/adhoc/sync-pulp.yml b/ansible/adhoc/sync-pulp.yml new file mode 100644 index 000000000..f26149bba --- /dev/null +++ b/ansible/adhoc/sync-pulp.yml @@ -0,0 +1,10 @@ +- hosts: localhost + tasks: + - ansible.builtin.include_role: + name: pulp_site + tasks_from: sync.yml + vars: + pulp_site_target_arch: "x86_64" + pulp_site_target_distribution: "rocky" + pulp_site_target_distribution_version: "9.4" + pulp_site_target_distribution_version_major: "9" diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml index 733d4b3f8..a504f3545 100644 --- a/ansible/bootstrap.yml +++ b/ansible/bootstrap.yml @@ -110,6 +110,20 @@ policy: "{{ selinux_policy }}" register: sestatus +- hosts: dnf_repos + become: yes + tasks: + - name: Check that creds won't be leaked to users + ansible.builtin.assert: + that: dnf_repos_password is undefined + fail_msg: Passwords should not be templated into repofiles during configure, unset 'dnf_repos_password' + when: appliances_mode == 'configure' + - name: Replace system repos with pulp repos + ansible.builtin.include_role: + name: dnf_repos + tasks_from: set_repos.yml + when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided + # --- tasks after here require access to package repos --- - hosts: squid tags: squid diff --git a/ansible/disable-repos.yml b/ansible/disable-repos.yml new file mode 100644 index 000000000..d7dc4fd55 --- /dev/null +++ b/ansible/disable-repos.yml @@ -0,0 +1,8 @@ +- hosts: dnf_repos + become: yes + tasks: + - name: Disable pulp repos + ansible.builtin.include_role: + name: dnf_repos + tasks_from: disable_repos.yml + when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided diff --git a/ansible/fatimage.yml b/ansible/fatimage.yml index 439c50e70..55e56e612 100644 --- a/ansible/fatimage.yml +++ b/ansible/fatimage.yml @@ -17,6 +17,16 @@ import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}" when: hook_path | exists +- name: Sync pulp repos with upstream + hosts: pulp + tasks: + - ansible.builtin.include_role: + name: pulp_site + tasks_from: sync.yml + apply: + delegate_to: localhost + when: appliances_mode != 'configure' + - import_playbook: bootstrap.yml - name: Run post-bootstrap.yml hook @@ -210,6 +220,8 @@ import_role: name: doca +- import_playbook: disable-repos.yml + - name: Run post.yml hook vars: appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}" diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml index a3e05d0e1..4a0c9fd2a 100644 --- a/ansible/roles/dnf_repos/defaults/main.yml +++ b/ansible/roles/dnf_repos/defaults/main.yml @@ -1,25 +1,23 @@ -dnf_repos_rocky_ark_prefix: https://ark.stackhpc.com/pulp/content/{{ ansible_distribution | lower }}/{{ ansible_distribution_version }} -dnf_repos_rocky_ark_suffix: "{{ ansible_architecture }}/os/{{ dnf_repos_rocky_ark_timestamp }}/" -# most stable from https://github.com/stackhpc/stackhpc-kayobe-config/blob/stackhpc/2024.1/etc/kayobe/pulp-repo-versions.yml -# note that some timestamps can't be used because not all repos have snapshots for them -dnf_repos_rocky_ark_timestamp: 20240816T002610 -dnf_repos_username: slurm-app-ci -dnf_repos_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}" +dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content" +dnf_repos_rocky_prefix: "{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}" +dnf_repos_epel_prefix: "epel/{{ ansible_distribution_major_version }}" +dnf_repos_username: "{{ omit }}" +dnf_repos_password: "{{ omit }}" # epel installed separately dnf_repos_repolist: - file: rocky name: baseos - base_url: "{{ dnf_repos_rocky_ark_prefix }}/BaseOS/{{ dnf_repos_rocky_ark_suffix }}" + base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/BaseOS/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.baseos[ansible_distribution_version] }}" - file: rocky name: appstream - base_url: "{{ dnf_repos_rocky_ark_prefix }}/AppStream/{{ dnf_repos_rocky_ark_suffix }}" + base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/AppStream/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.appstream[ansible_distribution_version] }}" - file: rocky name: crb - base_url: "{{ dnf_repos_rocky_ark_prefix }}/CRB/{{ dnf_repos_rocky_ark_suffix }}" + base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/CRB/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.crb[ansible_distribution_version] }}" - file: rocky-extras name: extras - base_url: "{{ dnf_repos_rocky_ark_prefix }}/extras/{{ dnf_repos_rocky_ark_suffix }}" + base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/extras/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.extras[ansible_distribution_version] }}" -dnf_repos_epel_timestamp: 20240902T080424 -dnf_repos_epel_baseurl: "https://ark.stackhpc.com/pulp/content/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ dnf_repos_epel_timestamp }}" +dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ appliances_repo_timestamps.epel[ansible_distribution_major_version] }}" +dnf_repos_epel_description: "epel" diff --git a/ansible/roles/dnf_repos/tasks/disable_repos.yml b/ansible/roles/dnf_repos/tasks/disable_repos.yml index f8997b741..2dbacc262 100644 --- a/ansible/roles/dnf_repos/tasks/disable_repos.yml +++ b/ansible/roles/dnf_repos/tasks/disable_repos.yml @@ -1,5 +1,5 @@ --- -- name: Disable Pulp repos and remove creds +- name: Disable Pulp repos ansible.builtin.yum_repository: file: "{{ item.file }}" name: "{{ item.name }}" @@ -8,11 +8,11 @@ enabled: false loop: "{{ dnf_repos_repolist }}" -- name: Disable EPEL repo and remove creds +- name: Disable EPEL repo ansible.builtin.yum_repository: name: epel file: epel - description: epel + description: "{{ dnf_repos_epel_description }}" baseurl: "{{ dnf_repos_epel_baseurl }}" gpgcheck: false enabled: false diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index f8cca5600..fe5e2c02c 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -19,8 +19,8 @@ ansible.builtin.yum_repository: name: epel file: epel - description: epel + description: "{{ dnf_repos_epel_description }}" gpgcheck: false + baseurl: "{{ dnf_repos_epel_baseurl }}" username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" - baseurl: "{{ dnf_repos_epel_baseurl }}" diff --git a/ansible/roles/passwords/defaults/main.yml b/ansible/roles/passwords/defaults/main.yml index d9a339efd..2587e8499 100644 --- a/ansible/roles/passwords/defaults/main.yml +++ b/ansible/roles/passwords/defaults/main.yml @@ -9,6 +9,7 @@ slurm_appliance_secrets: vault_freeipa_ds_password: "{{ vault_freeipa_ds_password | default(lookup('password', '/dev/null')) }}" vault_freeipa_admin_password: "{{ vault_freeipa_admin_password | default(lookup('password', '/dev/null')) }}" vault_k3s_token: "{{ vault_k3s_token | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}" + vault_pulp_admin_password: "{{ vault_pulp_admin_password | default(lookup('password', '/dev/null', chars=['ascii_letters', 'digits'])) }}" secrets_openhpc_mungekey_default: content: "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') }}" diff --git a/ansible/roles/pulp_site/.gitignore b/ansible/roles/pulp_site/.gitignore new file mode 100644 index 000000000..6738e49c1 --- /dev/null +++ b/ansible/roles/pulp_site/.gitignore @@ -0,0 +1 @@ +filter_plugins/__pycache__ \ No newline at end of file diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml new file mode 100644 index 000000000..d343d4998 --- /dev/null +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -0,0 +1,39 @@ +pulp_site_url: "{{ appliances_pulp_url }}" +pulp_site_port: 8080 +pulp_site_username: admin # shouldn't be changed +pulp_site_password: "{{ vault_pulp_admin_password }}" +pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content +_pulp_site_rocky_prefix: "{{ pulp_site_target_distribution }}/{{ pulp_site_target_distribution_version }}" +pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os" +pulp_site_validate_certs: false +pulp_site_install_dir: '/home/rocky/pulp' +pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}" +pulp_site_target_facts: "{{ hostvars[groups['builder'][0]]['ansible_facts'] }}" +pulp_site_target_arch: "{{ pulp_site_target_facts['architecture'] }}" +pulp_site_target_distribution: "{{ pulp_site_target_facts['distribution'] | lower }}" +pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}" +pulp_site_target_distribution_version_major: "{{ pulp_site_target_facts['distribution_major_version'] }}" + +pulp_site_rpm_info: +- name: "baseos-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.baseos[pulp_site_target_distribution_version] }}" + subpath: "{{ _pulp_site_rocky_prefix }}/BaseOS/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.baseos[pulp_site_target_distribution_version] }}" +- name: "appstream-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.appstream[pulp_site_target_distribution_version] }}" + subpath: "{{ _pulp_site_rocky_prefix }}/AppStream/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.appstream[pulp_site_target_distribution_version] }}" +- name: "crb-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.crb[pulp_site_target_distribution_version] }}" + subpath: "{{ _pulp_site_rocky_prefix }}/{{ 'PowerTools' if pulp_site_target_distribution_version_major == '8' else 'CRB' }}/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.crb[pulp_site_target_distribution_version] }}" +- name: "extras-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.extras[pulp_site_target_distribution_version] }}" + subpath: "{{ _pulp_site_rocky_prefix }}/extras/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.extras[pulp_site_target_distribution_version] }}" +- name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ appliances_repo_timestamps.epel[pulp_site_target_distribution_version_major] }}" + subpath: "epel/{{ pulp_site_target_distribution_version_major }}/Everything/{{ pulp_site_target_arch }}/{{ appliances_repo_timestamps.epel[pulp_site_target_distribution_version_major] }}" + +pulp_site_rpm_repo_defaults: + remote_username: "{{ pulp_site_upstream_username }}" + remote_password: "{{ pulp_site_upstream_password }}" + policy: on_demand + state: present + +_pulp_site_rpm_info_all: "{{ pulp_site_rpm_info | map('combine', pulp_site_rpm_repo_defaults) }}" + +pulp_site_rpm_repos: "{{ _pulp_site_rpm_info_all | to_rpm_repos(pulp_site_upstream_content_url) }}" +pulp_site_rpm_publications: "{{ _pulp_site_rpm_info_all | to_rpm_pubs }}" +pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info_all | to_rpm_distros }}" diff --git a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py new file mode 100644 index 000000000..50e912685 --- /dev/null +++ b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py @@ -0,0 +1,31 @@ +class FilterModule(object): + def filters(self): + return { + 'to_rpm_repos': self.to_rpm_repos, + 'to_rpm_pubs': self.to_rpm_pubs, + 'to_rpm_distros': self.to_rpm_distros + } + + def to_rpm_repos(self, list, pulp_url): + repo_list = map(lambda x: { + 'name': x['name'], + 'url': pulp_url+'/'+x['subpath'], + 'remote_username': x['remote_username'], + 'remote_password': x['remote_password'], + 'policy': x['policy'], + 'state': x['state'] }, list) + return repo_list + + def to_rpm_pubs(self, list): + pub_list = map(lambda x: { + 'repository': x['name'], + 'state': x['state'] }, list) + return pub_list + + def to_rpm_distros(self, list): + distro_list = map(lambda x: { + 'name': x['name'], + 'repository': x['name'], + 'base_path': x['subpath'], + 'state': x['state'] }, list) + return distro_list \ No newline at end of file diff --git a/ansible/roles/pulp_site/tasks/install.yml b/ansible/roles/pulp_site/tasks/install.yml new file mode 100644 index 000000000..39b4fcd97 --- /dev/null +++ b/ansible/roles/pulp_site/tasks/install.yml @@ -0,0 +1,43 @@ +--- + +- name: Install packages + dnf: + name: + - podman + +- name: Create install directories + ansible.builtin.file: + state: directory + path: "{{ pulp_site_install_dir }}/{{ item }}" + loop: + - settings/certs + - pulp_storage + - pgsql + - containers + +- name: Template settings file + ansible.builtin.template: + src: settings.py.j2 + dest: "{{ pulp_site_install_dir }}/settings/settings.py" + +- name: Install pulp podman container + containers.podman.podman_container: + name: pulp + publish: + - "{{ pulp_site_port }}:80" + volume: + - "{{ pulp_site_install_dir }}/settings:/etc/pulp{{ pulp_site_selinux_suffix }}" + - "{{ pulp_site_install_dir }}/pulp_storage:/var/lib/pulp{{ pulp_site_selinux_suffix }}" + - "{{ pulp_site_install_dir }}/pgsql:/var/lib/pgsql{{ pulp_site_selinux_suffix }}" + - "{{ pulp_site_install_dir }}/containers:/var/lib/containers{{ pulp_site_selinux_suffix }}" + device: /dev/fuse + image: docker.io/pulp/pulp:3.68.1 + +- name: Reset admin password once container has initialised + no_log: true + ansible.builtin.shell: + cmd: "podman exec pulp bash -c 'pulpcore-manager reset-admin-password -p {{ pulp_site_password }}'" + register: _admin_reset_output + until: 0 == _admin_reset_output.rc + retries: 6 + delay: 30 diff --git a/ansible/roles/pulp_site/tasks/sync.yml b/ansible/roles/pulp_site/tasks/sync.yml new file mode 100644 index 000000000..5ef2bc5f1 --- /dev/null +++ b/ansible/roles/pulp_site/tasks/sync.yml @@ -0,0 +1,78 @@ +--- + +- ansible.builtin.assert: + that: pulp_site_upstream_password != '' + quiet: true + fail_msg: "Upstream password not set. Either set env var ARK_PASSWORD or override pulp_site_upstream_password." + +- name: Wait for Pulp server + pulp.squeezer.status: + pulp_url: "{{ pulp_site_url }}" + username: "{{ pulp_site_username }}" + password: "{{ pulp_site_password }}" + register: _pulp_status + until: _pulp_status.failed == false + retries: 30 + delay: 20 + +- name: Ensure Pulp CLI config directory exists + ansible.builtin.file: + path: ~/.config/pulp + state: directory + +- name: Create config file + no_log: true + ansible.builtin.template: + src: cli.toml.j2 + dest: ~/.config/pulp/cli.toml + mode: '0644' + +- block: + - name: Ensure squeezer cache exists + ansible.builtin.file: + path: "{{ _cache_dir }}" + state: directory + + - name: Check if squeezer cache is populated + ansible.builtin.stat: + path: "{{ _cache_dir }}/api.json" + register: _cache_stat + + - name: Prepopulate squeezer cache # workaround for race on the cache + ansible.builtin.get_url: + url: "{{ pulp_site_url }}/pulp/api/v3/docs/api.json" + dest: "{{ _cache_dir }}/api.json" + timeout: 40 + when: not _cache_stat.stat.exists + vars: + _cache_dir: "~/.cache/squeezer/{{ pulp_site_url | regex_replace( ':|/' , '_' ) }}" + +- name: Get Pulp repos from release train + ansible.builtin.include_role: + name: stackhpc.pulp.pulp_repository + tasks_from: rpm.yml + vars: + pulp_url: "{{ pulp_site_url }}" + pulp_username: "{{ pulp_site_username }}" + pulp_password: "{{ pulp_site_password }}" + pulp_repository_rpm_repos: "{{ pulp_site_rpm_repos }}" + +- name: Create Pulp publications + ansible.builtin.include_role: + name: stackhpc.pulp.pulp_publication + tasks_from: rpm.yml + vars: + pulp_url: "{{ pulp_site_url }}" + pulp_username: "{{ pulp_site_username }}" + pulp_password: "{{ pulp_site_password }}" + pulp_publication_rpm: "{{ pulp_site_rpm_publications }}" + +- name: Create Pulp distributions + ansible.builtin.include_role: + name: stackhpc.pulp.pulp_distribution + tasks_from: rpm.yml + vars: + pulp_url: "{{ pulp_site_url }}" + pulp_username: "{{ pulp_site_username }}" + pulp_password: "{{ pulp_site_password }}" + pulp_distribution_rpm: "{{ pulp_site_rpm_distributions }}" diff --git a/ansible/roles/pulp_site/templates/cli.toml.j2 b/ansible/roles/pulp_site/templates/cli.toml.j2 new file mode 100644 index 000000000..06867902f --- /dev/null +++ b/ansible/roles/pulp_site/templates/cli.toml.j2 @@ -0,0 +1,14 @@ +[cli] +base_url = "{{ pulp_site_url }}" +username = "{{ pulp_site_username }}" +password = "{{ pulp_site_password }}" +api_root = "/pulp/" +domain = "default" +headers = [] +cert = "" +key = "" +verify_ssl = true +format = "json" +dry_run = false +timeout = 0 +verbose = 0 diff --git a/ansible/roles/pulp_site/templates/settings.py.j2 b/ansible/roles/pulp_site/templates/settings.py.j2 new file mode 100644 index 000000000..200212e2c --- /dev/null +++ b/ansible/roles/pulp_site/templates/settings.py.j2 @@ -0,0 +1,2 @@ +CONTENT_ORIGIN='http://{{ ansible_fqdn }}:{{ pulp_site_port }}' +TOKEN_AUTH_DISABLED=True diff --git a/ansible/site.yml b/ansible/site.yml index bb379399d..d973d9cb3 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -27,6 +27,7 @@ - import_playbook: slurm.yml - import_playbook: portal.yml - import_playbook: monitoring.yml +- import_playbook: disable-repos.yml - name: Run post.yml hook vars: diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md new file mode 100644 index 000000000..6d30bec6b --- /dev/null +++ b/docs/experimental/pulp.md @@ -0,0 +1,17 @@ +# Pulp Server + +In order to ensure reproducible builds, the appliance can build images using repository mirrors from StackHPC's "Ark" Pulp server. The appliance can sync relevant repositories to a local Pulp server which will then be used instead of Ark. + +## Deploying/configuring Pulp Server + +### Deploying a Pulp server +A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. This can be run with +`ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server="` +where `target_host` is any resolvable host. This will print a Pulp URL which can be copied to your environments as appropriate. Ensure that the server is accessible on the specified port. Note access to this server's content isn't authenticated so assumes the server is deployed behind a secure network. + +### Using an existing Pulp server +An existing Pulp server can be used to host Ark repos by overriding `pulp_site_password` and `appliances_pulp_url` in the target environment. Note that this assumes the same configuration as the appliance deployed pulp i.e no content authentication. + +## Syncing Pulp content with Ark + +If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp. Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for Rocky 9.4 with x86_64 architecture, but can be overriden by setting extravars for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version_major`. diff --git a/docs/image-build.md b/docs/image-build.md index a7d2e951b..db51265a3 100644 --- a/docs/image-build.md +++ b/docs/image-build.md @@ -17,7 +17,8 @@ The fat images StackHPC builds and tests in CI are available from [GitHub releas To build either a site-specific fat image from scratch, or to extend an existing StackHPC fat image: 1. Ensure the current OpenStack credentials have sufficient authorisation to upload images (this may or may not require the `member` role for an application credential, depending on your OpenStack configuration). -2. Create a Packer [variable definition file](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/variables#assigning-values-to-input-variables) at e.g. `environments//builder.pkrvars.hcl` containing at a minimum: +2. The provided dev credentials for StackHPC's "Ark" Pulp server must be added to the target environments. This is done by overriding `dnf_repos_username` and `dnf_repos_password` with your vault encrypted credentials in `environments//inventory/group_vars/all/pulp.yml`. See the [experimental docs](experimental/pulp.md) if you wish instead wish to use a local Pulp server. +3. Create a Packer [variable definition file](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/variables#assigning-values-to-input-variables) at e.g. `environments//builder.pkrvars.hcl` containing at a minimum: ```hcl flavor = "general.v1.small" # VM flavor to use for builder VMs @@ -35,9 +36,9 @@ To build either a site-specific fat image from scratch, or to extend an existing - `update,control,login,compute`: The resultant image has all packages in the source image updated, and then packages for all types of nodes in the cluster are added. When using a GenericCloud image for `source_image_name` this builds a site-specific fat image from scratch. - One or more specific groups which are not enabled in the appliance by default, e.g. `lustre`. When using a StackHPC fat image for `source_image_name` this extends the image with just this additional functionality. -3. Activate the venv and the relevant environment. +4. Activate the venv and the relevant environment. -4. Build images using the relevant variable definition file, e.g.: +5. Build images using the relevant variable definition file, e.g.: cd packer/ PACKER_LOG=1 /usr/bin/packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl @@ -52,7 +53,7 @@ To build either a site-specific fat image from scratch, or to extend an existing then delete the failed volume, select cancelling the build when Packer queries, and then retry. This is [Openstack bug 1823445](https://bugs.launchpad.net/cinder/+bug/1823445). -5. The built image will be automatically uploaded to OpenStack with a name prefixed `openhpc` and including a timestamp and a shortened git hash. +6. The built image will be automatically uploaded to OpenStack with a name prefixed `openhpc` and including a timestamp and a shortened git hash. # Build Process diff --git a/environments/.stackhpc/hooks/post.yml b/environments/.stackhpc/hooks/post.yml index 98e366304..9d506d725 100644 --- a/environments/.stackhpc/hooks/post.yml +++ b/environments/.stackhpc/hooks/post.yml @@ -12,12 +12,3 @@ - "/opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/{{ ondemand_package_version }}-1/gems/bootstrap_form-2.7.0/test/dummy/Gemfile.lock" - "/opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/{{ ondemand_package_version }}-1/gems/bootstrap_form-4.5.0/demo/yarn.lock" - /var/www/ood/apps/sys/dashboard/node_modules/data-confirm-modal/Gemfile.lock - -- hosts: builder - become: yes - tasks: - - name: Disable ark repos - ansible.builtin.include_role: - name: dnf_repos - tasks_from: disable_repos.yml - when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided diff --git a/environments/.stackhpc/hooks/pre.yml b/environments/.stackhpc/hooks/pre.yml index 9ea84740d..0fdbf9f60 100644 --- a/environments/.stackhpc/hooks/pre.yml +++ b/environments/.stackhpc/hooks/pre.yml @@ -17,12 +17,3 @@ - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/hosts.yml" - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/all/secrets.yml" - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/all/test_user.yml" - -- hosts: builder - become: yes - tasks: - - name: Replace system repos with ark - ansible.builtin.include_role: - name: dnf_repos - tasks_from: set_repos.yml - when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided diff --git a/environments/.stackhpc/inventory/group_vars/builder.yml b/environments/.stackhpc/inventory/group_vars/builder.yml index 8d7ee98d2..5130e9d84 100644 --- a/environments/.stackhpc/inventory/group_vars/builder.yml +++ b/environments/.stackhpc/inventory/group_vars/builder.yml @@ -1 +1,19 @@ -#update_enable: false # Can uncomment for speed debugging non-update related build issues +# update_enable: false # Can uncomment for speed debugging non-update related build issues + +# Uncomment below to use CI pulp servers + +# pulp_server_config: +# LEAFCLOUD: +# url: http://192.168.10.157:8080 +# password: lookup('env','LEAFCLOUD_PULP_PASSWORD') + +# appliances_pulp_url: "{{ pulp_server_config[lookup('env','CI_CLOUD')].url }}" +# pulp_site_password: "{{ pulp_server_config[lookup('env','CI_CLOUD')].password }}" + +# Alternatively, configure to use ark directly: +dnf_repos_username: slurm-app-ci +dnf_repos_password: "{{ lookup('env','ARK_PASSWORD') }}" + +# Can be set regardless of approach above: +pulp_site_upstream_username: slurm-app-ci +pulp_site_upstream_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}" diff --git a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json index 5b9d845ef..7c59abf36 100644 --- a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json +++ b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json @@ -1,6 +1,6 @@ { "cluster_image": { - "RL8": "openhpc-RL8-241211-1322-ded60c2c", - "RL9": "openhpc-RL9-241211-1322-ded60c2c" + "RL8": "openhpc-RL8-241216-1607-2357a730", + "RL9": "openhpc-RL9-241216-1607-2357a730" } } diff --git a/environments/common/inventory/group_vars/all/defaults.yml b/environments/common/inventory/group_vars/all/defaults.yml index 2a88f035d..1bac4590d 100644 --- a/environments/common/inventory/group_vars/all/defaults.yml +++ b/environments/common/inventory/group_vars/all/defaults.yml @@ -7,6 +7,7 @@ appliances_environment_name: "{{ appliances_environment_root | basename | regex_ appliances_cockpit_state: absent # RHEL cockpit installed but not enabled in genericcloud images; appliance defaults to removing it #appliances_state_dir: # define an absolute path here to use for persistent state: NB: This is defined as /var/lib/state in inventory by the default Terraform appliances_mode: configure +appliances_pulp_url: https://ark.stackhpc.com # Address(ip/dns) for internal communication between services. This is # normally traffic you do no want to expose to users. @@ -80,3 +81,15 @@ appliances_local_users_extra: [] # see format of appliances_local_users_default appliances_local_users: "{{ appliances_local_users_default + appliances_local_users_extra }}" ########################################################################################### + +appliances_repo_timestamps: + baseos: + '9.4': 20240816T002610 + appstream: + '9.4': 20240816T002610 + crb: + '9.4': 20240816T002610 + extras: + '9.4': 20240816T002610 + epel: + '9': 20240902T080424 diff --git a/environments/common/inventory/group_vars/all/openhpc.yml b/environments/common/inventory/group_vars/all/openhpc.yml index c613fc697..a23bc77ba 100644 --- a/environments/common/inventory/group_vars/all/openhpc.yml +++ b/environments/common/inventory/group_vars/all/openhpc.yml @@ -38,3 +38,13 @@ openhpc_config_default: openhpc_config_extra: {} openhpc_config: "{{ openhpc_config_default | combine(openhpc_config_extra, list_merge='append') }}" openhpc_state_save_location: "{{ appliances_state_dir + '/slurmctld' if appliances_state_dir is defined else '/var/spool' }}" + +ohpc_default_extra_repos: + "9": [] #overriding to ensure doesn't overwrite ark epel repo + "8": + - name: epel + file: epel + description: "Extra Packages for Enterprise Linux 8 - $basearch" + metalink: "https://mirrors.fedoraproject.org/metalink?repo=epel-8&arch=$basearch&infra=$infra&content=$contentdir" + gpgcheck: true + gpgkey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8" diff --git a/environments/common/inventory/group_vars/all/pulp.yml b/environments/common/inventory/group_vars/all/pulp.yml new file mode 100644 index 000000000..22bb83216 --- /dev/null +++ b/environments/common/inventory/group_vars/all/pulp.yml @@ -0,0 +1,11 @@ +pulp_site_port: 8080 + +# If using Ark directly (no local Pulp server), override the following with Ark creds + +# dnf_repos_username: +# dnf_repos_password: + +# If instead using local Pulp server, override below with Ark creds + +# pulp_site_upstream_username: +# pulp_site_upstream_password: diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups index 9b9aa5bf0..062276f76 100644 --- a/environments/common/inventory/groups +++ b/environments/common/inventory/groups @@ -144,3 +144,11 @@ freeipa_client [lustre] # Hosts to run lustre client + +[dnf_repos:children] +# Hosts to replace system repos with Pulp repos +# Warning: when using Ark directly rather than a local Pulp server, adding hosts other than `builder` will leak Ark creds to users +builder + +[pulp] +# Add builder to this group to enable automatically syncing of pulp during image build diff --git a/requirements.txt b/requirements.txt index 6651506fb..7d81f3285 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ansible==6.0.0 +ansible==8.0.0 openstacksdk python-openstackclient==6.6.1 # v7.0.0 has a bug re. rebuild python-manilaclient @@ -9,3 +9,4 @@ cookiecutter selinux # this is a shim to avoid having to use --system-site-packages, you still need sudo yum install libselinux-python3 netaddr matplotlib +pulp-cli==0.29.2 diff --git a/requirements.yml b/requirements.yml index 579cdd5d5..577ca1c78 100644 --- a/requirements.yml +++ b/requirements.yml @@ -49,4 +49,6 @@ collections: - name: https://github.com/azimuth-cloud/ansible-collection-image-utils type: git version: 0.4.0 + - name: stackhpc.pulp + version: 0.5.5 ...