Merge pull request #490 from stackhpc/feat/pulp-builds

Support site Pulp server for image builds

Author: wtripp180901

.github/workflows/fatimage.yml

@@ -33,6 +33,7 @@ jobs:
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
       ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+      LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/nightlybuild.yml

@@ -35,6 +35,7 @@ jobs:
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud || vars.CI_CLOUD }}
       ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+      LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
 
     steps:
       - uses: actions/checkout@v2

README.md

@@ -32,7 +32,7 @@ It requires an OpenStack cloud, and an Ansible "deploy host" with access to that
 Before starting ensure that:
 - You have root access on the deploy host.
 - You can create instances using a Rocky 9 GenericCloud image (or an image based on that).
-    - **NB**: In general it is recommended to use the [latest released image](https://github.com/stackhpc/ansible-slurm-appliance/releases) which already contains the required packages. This is built and tested in StackHPC's CI. However the appliance will install the necessary packages if a GenericCloud image is used.
+    - **NB**: In general it is recommended to use the [latest released image](https://github.com/stackhpc/ansible-slurm-appliance/releases) which already contains the required packages. This is built and tested in StackHPC's CI.
 - You have a SSH keypair defined in OpenStack, with the private key available on the deploy host.
 - Created instances have access to internet (note proxies can be setup through the appliance if necessary).
 - Created instances have accurate/synchronised time (for VM instances this is usually provided by the hypervisor; if not or for bare metal instances it may be necessary to configure a time service via the appliance).

ansible/.gitignore

@@ -66,5 +66,7 @@ roles/*
 !roles/lustre/**
 !roles/dnf_repos/
 !roles/dnf_repos/**
+!roles/pulp_site/
+!roles/pulp_site/**
 !roles/doca/
 !roles/doca/**

ansible/adhoc/deploy-pulp.yml

@@ -0,0 +1,26 @@
+# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
+
+- name: Add temporary pulp server host
+  hosts: localhost
+  tasks:
+  - ansible.builtin.add_host:
+      name: "{{ pulp_server }}"
+      group: "_pulp_host"
+
+- name: Install pulp on server and add to config
+  become: yes
+  hosts: _pulp_host
+  tasks:
+  - name: Install pulp
+    ansible.builtin.include_role:
+      name: pulp_site
+      tasks_from: install.yml
+      public: true
+
+  - name: Print Pulp endpoint
+    become: no
+    debug:
+      msg: | 
+        Server configured, override 'appliances_pulp_url' with
+          appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"
+        in your environments

ansible/adhoc/sync-pulp.yml

@@ -0,0 +1,10 @@
+- hosts: localhost
+  tasks:
+    - ansible.builtin.include_role:
+        name: pulp_site
+        tasks_from: sync.yml
+      vars:
+        pulp_site_target_arch: "x86_64"
+        pulp_site_target_distribution: "rocky"
+        pulp_site_target_distribution_version: "9.4"
+        pulp_site_target_distribution_version_major: "9"

ansible/bootstrap.yml

@@ -110,6 +110,20 @@
         policy: "{{ selinux_policy }}"
       register: sestatus
 
+- hosts: dnf_repos
+  become: yes
+  tasks:
+  - name: Check that creds won't be leaked to users
+    ansible.builtin.assert:
+      that: dnf_repos_password is undefined
+      fail_msg: Passwords should not be templated into repofiles during configure, unset 'dnf_repos_password'
+    when: appliances_mode == 'configure'
+  - name: Replace system repos with pulp repos 
+    ansible.builtin.include_role:
+      name: dnf_repos
+      tasks_from: set_repos.yml
+    when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided
+
 # --- tasks after here require access to package repos ---
 - hosts: squid
   tags: squid

ansible/disable-repos.yml

@@ -0,0 +1,8 @@
+- hosts: dnf_repos
+  become: yes
+  tasks:
+    - name: Disable pulp repos
+      ansible.builtin.include_role:
+        name: dnf_repos
+        tasks_from: disable_repos.yml
+      when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided

ansible/fatimage.yml

@@ -17,6 +17,16 @@
   import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
   when: hook_path | exists
 
+- name: Sync pulp repos with upstream
+  hosts: pulp
+  tasks:
+  - ansible.builtin.include_role:
+      name: pulp_site
+      tasks_from: sync.yml
+      apply:
+        delegate_to: localhost
+    when: appliances_mode != 'configure'
+
 - import_playbook: bootstrap.yml
 
 - name: Run post-bootstrap.yml hook
@@ -210,6 +220,8 @@
       import_role:
         name: doca
 
+- import_playbook: disable-repos.yml
+
 - name: Run post.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"

ansible/roles/dnf_repos/defaults/main.yml

@@ -1,25 +1,23 @@
-dnf_repos_rocky_ark_prefix: https://ark.stackhpc.com/pulp/content/{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}
-dnf_repos_rocky_ark_suffix: "{{ ansible_architecture }}/os/{{ dnf_repos_rocky_ark_timestamp }}/"
-# most stable from https://github.com/stackhpc/stackhpc-kayobe-config/blob/stackhpc/2024.1/etc/kayobe/pulp-repo-versions.yml
-# note that some timestamps can't be used because not all repos have snapshots for them
-dnf_repos_rocky_ark_timestamp: 20240816T002610
-dnf_repos_username: slurm-app-ci
-dnf_repos_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"
+dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
+dnf_repos_rocky_prefix: "{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}"
+dnf_repos_epel_prefix: "epel/{{ ansible_distribution_major_version }}"
+dnf_repos_username: "{{ omit }}"
+dnf_repos_password: "{{ omit }}"
 
 # epel installed separately
 dnf_repos_repolist:
 - file: rocky
   name: baseos
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/BaseOS/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/BaseOS/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.baseos[ansible_distribution_version] }}"
 - file: rocky
   name: appstream
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/AppStream/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/AppStream/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.appstream[ansible_distribution_version] }}"
 - file: rocky
   name: crb
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/CRB/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/CRB/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.crb[ansible_distribution_version] }}"
 - file: rocky-extras
   name: extras
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/extras/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/extras/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.extras[ansible_distribution_version] }}"
 
-dnf_repos_epel_timestamp: 20240902T080424
-dnf_repos_epel_baseurl: "https://ark.stackhpc.com/pulp/content/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ dnf_repos_epel_timestamp }}"
+dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ appliances_repo_timestamps.epel[ansible_distribution_major_version] }}"
+dnf_repos_epel_description: "epel"