Support site Pulp server for image builds

ansible/.gitignore

@@ -66,3 +66,5 @@ roles/*
 !roles/lustre/**
 !roles/dnf_repos/
 !roles/dnf_repos/**
+!roles/pulp_site/
+!roles/pulp_site/**

ansible/adhoc/deploy-pulp.yml

@@ -0,0 +1,37 @@
+# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
+
+- name: Add temporary pulp server host
+  hosts: localhost
+  tasks:
+  - ansible.builtin.add_host:
+      name: "{{ pulp_server }}"
+      group: "_pulp_host"
+
+- name: Install pulp on server and add to config
+  become: yes
+  hosts: _pulp_host
+  tasks:
+
+  - name: Install pulp
+    ansible.builtin.include_role:
+      name: pulp_site
+      tasks_from: install.yml
+      public: true
+
+  - name: Add pulp host to environment
+    become: no
+    delegate_to: localhost
+    ansible.builtin.copy:
+      dest: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/all/pulp_server.yml"
+      content: |
+        # ansible managed
+        appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"
+
+# - name: Add pulp host to environment
+#   hosts: localhost
+#   tasks:
+#   - ansible.builtin.copy:
+#       dest: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/all/pulp_server.yml"
+#       content: |
+#         # ansible managed
+#         appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"

ansible/bootstrap.yml

@@ -110,6 +110,23 @@
         policy: "{{ selinux_policy }}"
       register: sestatus
 
+- name: Sync pulp repos with upstream
+  hosts: localhost
+  tasks:
+  - ansible.builtin.include_role:
+      name: pulp_site
+      tasks_from: sync.yml
+    when: appliances_mode != 'configure'
+
+- hosts: dnf_repos
+  become: yes
+  tasks:
+  - name: Replace system repos with pulp repos 
+    ansible.builtin.include_role:
+      name: dnf_repos
+      tasks_from: set_repos.yml
+    when: appliances_mode != 'configure' and ansible_distribution_major_version == "9" #TODO update role once RL8 config decided
+
 # --- tasks after here require access to package repos ---
 - hosts: squid
   tags: squid

ansible/roles/dnf_repos/defaults/main.yml

@@ -1,25 +1,22 @@
-dnf_repos_rocky_ark_prefix: https://ark.stackhpc.com/pulp/content/{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}
-dnf_repos_rocky_ark_suffix: "{{ ansible_architecture }}/os/{{ dnf_repos_rocky_ark_timestamp }}/"
-# most stable from https://github.com/stackhpc/stackhpc-kayobe-config/blob/stackhpc/2024.1/etc/kayobe/pulp-repo-versions.yml
-# note that some timestamps can't be used because not all repos have snapshots for them
-dnf_repos_rocky_ark_timestamp: 20240816T002610
-dnf_repos_username: slurm-app-ci
-dnf_repos_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"
+dnf_repos_pulp_url: "{{ appliances_pulp_url }}"
+dnf_repos_pulp_content_url: "{{ dnf_repos_pulp_url }}/pulp/content"
+dnf_repos_rocky_prefix: "{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}"
+dnf_repos_epel_prefix: "epel/{{ ansible_distribution_major_version }}"
+dnf_repos_disable: true
 
 # epel installed separately
 dnf_repos_repolist:
 - file: rocky
   name: baseos
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/BaseOS/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/baseos"
 - file: rocky
   name: appstream
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/AppStream/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/appstream"
 - file: rocky
   name: crb
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/CRB/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/crb"
 - file: rocky-extras
   name: extras
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/extras/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/extras"
 
-dnf_repos_epel_timestamp: 20240902T080424
-dnf_repos_epel_baseurl: "https://ark.stackhpc.com/pulp/content/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ dnf_repos_epel_timestamp }}"
+dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/epel/{{ ansible_distribution_major_version }}"

ansible/roles/dnf_repos/tasks/disable_repos.yml

@@ -1,18 +1,6 @@
 ---
 - name: Disable Pulp repos and remove creds
-  ansible.builtin.yum_repository:
-    file: "{{ item.file }}"
-    name: "{{ item.name }}"
-    baseurl: "{{ item.base_url }}"
-    description: "{{ item.name }}"
-    enabled: false
-  loop: "{{ dnf_repos_repolist }}"
-
-- name: Disable EPEL repo and remove creds
-  ansible.builtin.yum_repository:
-    name: epel
-    file: epel
-    description: epel
-    baseurl: "{{ dnf_repos_epel_baseurl }}"
-    gpgcheck: false
-    enabled: false
+  ansible.builtin.yum:
+    disablerepo: "{{ item.name }}"
+  loop: "{{ dnf_repos_repolist + [epel] }}"
+  when: dnf_repos_disable

ansible/roles/dnf_repos/tasks/set_repos.yml

@@ -6,8 +6,6 @@
     name: "{{ item.name }}"
     baseurl: "{{ item.base_url }}"
     description: "{{ item.name }}"
-    username: "{{ dnf_repos_username }}"
-    password: "{{ dnf_repos_password }}"
   loop: "{{ dnf_repos_repolist }}"
 
 - name: Install epel-release
@@ -21,6 +19,4 @@
     file: epel
     description: epel
     gpgcheck: false
-    username: "{{ dnf_repos_username }}"
-    password: "{{ dnf_repos_password }}"
     baseurl: "{{ dnf_repos_epel_baseurl }}"

ansible/roles/passwords/defaults/main.yml

@@ -9,6 +9,7 @@ slurm_appliance_secrets:
   vault_freeipa_ds_password: "{{ vault_freeipa_ds_password | default(lookup('password', '/dev/null')) }}"
   vault_freeipa_admin_password: "{{ vault_freeipa_admin_password | default(lookup('password', '/dev/null')) }}"
   vault_k3s_token: "{{ vault_k3s_token | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
+  vault_pulp_admin_password: "{{ vault_pulp_admin_password | default(lookup('password', '/dev/null', chars=['ascii_letters', 'digits'])) }}"
 
 secrets_openhpc_mungekey_default:
   content: "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') }}"

ansible/roles/pulp_site/defaults/main.yml

@@ -0,0 +1,75 @@
+pulp_site_url: "http://{{ appliances_pulp_url }}"
+pulp_site_port: 8080
+pulp_site_username: admin # shouldn't be changed
+pulp_site_upstream_username: slurm-app-ci
+pulp_site_upstream_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"
+pulp_site_password: "{{ vault_pulp_admin_password }}"
+pulp_site_validate_certs: false
+pulp_site_install_dir: '/home/rocky/pulp'
+pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}"
+
+pulp_site_rpm_repos:
+  - name: baseos
+    url: https://ark.stackhpc.com/pulp/content/rocky/9.4/BaseOS/x86_64/os/20240816T002610
+    remote_username: "{{ pulp_site_upstream_username }}"
+    remote_password: "{{ pulp_site_upstream_password }}"
+    policy: on_demand
+    state: present
+  - name: appstream
+    url: https://ark.stackhpc.com/pulp/content/rocky/9.4/AppStream/x86_64/os/20240816T002610
+    remote_username: "{{ pulp_site_upstream_username }}"
+    remote_password: "{{ pulp_site_upstream_password }}"
+    policy: on_demand
+    state: present
+  - name: crb
+    url: https://ark.stackhpc.com/pulp/content/rocky/9.4/CRB/x86_64/os/20240816T002610
+    remote_username: "{{ pulp_site_upstream_username }}"
+    remote_password: "{{ pulp_site_upstream_password }}"
+    policy: on_demand
+    state: present
+  - name: extras
+    url: https://ark.stackhpc.com/pulp/content/rocky/9.4/extras/x86_64/os/20240816T002610
+    remote_username: "{{ pulp_site_upstream_username }}"
+    remote_password: "{{ pulp_site_upstream_password }}"
+    policy: on_demand
+    state: present
+  - name: epel
+    url: https://ark.stackhpc.com/pulp/content/epel/9/Everything/x86_64/20240902T080424
+    remote_username: "{{ pulp_site_upstream_username }}"
+    remote_password: "{{ pulp_site_upstream_password }}"
+    policy: on_demand
+    state: present
+
+pulp_site_rpm_publications:
+- repository: baseos
+  state: present
+- repository: appstream
+  state: present
+- repository: crb
+  state: present
+- repository: extras
+  state: present
+- repository: epel
+  state: present
+
+pulp_site_rpm_distributions:
+- name: baseos
+  base_path: rocky/9.4/baseos
+  repository: baseos
+  state: present
+- name: appstream
+  base_path: rocky/9.4/appstream
+  repository: appstream
+  state: present
+- name: crb
+  base_path: rocky/9.4/crb
+  repository: crb
+  state: present
+- name: extras
+  base_path: rocky/9.4/extras
+  repository: extras
+  state: present
+- name: epel
+  base_path: epel/9
+  repository: epel
+  state: present

ansible/roles/pulp_site/tasks/install.yml

@@ -0,0 +1,43 @@
+---
+
+- name: Install packages
+  dnf:
+    name:
+    - podman
+
+- name: Create install directories
+  ansible.builtin.file:
+    state: directory
+    path: "{{ pulp_site_install_dir }}/{{ item }}"
+  loop:
+  - settings/certs
+  - pulp_storage
+  - pgsql
+  - containers
+
+- name: Template settings file
+  ansible.builtin.template:
+    src: settings.py.j2
+    dest: "{{ pulp_site_install_dir }}/settings/settings.py"
+
+- name: Install pulp podman container
+  containers.podman.podman_container:
+    name: pulp
+    publish:
+      - "{{ pulp_site_port }}:80"
+    volume:
+    - "{{ pulp_site_install_dir }}/settings:/etc/pulp{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/pulp_storage:/var/lib/pulp{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/pgsql:/var/lib/pgsql{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/containers:/var/lib/containers{{ pulp_site_selinux_suffix }}"
+    device: /dev/fuse
+    image: docker.io/pulp/pulp:3.68.1
+
+- name: Reset admin password once container has initialised
+  no_log: true
+  ansible.builtin.shell:
+    cmd: "podman exec pulp bash -c 'pulpcore-manager reset-admin-password -p {{ pulp_site_password }}'"
+  register: _admin_reset_output
+  until: 0 == _admin_reset_output.rc
+  retries: 6
+  delay: 30

ansible/roles/pulp_site/tasks/sync.yml

@@ -0,0 +1,73 @@
+---
+
+- name: Wait for Pulp server
+  pulp.squeezer.status:
+    pulp_url: "{{ pulp_site_url }}"
+    username: "{{ pulp_site_username }}"
+    password: "{{ pulp_site_password }}"
+  register: _pulp_status
+  until: _pulp_status.failed == false
+  retries: 30
+  delay: 20
+
+- name: Ensure Pulp CLI config directory exists
+  ansible.builtin.file:
+    path: ~/.config/pulp
+    state: directory
+
+- name: Create config file
+  no_log: true
+  ansible.builtin.template:
+    src: cli.toml.j2
+    dest: ~/.config/pulp/cli.toml
+    mode: '0644'
+
+- block:
+  - name: Ensure squeezer cache exists
+    ansible.builtin.file:
+      path: "{{ _cache_dir }}"
+      state: directory
+
+  - name: Check if squeezer cache is populated
+    ansible.builtin.stat:
+      path: "{{ _cache_dir }}/api.json"
+    register: _cache_stat
+
+  - name: Prepopulate squeezer cache # workaround for race on the cache
+    ansible.builtin.get_url:
+      url: "{{ pulp_site_url }}/pulp/api/v3/docs/api.json"
+      dest: "{{ _cache_dir }}/api.json"
+      timeout: 40
+    when: not _cache_stat.stat.exists
+  vars:
+    _cache_dir: "~/.cache/squeezer/{{ pulp_site_url | regex_replace( ':|/' , '_' ) }}"
+
+- name: Get Pulp repos from release train
+  ansible.builtin.include_role:
+    name: stackhpc.pulp.pulp_repository
+    tasks_from: rpm.yml
+  vars:
+    pulp_url: "{{ pulp_site_url }}"
+    pulp_username: "{{ pulp_site_username }}"
+    pulp_password: "{{ pulp_site_password }}"
+    pulp_repository_rpm_repos: "{{ pulp_site_rpm_repos }}"
+
+- name: Create Pulp publications
+  ansible.builtin.include_role:
+    name: stackhpc.pulp.pulp_publication
+    tasks_from: rpm.yml
+  vars:
+    pulp_url: "{{ pulp_site_url }}"
+    pulp_username: "{{ pulp_site_username }}"
+    pulp_password: "{{ pulp_site_password }}"
+    pulp_publication_rpm: "{{ pulp_site_rpm_publications }}"
+
+- name: Create Pulp distributions
+  ansible.builtin.include_role:
+    name: stackhpc.pulp.pulp_distribution
+    tasks_from: rpm.yml
+  vars:
+    pulp_url: "{{ pulp_site_url }}"
+    pulp_username: "{{ pulp_site_username }}"
+    pulp_password: "{{ pulp_site_password }}"
+    pulp_distribution_rpm: "{{ pulp_site_rpm_distributions }}"

ansible/roles/pulp_site/templates/cli.toml.j2

@@ -0,0 +1,14 @@
+[cli]
+base_url = "{{ pulp_site_url }}"
+username = "{{ pulp_site_username }}"
+password = "{{ pulp_site_password }}"
+api_root = "/pulp/"
+domain = "default"
+headers = []
+cert = ""
+key = ""
+verify_ssl = true
+format = "json"
+dry_run = false
+timeout = 0
+verbose = 0

ansible/roles/pulp_site/templates/settings.py.j2

@@ -0,0 +1,2 @@
+CONTENT_ORIGIN='http://{{ ansible_fqdn }}:{{ pulp_site_port }}'
+TOKEN_AUTH_DISABLED=True

ansible/site.yml

@@ -28,6 +28,15 @@
 - import_playbook: portal.yml
 - import_playbook: monitoring.yml
 
+- hosts: dnf_repos
+  become: yes
+  tasks:
+    - name: Disable pulp repos
+      ansible.builtin.include_role:
+        name: dnf_repos
+        tasks_from: disable_repos.yml
+      when: appliances_mode != 'configure' and ansible_distribution_major_version == "9" #TODO update role once RL8 config decided
+
 - name: Run post.yml hook
   vars:
     # hostvars not available here, so have to recalculate environment root:

environments/.stackhpc/hooks/post.yml

@@ -12,12 +12,3 @@
         - "/opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/{{ ondemand_package_version }}-1/gems/bootstrap_form-2.7.0/test/dummy/Gemfile.lock"
         - "/opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/{{ ondemand_package_version }}-1/gems/bootstrap_form-4.5.0/demo/yarn.lock"
         - /var/www/ood/apps/sys/dashboard/node_modules/data-confirm-modal/Gemfile.lock
-
-- hosts: builder
-  become: yes
-  tasks:
-    - name: Disable ark repos
-      ansible.builtin.include_role:
-        name: dnf_repos
-        tasks_from: disable_repos.yml
-      when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided

environments/.stackhpc/inventory/group_vars/builder.yml

@@ -1 +1,2 @@
 #update_enable: false # Can uncomment for speed debugging non-update related build issues
+appliances_pulp_url: http://192.168.10.157:8080