Skip to content

Commit 127b792

Browse files
wtripp180901wtripp180901
committed
pulp integration
1 parent 9723782 commit 127b792

File tree

18 files changed

+272
-47
lines changed

18 files changed

+272
-47
lines changed

‎ansible/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,5 @@ roles/*
6666
!roles/lustre/**
6767
!roles/dnf_repos/
6868
!roles/dnf_repos/**
69+
!roles/pulp_site/
70+
!roles/pulp_site/**

‎ansible/adhoc/deploy-pulp.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
2+
3+
- name: Add temporary pulp server host
4+
hosts: localhost
5+
tasks:
6+
- ansible.builtin.add_host:
7+
name: "{{ pulp_server }}"
8+
group: "_pulp_host"
9+
10+
- name: Install pulp on server
11+
become: yes
12+
hosts: _pulp_host
13+
tasks:
14+
- ansible.builtin.import_role:
15+
name: pulp_site
16+
tasks_from: install.yml
17+
18+
- name: Add pulp host to environment
19+
hosts: localhost
20+
tasks:
21+
- ansible.builtin.copy:
22+
dest: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/all/pulp_server.yml"
23+
content: |
24+
# ansible managed
25+
appliances_pulp_server: "http://{{ pulp_server }}"

‎ansible/bootstrap.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,23 @@
110110
policy: "{{ selinux_policy }}"
111111
register: sestatus
112112

113+
- name: Sync pulp repos with upstream
114+
hosts: localhost
115+
tasks:
116+
- ansible.builtin.include_role:
117+
name: pulp_site
118+
tasks_from: sync.yml
119+
when: appliances_mode != 'configure'
120+
121+
- hosts: dnf_repos
122+
become: yes
123+
tasks:
124+
- name: Replace system repos with pulp repos
125+
ansible.builtin.include_role:
126+
name: dnf_repos
127+
tasks_from: set_repos.yml
128+
when: appliances_mode != 'configure' and ansible_distribution_major_version == "9" #TODO update role once RL8 config decided
129+
113130
# --- tasks after here require access to package repos ---
114131
- hosts: squid
115132
tags: squid

‎ansible/roles/dnf_repos/defaults/main.yml‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,4 @@
1-
# dnf_repos_rocky_ark_prefix: https://ark.stackhpc.com/pulp/content/{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}
2-
# dnf_repos_rocky_ark_suffix: "{{ ansible_architecture }}/os/{{ dnf_repos_rocky_ark_timestamp }}/"
3-
# # most stable from https://github.com/stackhpc/stackhpc-kayobe-config/blob/stackhpc/2024.1/etc/kayobe/pulp-repo-versions.yml
4-
# # note that some timestamps can't be used because not all repos have snapshots for them
5-
# dnf_repos_rocky_ark_timestamp: 20240816T002610
6-
# dnf_repos_username: slurm-app-ci
7-
# dnf_repos_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"
8-
9-
# # epel installed separately
10-
# dnf_repos_repolist:
11-
# - file: rocky
12-
# name: baseos
13-
# base_url: "{{ dnf_repos_rocky_ark_prefix }}/BaseOS/{{ dnf_repos_rocky_ark_suffix }}"
14-
# - file: rocky
15-
# name: appstream
16-
# base_url: "{{ dnf_repos_rocky_ark_prefix }}/AppStream/{{ dnf_repos_rocky_ark_suffix }}"
17-
# - file: rocky
18-
# name: crb
19-
# base_url: "{{ dnf_repos_rocky_ark_prefix }}/CRB/{{ dnf_repos_rocky_ark_suffix }}"
20-
# - file: rocky-extras
21-
# name: extras
22-
# base_url: "{{ dnf_repos_rocky_ark_prefix }}/extras/{{ dnf_repos_rocky_ark_suffix }}"
23-
24-
# dnf_repos_epel_timestamp: 20240902T080424
25-
# dnf_repos_epel_baseurl: "https://ark.stackhpc.com/pulp/content/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ dnf_repos_epel_timestamp }}"
26-
27-
dnf_repos_pulp_url: # required
1+
dnf_repos_pulp_url: "{{ appliances_pulp_url }}"
282
dnf_repos_pulp_content_url: "{{ dnf_repos_pulp_url }}/pulp/content"
293
dnf_repos_rocky_prefix: "{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}"
304
dnf_repos_epel_prefix: "epel/{{ ansible_distribution_major_version }}"

‎ansible/roles/passwords/defaults/main.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ slurm_appliance_secrets:
99
vault_freeipa_ds_password: "{{ vault_freeipa_ds_password | default(lookup('password', '/dev/null')) }}"
1010
vault_freeipa_admin_password: "{{ vault_freeipa_admin_password | default(lookup('password', '/dev/null')) }}"
1111
vault_k3s_token: "{{ vault_k3s_token | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
12+
vault_pulp_admin_password: "{{ vault_pulp_admin_password | default(lookup('password', '/dev/null', chars=['ascii_letters', 'digits'])) }}"
1213

1314
secrets_openhpc_mungekey_default:
1415
content: "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') }}"

‎ansible/roles/pulp_site/defaults/main.yml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
pulp_site_url: "http://{{ appliances_pulp_url }}:{{ pulp_site_port }}"
2+
pulp_site_port: 8080
3+
pulp_site_username: admin # shouldn't be changed
4+
pulp_site_upstream_username: slurm-app-ci
5+
pulp_site_upstream_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"
6+
pulp_site_password: "{{ vault_pulp_admin_password }}"
7+
pulp_site_validate_certs: false
8+
pulp_site_install_dir: '/home/rocky/pulp'
9+
pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}"
10+
11+
pulp_site_rpm_repos:
12+
- name: baseos
13+
url: https://ark.stackhpc.com/pulp/content/rocky/9.4/BaseOS/x86_64/os/20240816T002610
14+
remote_username: "{{ pulp_site_upstream_username }}"
15+
remote_password: "{{ pulp_site_upstream_password }}"
16+
policy: on_demand
17+
state: present
18+
- name: appstream
19+
url: https://ark.stackhpc.com/pulp/content/rocky/9.4/AppStream/x86_64/os/20240816T002610
20+
remote_username: "{{ pulp_site_upstream_username }}"
21+
remote_password: "{{ pulp_site_upstream_password }}"
22+
policy: on_demand
23+
state: present
24+
- name: crb
25+
url: https://ark.stackhpc.com/pulp/content/rocky/9.4/CRB/x86_64/os/20240816T002610
26+
remote_username: "{{ pulp_site_upstream_username }}"
27+
remote_password: "{{ pulp_site_upstream_password }}"
28+
policy: on_demand
29+
state: present
30+
- name: extras
31+
url: https://ark.stackhpc.com/pulp/content/rocky/9.4/extras/x86_64/os/20240816T002610
32+
remote_username: "{{ pulp_site_upstream_username }}"
33+
remote_password: "{{ pulp_site_upstream_password }}"
34+
policy: on_demand
35+
state: present
36+
- name: epel
37+
url: https://ark.stackhpc.com/pulp/content/epel/9/Everything/x86_64/20240902T080424
38+
remote_username: "{{ pulp_site_upstream_username }}"
39+
remote_password: "{{ pulp_site_upstream_password }}"
40+
policy: on_demand
41+
state: present
42+
43+
pulp_site_rpm_publications:
44+
- repository: baseos
45+
state: present
46+
- repository: appstream
47+
state: present
48+
- repository: crb
49+
state: present
50+
- repository: extras
51+
state: present
52+
- repository: epel
53+
state: present
54+
55+
pulp_site_rpm_distributions:
56+
- name: baseos
57+
base_path: rocky/9.4/baseos
58+
repository: baseos
59+
state: present
60+
- name: appstream
61+
base_path: rocky/9.4/appstream
62+
repository: appstream
63+
state: present
64+
- name: crb
65+
base_path: rocky/9.4/crb
66+
repository: crb
67+
state: present
68+
- name: extras
69+
base_path: rocky/9.4/extras
70+
repository: extras
71+
state: present
72+
- name: epel
73+
base_path: epel/9
74+
repository: epel
75+
state: present

‎ansible/roles/pulp_site/tasks/install.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
---
2+
3+
- name: Install packages
4+
dnf:
5+
name:
6+
- podman
7+
8+
- name: Create install directories
9+
ansible.builtin.file:
10+
state: directory
11+
path: "{{ pulp_site_install_dir }}/{{ item }}"
12+
loop:
13+
- settings/certs
14+
- pulp_storage
15+
- pgsql
16+
- containers
17+
18+
- name: Template settings file
19+
ansible.builtin.template:
20+
src: settings.py.j2
21+
dest: "{{ pulp_site_install_dir }}/settings/settings.py"
22+
23+
- name: Install pulp podman container
24+
containers.podman.podman_container:
25+
name: pulp
26+
publish:
27+
- "{{ pulp_site_port }}:80"
28+
volume:
29+
- "{{ pulp_site_install_dir }}/settings:/etc/pulp{{ pulp_site_selinux_suffix }}"
30+
- "{{ pulp_site_install_dir }}/pulp_storage:/var/lib/pulp{{ pulp_site_selinux_suffix }}"
31+
- "{{ pulp_site_install_dir }}/pgsql:/var/lib/pgsql{{ pulp_site_selinux_suffix }}"
32+
- "{{ pulp_site_install_dir }}/containers:/var/lib/containers{{ pulp_site_selinux_suffix }}"
33+
device: /dev/fuse
34+
image: docker.io/pulp/pulp:3.68.1
35+
36+
- name: Reset admin password once container has initialised
37+
no_log: true
38+
ansible.builtin.shell:
39+
cmd: "podman exec pulp bash -c 'pulpcore-manager reset-admin-password -p {{ pulp_site_password }}'"
40+
register: _admin_reset_output
41+
until: 0 == _admin_reset_output.rc
42+
retries: 6
43+
delay: 30

‎ansible/roles/pulp_site/tasks/sync.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
---
2+
3+
- name: Wait for Pulp server
4+
pulp.squeezer.status:
5+
pulp_url: "{{ pulp_site_url }}"
6+
username: "{{ pulp_site_username }}"
7+
password: "{{ pulp_site_password }}"
8+
register: _pulp_status
9+
until: _pulp_status.failed == false
10+
retries: 30
11+
delay: 20
12+
13+
- name: Ensure Pulp CLI config directory exists
14+
ansible.builtin.file:
15+
path: ~/.config/pulp
16+
state: directory
17+
18+
- name: Create config file
19+
no_log: true
20+
ansible.builtin.template:
21+
src: cli.toml.j2
22+
dest: ~/.config/pulp/cli.toml
23+
mode: '0644'
24+
25+
- block:
26+
- name: Ensure squeezer cache exists
27+
ansible.builtin.file:
28+
path: "{{ _cache_dir }}"
29+
state: directory
30+
31+
- name: Check if squeezer cache is populated
32+
ansible.builtin.stat:
33+
path: "{{ _cache_dir }}/api.json"
34+
register: _cache_stat
35+
36+
- name: Prepopulate squeezer cache # workaround for race on the cache
37+
ansible.builtin.get_url:
38+
url: "{{ pulp_site_url }}/pulp/api/v3/docs/api.json"
39+
dest: "{{ _cache_dir }}/api.json"
40+
timeout: 40
41+
when: not _cache_stat.stat.exists
42+
vars:
43+
_cache_dir: "~/.cache/squeezer/{{ pulp_site_url | regex_replace( ':|/' , '_' ) }}"
44+
45+
- name: Get Pulp repos from release train
46+
ansible.builtin.include_role:
47+
name: stackhpc.pulp.pulp_repository
48+
tasks_from: rpm.yml
49+
vars:
50+
pulp_url: "{{ pulp_site_url }}"
51+
pulp_username: "{{ pulp_site_username }}"
52+
pulp_password: "{{ pulp_site_password }}"
53+
pulp_repository_rpm_repos: "{{ pulp_site_rpm_repos }}"
54+
55+
- name: Create Pulp publications
56+
ansible.builtin.include_role:
57+
name: stackhpc.pulp.pulp_publication
58+
tasks_from: rpm.yml
59+
vars:
60+
pulp_url: "{{ pulp_site_url }}"
61+
pulp_username: "{{ pulp_site_username }}"
62+
pulp_password: "{{ pulp_site_password }}"
63+
pulp_publication_rpm: "{{ pulp_site_rpm_publications }}"
64+
65+
- name: Create Pulp distributions
66+
ansible.builtin.include_role:
67+
name: stackhpc.pulp.pulp_distribution
68+
tasks_from: rpm.yml
69+
vars:
70+
pulp_url: "{{ pulp_site_url }}"
71+
pulp_username: "{{ pulp_site_username }}"
72+
pulp_password: "{{ pulp_site_password }}"
73+
pulp_distribution_rpm: "{{ pulp_site_rpm_distributions }}"

‎ansible/roles/pulp_site/templates/cli.toml.j2‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[cli]
2+
base_url = "{{ pulp_site_url }}"
3+
username = "{{ pulp_site_username }}"
4+
password = "{{ pulp_site_password }}"
5+
api_root = "/pulp/"
6+
domain = "default"
7+
headers = []
8+
cert = ""
9+
key = ""
10+
verify_ssl = true
11+
format = "json"
12+
dry_run = false
13+
timeout = 0
14+
verbose = 0

‎ansible/roles/pulp_site/templates/settings.py.j2‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
CONTENT_ORIGIN='http://{{ ansible_fqdn }}:{{ pulp_site_port }}'
2+
TOKEN_AUTH_DISABLED=True

0 commit comments

Comments
 (0)