Merge pull request #499 from stackhpc/feat/extra-packages

Add list of additional package installs

Author: bertiethorpe

ansible/extras.yml

@@ -45,3 +45,13 @@
   tasks:
   - import_role:
       name: k9s
+
+- hosts: extra_packages
+  become: yes
+  tags:
+   - extra_packages
+  tasks:
+  - name: Install additional packages
+    dnf:
+      name: "{{ appliances_extra_packages }}"
+    when: appliances_mode != 'configure' or appliances_extra_packages_during_configure

docs/operations.md

@@ -63,17 +63,30 @@ This is a usually a two-step process:
 Deploying the additional nodes and applying these changes requires rerunning both Terraform and the Ansible site.yml playbook - follow [Deploying a Cluster](#Deploying-a-Cluster).
 
 # Adding Additional Packages
-Packages from any enabled DNF repositories (which always includes EPEL, PowerTools and OpenHPC) can be added to all nodes by defining a list `openhpc_packages_extra` (defaulted to the empty list in the common environment) in e.g. `environments/$SITE_ENV/inventory/group_vars/all/openhpc.yml`. For example:
-
-    # environments/foo-base/inventory/group_vars/all/openhpc.yml:
-    openhpc_packages_extra:
+By default, the following utility packages are installed during build:
+- htop
+- nano
+- screen
+- tmux
+- wget
+- bind-utils
+- net-tools
+- postfix
+- git
+- latest python version for system (3.6 for for Rocky 8.9 and 3.12 for Rocky 9.4)
+
+Additional packages from any DNF repositories which are enabled during build (which always includes EPEL, PowerTools and OpenHPC) can be added to the image by defining a list `appliances_extra_packages_other` (defaulted to the empty list in the common environment) in e.g. `environments/$SITE_ENV/inventory/group_vars/all/defaults.yml`. For example:
+
+```yaml
+    # environments/foo-base/inventory/group_vars/all/defaults.yml:
+    appliances_extra_packages_other:
     - somepackage
     - anotherpackage
 
 
 The packages available from the OpenHPC repos are described in Appendix E of the OpenHPC installation guide (linked from the [OpenHPC releases page](https://github.com/openhpc/ohpc/releases/)). Note "user-facing" OpenHPC packages such as compilers, mpi libraries etc. include corresponding `lmod` modules.
 
-To add these packages to the current cluster, run the same command as for [Reconfiguring Slurm](#Reconfiguring-Slurm). TODO: describe what's required to add these to site-specific images.
+If you wish to install packages during runtime, the `site.yml` playbook should be run with `appliances_packages_during_configure` overriden to `true` and `cluster` should be added as a child of the `dnf_repos` group in order to temporarily re-enable DNF repositories during runtime (WARNING: this should only be done if using an unauthenticated local Pulp server. If using StackHPC Ark directly, doing this WILL leak credentials to users).
 
 If additional repositories are required, these could be added/enabled as necessary in a play added to `environments/$SITE_ENV/hooks/{pre,post}.yml` as appropriate. Note such a plat should NOT exclude the builder group, so that the repositories are also added to built images. There are various Ansible modules which might be useful for this:
     - `ansible.builtin.yum_repository`: Add a repo from an URL providing a 'repodata' directory.

environments/.stackhpc/terraform/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-241218-1705-09ac4268",
-        "RL9": "openhpc-RL9-241218-1705-09ac4268"
+        "RL8": "openhpc-RL8-241219-1232-7f84fed4",
+        "RL9": "openhpc-RL9-241219-1145-7f84fed4"
     }
 }

environments/common/inventory/group_vars/all/defaults.yml

@@ -80,7 +80,28 @@ appliances_local_users_default:
 appliances_local_users_extra: [] # see format of appliances_local_users_default above
 appliances_local_users: "{{ appliances_local_users_default + appliances_local_users_extra }}"
 
-###########################################################################################
+################## bootstrap: extra package installs ######################################
+
+appliances_extra_packages_default:
+ - htop
+ - nano
+ - screen
+ - tmux
+ - wget
+ - bind-utils
+ - net-tools
+ - postfix
+ - git
+ - "{{ 'python36' if ansible_distribution_version == '8.9' else 'python312' }}"
+ 
+
+appliances_extra_packages_during_configure: false
+
+appliances_extra_packages_other: []
+  
+appliances_extra_packages: "{{ appliances_extra_packages_default + appliances_extra_packages_other }}"
+
+###################### ark repo timestamps ###################################################
 
 appliances_repo_timestamps:
   baseos:

environments/common/inventory/groups

@@ -145,6 +145,9 @@ freeipa_client
 [lustre]
 # Hosts to run lustre client
 
+[extra_packages]
+# Hosts to install specified additional packages on
+
 [dnf_repos:children]
 # Hosts to replace system repos with Pulp repos
 # Warning: when using Ark directly rather than a local Pulp server, adding hosts other than `builder` will leak Ark creds to users

environments/common/layouts/everything

@@ -92,3 +92,7 @@ control
 
 [lustre]
 # Hosts to run lustre client
+
+[extra_packages:children]
+# Hosts to install specified additional packages on
+cluster