diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml index 5425eb4e3..947f9410f 100644 --- a/.github/workflows/fatimage.yml +++ b/.github/workflows/fatimage.yml @@ -117,4 +117,4 @@ jobs: path: | ./image-id.txt ./image-name.txt - overwrite: true \ No newline at end of file + overwrite: true diff --git a/ansible/.gitignore b/ansible/.gitignore index 2ceeb596b..f6f5c5f4d 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -58,4 +58,5 @@ roles/* !roles/squid/** !roles/tuned/ !roles/tuned/** - +!roles/lustre/ +!roles/lustre/** diff --git a/ansible/fatimage.yml b/ansible/fatimage.yml index e623c2794..7cad2dc59 100644 --- a/ansible/fatimage.yml +++ b/ansible/fatimage.yml @@ -25,7 +25,7 @@ - hosts: builder become: yes - gather_facts: no + gather_facts: yes tasks: # - import_playbook: iam.yml - name: Install FreeIPA client @@ -44,6 +44,11 @@ name: stackhpc.os-manila-mount tasks_from: install.yml when: "'manila' in group_names" + - name: Install Lustre packages + include_role: + name: lustre + tasks_from: install.yml + when: "'lustre' in group_names" - import_playbook: extras.yml @@ -57,6 +62,7 @@ name: mysql tasks_from: install.yml when: "'mysql' in group_names" + - name: OpenHPC import_role: name: stackhpc.openhpc @@ -83,18 +89,21 @@ import_role: name: openondemand tasks_from: vnc_compute.yml + when: "'openondemand_desktop' in group_names" + - name: Open Ondemand jupyter node import_role: name: openondemand tasks_from: jupyter_compute.yml - when: "'openondemand' in group_names" + when: "'openondemand_jupyter' in group_names" # - import_playbook: monitoring.yml: - import_role: name: opensearch tasks_from: install.yml when: "'opensearch' in group_names" + # slurm_stats - nothing to do - import_role: name: filebeat diff --git a/ansible/filesystems.yml b/ansible/filesystems.yml index e1a782bad..4665c0f8f 100644 --- a/ansible/filesystems.yml +++ b/ansible/filesystems.yml @@ -24,3 +24,13 @@ tasks: - include_role: name: stackhpc.os-manila-mount + +- name: Setup Lustre clients + hosts: lustre + become: true + tags: lustre + tasks: + - include_role: + name: lustre + # NB install is ONLY run in builder + tasks_from: configure.yml diff --git a/ansible/roles/lustre/README.md b/ansible/roles/lustre/README.md new file mode 100644 index 000000000..c0a25e037 --- /dev/null +++ b/ansible/roles/lustre/README.md @@ -0,0 +1,27 @@ +# lustre + +Install and configure a Lustre client. This builds RPM packages from source. + +**NB:** The `install.yml` playbook in this role should only be run during image build and is not idempotent. This will install the `kernel-devel` package; if not already installed (e.g. from an `ofed` installation), this may require enabling update of DNF packages during build using `update_enable=true`, which will upgrade the kernel as well. + +**NB:** Currently this only supports RockyLinux 9. + +## Role Variables + +- `lustre_version`: Optional str. Version of lustre to build, default `2.15.5` which is the first version with EL9 support +- `lustre_lnet_label`: Optional str. The "lnet label" part of the host's NID, e.g. `tcp0`. Only the `tcp` protocol type is currently supported. Default `tcp`. +- `lustre_mgs_nid`: Required str. The NID(s) for the MGS, e.g. `192.168.227.11@tcp1` (separate mutiple MGS NIDs using `:`). +- `lustre_mounts`: Required list. Define Lustre filesystems and mountpoints as a list of dicts with keys: + - `fs_name`: Required str. The name of the filesystem to mount + - `mount_point`: Required str. Path to mount filesystem at. + - `mount_state`: Optional mount state, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default is `lustre_mount_state`. + - `mount_options`: Optional mount options. Default is `lustre_mount_options`. +- `lustre_mount_state`. Optional default mount state for all mounts, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default is `mounted`. +- `lustre_mount_options`. Optional default mount options. Default values are systemd defaults from [Lustre client docs](http://wiki.lustre.org/Mounting_a_Lustre_File_System_on_Client_Nodes). + +The following variables control the package build and and install and should not generally be required: +- `lustre_build_packages`: Optional list. Prerequisite packages required to build Lustre. See `defaults/main.yml`. +- `lustre_build_dir`: Optional str. Path to build lustre at, default `/tmp/lustre-release`. +- `lustre_configure_opts`: Optional list. Options to `./configure` command. Default builds client rpms supporting Mellanox OFED, without support for GSS keys. +- `lustre_rpm_globs`: Optional list. Shell glob patterns for rpms to install. Note order is important as the built RPMs are not in a yum repo. Default is just the `kmod-lustre-client` and `lustre-client` packages. +- `lustre_build_cleanup`: Optional bool. Whether to uninstall prerequisite packages and delete the build directories etc. Default `true`. diff --git a/ansible/roles/lustre/defaults/main.yml b/ansible/roles/lustre/defaults/main.yml new file mode 100644 index 000000000..be008ad55 --- /dev/null +++ b/ansible/roles/lustre/defaults/main.yml @@ -0,0 +1,36 @@ +lustre_version: '2.15.5' # https://www.lustre.org/lustre-2-15-5-released/ +lustre_lnet_label: tcp +#lustre_mgs_nid: +lustre_mounts: [] +lustre_mount_state: mounted +lustre_mount_options: 'defaults,_netdev,noauto,x-systemd.automount,x-systemd.requires=lnet.service' + +# below variables are for build and should not generally require changes +lustre_build_packages: + - "kernel-devel-{{ ansible_kernel }}" + - git + - gcc + - libtool + - python3 + - python3-devel + - openmpi + - elfutils-libelf-devel + - libmount-devel + - libnl3-devel + - libyaml-devel + - rpm-build + - kernel-abi-stablelists + - libaio + - libaio-devel +lustre_build_dir: /tmp/lustre-release +lustre_configure_opts: + - --disable-server + - --with-linux=/usr/src/kernels/* + - --with-o2ib=/usr/src/ofa_kernel/default + - --disable-maintainer-mode + - --disable-gss-keyring + - --enable-mpitests=no +lustre_rpm_globs: # NB: order is important here, as not installing from a repo + - "kmod-lustre-client-{{ lustre_version | split('.') | first }}*" # only take part of the version as -RC versions produce _RC rpms + - "lustre-client-{{ lustre_version | split('.') | first }}*" +lustre_build_cleanup: true diff --git a/ansible/roles/lustre/tasks/configure.yml b/ansible/roles/lustre/tasks/configure.yml new file mode 100644 index 000000000..b77e02ed9 --- /dev/null +++ b/ansible/roles/lustre/tasks/configure.yml @@ -0,0 +1,47 @@ +- name: Gather Lustre interface info + shell: + cmd: | + ip r get {{ _lustre_mgs_ip }} + changed_when: false + register: _lustre_ip_r_mgs + vars: + _lustre_mgs_ip: "{{ lustre_mgs_nid | split('@') | first }}" + +- name: Set facts for Lustre interface + set_fact: + _lustre_interface: "{{ _lustre_ip_r_mgs_info[4] }}" + _lustre_ip: "{{ _lustre_ip_r_mgs_info[6] }}" + vars: + _lustre_ip_r_mgs_info: "{{ _lustre_ip_r_mgs.stdout_lines.0 | split }}" + # first line e.g. "10.167.128.1 via 10.179.0.2 dev eth0 src 10.179.3.149 uid 1000" + +- name: Write LNet configuration file + template: + src: lnet.conf.j2 + dest: /etc/lnet.conf # exists from package install, expected by lnet service + owner: root + group: root + mode: u=rw,go=r # from package install + register: _lnet_conf + +- name: Ensure lnet service state + systemd: + name: lnet + state: "{{ 'restarted' if _lnet_conf.changed else 'started' }}" + +- name: Ensure mount points exist + ansible.builtin.file: + path: "{{ item.mount_point }}" + state: directory + loop: "{{ lustre_mounts }}" + when: "(item.mount_state | default(lustre_mount_state)) != 'absent'" + +- name: Mount lustre filesystem + ansible.posix.mount: + fstype: lustre + src: "{{ lustre_mgs_nid }}:/{{ item.fs_name }}" + path: "{{ item.mount_point }}" + state: "{{ (item.mount_state | default(lustre_mount_state)) }}" + opts: "{{ item.mount_options | default(lustre_mount_options) }}" + loop: "{{ lustre_mounts }}" + \ No newline at end of file diff --git a/ansible/roles/lustre/tasks/install.yml b/ansible/roles/lustre/tasks/install.yml new file mode 100644 index 000000000..e0af857cf --- /dev/null +++ b/ansible/roles/lustre/tasks/install.yml @@ -0,0 +1,70 @@ +- name: Install lustre build prerequisites + ansible.builtin.dnf: + name: "{{ lustre_build_packages }}" + register: _lustre_dnf_build_packages + +- name: Clone lustre git repo + # https://git.whamcloud.com/?p=fs/lustre-release.git;a=summary + ansible.builtin.git: + repo: git://git.whamcloud.com/fs/lustre-release.git + dest: "{{ lustre_build_dir }}" + version: "{{ lustre_version }}" + +- name: Prepare for lustre configuration + ansible.builtin.command: + cmd: sh ./autogen.sh + chdir: "{{ lustre_build_dir }}" + +- name: Configure lustre build + ansible.builtin.command: + cmd: "./configure {{ lustre_configure_opts | join(' ') }}" + chdir: "{{ lustre_build_dir }}" + +- name: Build lustre + ansible.builtin.command: + cmd: make rpms + chdir: "{{ lustre_build_dir }}" + +- name: Find rpms + ansible.builtin.find: + paths: "{{ lustre_build_dir }}" + patterns: "{{ lustre_rpm_globs }}" + use_regex: false + register: _lustre_find_rpms + +- name: Check rpms found + assert: + that: _lustre_find_rpms.files | length + fail_msg: "No lustre repos found with lustre_rpm_globs = {{ lustre_rpm_globs }}" + +- name: Install lustre rpms + ansible.builtin.dnf: + name: "{{ _lustre_find_rpms.files | map(attribute='path')}}" + disable_gpg_check: yes + +- block: + - name: Remove lustre build prerequisites + # NB Only remove ones this role installed which weren't upgrades + ansible.builtin.dnf: + name: "{{ _new_pkgs }}" + state: absent + vars: + _installed_pkgs: | + {{ + _lustre_dnf_build_packages.results | + select('match', 'Installed:') | + map('regex_replace', '^Installed: (.+?)-[0-9].*$', '\1') + }} + _removed_pkgs: | + {{ + _lustre_dnf_build_packages.results | + select('match', 'Removed:') | + map('regex_replace', '^Removed: (.+?)-[0-9].*$', '\1') + }} + _new_pkgs: "{{ _installed_pkgs | difference(_removed_pkgs) }}" + + - name: Delete lustre build dir + file: + path: "{{ lustre_build_dir }}" + state: absent + when: lustre_build_cleanup | bool diff --git a/ansible/roles/lustre/tasks/validate.yml b/ansible/roles/lustre/tasks/validate.yml new file mode 100644 index 000000000..fe65a4d1a --- /dev/null +++ b/ansible/roles/lustre/tasks/validate.yml @@ -0,0 +1,27 @@ +- name: Assert using RockyLinux 9 + assert: + that: ansible_distribution_major_version | int == 9 + fail_msg: The 'lustre' role requires RockyLinux 9 + +- name: Check kernel-devel package is installed + command: "dnf list --installed kernel-devel-{{ ansible_kernel }}" + changed_when: false + # NB: we don't check here the kernel will remain the same after reboot etc, see ofed/install.yml + +- name: Ensure SELinux in permissive mode + assert: + that: selinux_state in ['permissive', 'disabled'] + fail_msg: "SELinux must be permissive for Lustre not '{{ selinux_state }}'; see variable selinux_state" + +- name: Ensure lustre_mgs_nid is defined + assert: + that: lustre_mgs_nid is defined + fail_msg: Variable lustre_mgs_nid must be defined + +- name: Ensure lustre_mounts entries define filesystem name and mount point + assert: + that: + - item.fs_name is defined + - item.mount_point is defined + fail_msg: All lustre_mounts entries must specify fs_name and mount_point + loop: "{{ lustre_mounts }}" diff --git a/ansible/roles/lustre/templates/lnet.conf.j2 b/ansible/roles/lustre/templates/lnet.conf.j2 new file mode 100644 index 000000000..363308e32 --- /dev/null +++ b/ansible/roles/lustre/templates/lnet.conf.j2 @@ -0,0 +1,6 @@ +net: + - net type: {{ lustre_lnet_label }} + local NI(s): + - nid: {{ _lustre_ip }}@{{ lustre_lnet_label }} + interfaces: + 0: {{ _lustre_interface }} diff --git a/ansible/validate.yml b/ansible/validate.yml index fae9c2f68..d02caac60 100644 --- a/ansible/validate.yml +++ b/ansible/validate.yml @@ -85,3 +85,11 @@ - import_role: name: freeipa tasks_from: validate.yml + +- name: Validate lustre configuration + hosts: lustre + tags: lustre + tasks: + - import_role: + name: lustre + tasks_from: validate.yml diff --git a/docs/image-build.md b/docs/image-build.md new file mode 100644 index 000000000..4896bde57 --- /dev/null +++ b/docs/image-build.md @@ -0,0 +1,113 @@ +# Packer-based image build + +The appliance contains code and configuration to use [Packer](https://developer.hashicorp.com/packer) with the [OpenStack builder](https://www.packer.io/plugins/builders/openstack) to build images. + +The Packer configuration defined here builds "fat images" which contain binaries for all nodes, but no cluster-specific configuration. Using these: +- Enables the image to be tested in CI before production use. +- Ensures re-deployment of the cluster or deployment of additional nodes can be completed even if packages are changed in upstream repositories (e.g. due to RockyLinux or OpenHPC updates). +- Improves deployment speed by reducing the number of package downloads to improve deployment speed. + +By default, a fat image build starts from a nightly image build containing Mellanox OFED, and updates all DNF packages already present. The 'latest' nightly build itself is from a RockyLinux GenericCloud image. + +The fat images StackHPC builds and test in CI are available from [GitHub releases](https://github.com/stackhpc/ansible-slurm-appliance/releases). However with some additional configuration it is also possible to: +1. Build site-specific fat images from scratch. +2. Extend an existing fat image with additional software. + + +# Usage + +The steps for building site-specific fat images or extending an existing fat image are the same: + +1. Ensure the current OpenStack credentials have sufficient authorisation to upload images (this may or may not require the `member` role for an application credential, depending on your OpenStack configuration). +2. Create a Packer [variable definition file](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/variables#assigning-values-to-input-variables) at e.g. `environments//builder.pkrvars.hcl` containing at a minimum e.g.: + + ```hcl + flavor = "general.v1.small" # VM flavor to use for builder VMs + networks = ["26023e3d-bc8e-459c-8def-dbd47ab01756"] # List of network UUIDs to attach the VM to + ``` + Note that: + - The network used for the Packer VM must provide outbound internet access but does not need to provide access to resources which the final cluster nodes require (e.g. Slurm control node, network filesystem servers etc.). + - For additional options such as non-default private key locations or jumphost configuration see the variable descriptions in `./openstack.pkr.hcl`. + - For an example of configuration for extending an existing fat image see below. + +3. Activate the venv and the relevant environment. + +4. Build images using the relevant variable definition file, e.g.: + + cd packer/ + PACKER_LOG=1 /usr/bin/packer build -only=openstack.openhpc --on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl + + Note that the `-only` flag here restricts Packer to a single specific "build" definition (in Packer terminology). Options here are: + - `-only=openstack.openhpc`: Build a fat image including Mellanox OFED + - `-only=openstack.openhpc-cuda`: Build a fat image including Mellanox OFED, Nvidia drivers and CUDA + - `-only=openstack.openhpc-extra`: Build an image which *extends* an existing fat image + +5. The built image will be automatically uploaded to OpenStack with a name prefixed `openhpc-` and including a timestamp and a shortened git hash. + +# Defining an "extra" image build + +An "extra" image build starts with an existing fat image (e.g. one provided by StackHPC) rather than a RockyLinux GenericCloud image, and only runs a specific subset of the +Ansible in the appliance. This allows adding additional functionality into site-specific images, without modifying the existing functionality in the base fat image. This is the recommended way to build site-specific images. + +To configure an "extra" image build, prepare a Packer variable definition file as described above but also including: + +- `extra_build_image_name`: A string to add into the final image name. +- `source_image` or `source_image_name`: The UUID or name of the fat image to start from (which must already be present in OpenStack). +- `extra_build_groups`: A list of Ansible inventory groups to put the build VM into, in addition to the `builder` group. This defines the roles/functionality + which are added to the image. +- `extra_build_volume_size`: A number giving the size in GB of the volume for the build VM's root disk and therefore the resulting image size. + Note this assumes the default of `use_blockstorage_volume = true`. + +E.g. to add the lustre client to an RockyLinux 9 image: + + # environments/site/lustre.pkvars.hcl + + extra_build_image_name = "lustre" # output image name will be like "openhpc-lustre-RL9-$timestamp-$commit" + source_image_name = "openhpc-ofed-RL9-240906-1041-32568dbb" # e.g. current StackHPC RL9 image + extra_build_groups = ["lustre"] # only run lustre role during this extra build + extra_build_volume_size = 15 # default non-CUDA build image size has enough free space + + # ... define flavor, network, etc as normal + + +Then, reference this build and variables file in the Packer build command: + + PACKER_LOG=1 /usr/bin/packer build -only=openstack.openhpc-extra --on-error=ask -var-file=environments/site/lustre.pkvars.hcl openstack.pkr.hcl + +**NB:** If the build fails while creating the volume, check if the source image has the `signature_verified` property: + + openstack image show $SOURCE_IMAGE + +If it does, remove this property: + + openstack image unset --property signature_verified $SOURCE_IMAGE + +then delete the failed volume, select cancelling the build when Packer queries, and then retry. This is [Openstack bug 1823445](https://bugs.launchpad.net/cinder/+bug/1823445). + +# Build Process + +In summary, Packer creates an OpenStack VM, runs Ansible on that, shuts it down, then creates an image from the root disk. + +Many of the Packer variables defined in `openstack.pkr.hcl` control the definition of the build VM and how to SSH to it to run Ansible. These are generic OpenStack builder options +and are not specific to the Slurm Appliance. Packer varibles can be set in a file at any convenient path; the build example above +shows the use of the environment variable `$PKR_VAR_environment_root` (which itself sets the Packer variable +`environment_root`) to automatically select a variable file from the current environment, but for site-specific builds +using a path in a "parent" environment is likely to be more appropriate (as builds should not be environment-specific to allow testing before deployment to a production environment). + +What is Slurm Appliance-specific are the details of how Ansible is run: +- The build VM is always added to the `builder` inventory group, which differentiates it from nodes in a cluster. This allows + Ansible variables to be set differently during Packer builds, e.g. to prevent services starting. The defaults for this are in `environments/common/inventory/group_vars/builder/`, which could be extended or overriden for site-specific fat image builds using `builder` groupvars for the relevant environment. It also runs some builder-specific code (e.g. to clean up the image). +- The default fat image builds also add the build VM to the "top-level" `compute`, `control` and `login` groups. This ensures + the Ansible specific to all of these types of nodes run. Note other inventory groups are constructed from these by `environments/common/inventory/groups file` - this is not builder-specific. +- As noted above, for "extra" builds the additional groups can be specified directly. In this way an existing image can be extended with site-specific Ansible, without modifying the + part of the image which has already been tested in the StackHPC CI. +- The playbook `ansible/fatimage.yml` is run which is only a subset of `ansible/site.yml`. This allows restricting the code which runs during build for cases where setting `builder` + groupvars is not sufficient (e.g. a role always attempts to configure or start services). + +There are some things to be aware of when developing Ansible to run in a Packer build VM: + - Only some tasks make sense. E.g. any services with a reliance on the network cannot be started, and should not be enabled if, when creating an instance with the resulting image, the remote service will not be immediately present. + - Nothing should be written to the persistent state directory `appliances_state_dir`, as this is on the root filesystem rather than an OpenStack volume. + - Care should be taken not to leave data on the root filesystem which is not wanted in the final image (e.g secrets). + - Build VM hostnames are not the same as for equivalent "real" hosts and do not contain `login`, `control` etc. Therefore variables used by the build VM must be defined as groupvars not hostvars. + - Ansible may need to use a proxyjump to reach cluster nodes, which can be defined via Ansible's `ansible_ssh_common_args` variable. If Packer should not use the same proxy + to connect to build VMs (e.g. because build happens on a different network), this proxy configuration should not be added to the `all` group. diff --git a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json index c07b2c4ac..9f396e964 100644 --- a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json +++ b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json @@ -1,7 +1,7 @@ { "cluster_image": { - "RL8": "openhpc-RL8-241022-0441-a5affa58", - "RL9": "openhpc-RL9-241022-0038-a5affa58", - "RL9-cuda": "openhpc-cuda-RL9-241022-0441-a5affa58" + "RL8": "openhpc-RL8-241024-1439-177083b1", + "RL9": "openhpc-RL9-241024-1438-177083b1", + "RL9-cuda": "openhpc-cuda-RL9-241024-1628-177083b1" } } \ No newline at end of file diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups index ea0bebebc..cd9a5cb0c 100644 --- a/environments/common/inventory/groups +++ b/environments/common/inventory/groups @@ -134,4 +134,7 @@ freeipa_client # Hosts to run TuneD configuration [ansible_init] -# Hosts to run linux-anisble-init \ No newline at end of file +# Hosts to run linux-anisble-init + +[lustre] +# Hosts to run lustre client diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything index 205f1d334..be0b3d1b7 100644 --- a/environments/common/layouts/everything +++ b/environments/common/layouts/everything @@ -80,4 +80,7 @@ openhpc [ansible_init:children] # Hosts to run ansible-init -cluster \ No newline at end of file +cluster + +[lustre] +# Hosts to run lustre client diff --git a/packer/README.md b/packer/README.md deleted file mode 100644 index 5e1d57dc2..000000000 --- a/packer/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Packer-based image build - -The appliance contains code and configuration to use [Packer](https://developer.hashicorp.com/packer) with the [OpenStack builder](https://www.packer.io/plugins/builders/openstack) to build images. - -The Packer configuration defined here builds "fat images" which contain binaries for all nodes, but no cluster-specific configuration. Using these: -- Enables the image to be tested in CI before production use. -- Ensures re-deployment of the cluster or deployment of additional nodes can be completed even if packages are changed in upstream repositories (e.g. due to RockyLinux or OpenHPC updates). -- Improves deployment speed by reducing the number of package downloads to improve deployment speed. - -By default, a fat image build starts from a nightly image build containing Mellanox OFED, and updates all DNF packages already present. The 'latest' nightly build itself is from a RockyLinux GenericCloud image. - -The fat images StackHPC builds and test in CI are available from [GitHub releases](https://github.com/stackhpc/ansible-slurm-appliance/releases). However with some additional configuration it is also possible to: -1. Build site-specific fat images from scratch. -2. Extend an existing fat image with additional software. - - -# Usage - -The steps for building site-specific fat images or extending an existing fat image are the same: - -1. Ensure the current OpenStack credentials have sufficient authorisation to upload images (this may or may not require the `member` role for an application credential, depending on your OpenStack configuration). -2. Create a Packer [variable definition file](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/variables#assigning-values-to-input-variables) at e.g. `environments//builder.pkrvars.hcl` containing at a minimum e.g.: - - ```hcl - flavor = "general.v1.small" # VM flavor to use for builder VMs - networks = ["26023e3d-bc8e-459c-8def-dbd47ab01756"] # List of network UUIDs to attach the VM to - ``` - - - The network used for the Packer VM must provide outbound internet access but does not need to provide access to resources which the final cluster nodes require (e.g. Slurm control node, network filesystem servers etc.). - - - For additional options such as non-default private key locations or jumphost configuration see the variable descriptions in `./openstack.pkr.hcl`. - - - For an example of configuration for extending an existing fat image see below. - -3. Activate the venv and the relevant environment. - -4. Build images using the relevant variable definition file, e.g.: - - cd packer/ - PACKER_LOG=1 /usr/bin/packer build -only=openstack.openhpc --on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl - - Note that the `-only` flag here restricts the build to the non-CUDA fat image "source" (in Packer terminology). Other - source options are: - - `-only=openstack.openhpc-cuda`: Build a fat image including CUDA packages. - - `-only=openstack.openhpc-extra`: Build an image which extends an existing fat image - in this case the variable `source_image` or `source_image_name}` must also be set in the Packer variables file. - -5. The built image will be automatically uploaded to OpenStack with a name prefixed `openhpc-` and including a timestamp and a shortened git hash. - -# Build Process - -In summary, Packer creates an OpenStack VM, runs Ansible on that, shuts it down, then creates an image from the root disk. - -Many of the Packer variables defined in `openstack.pkr.hcl` control the definition of the build VM and how to SSH to it to run Ansible, which are generic OpenStack builder options. Packer varibles can be set in a file at any convenient path; the above -example shows the use of the environment variable `$PKR_VAR_environment_root` (which itself sets the Packer variable -`environment_root`) to automatically select a variable file from the current environment, but for site-specific builds -using a path in a "parent" environment is likely to be more appropriate (as builds should not be environment-specific, to allow testing). - -What is Slurm Appliance-specific are the details of how Ansible is run: -- The build VM is always added to the `builder` inventory group, which differentiates it from "real" nodes. This allows - variables to be set differently during Packer builds, e.g. to prevent services starting. The defaults for this are in `environments/common/inventory/group_vars/builder/`, which could be extended or overriden for site-specific fat image builds using `builder` groupvars for the relevant environment. It also runs some builder-specific code (e.g. to ensure Packer's SSH - keys are removed from the image). -- The default fat image build also adds the build VM to the "top-level" `compute`, `control` and `login` groups. This ensures - the Ansible specific to all of these types of nodes run (other inventory groups are constructed from these by `environments/common/inventory/groups file` - this is not builder-specific). -- Which groups the build VM is added to is controlled by the Packer `groups` variable. This can be redefined for builds using the `openhpc-extra` source to add the build VM into specific groups. E.g. with a Packer variable file: - - source_image_name = { - RL9 = "openhpc-ofed-RL9-240619-0949-66c0e540" - } - groups = { - openhpc-extra = ["foo"] - } - - the build VM uses an existing "fat image" (rather than a 'latest' nightly one) and is added to the `builder` and `foo` groups. This means only code targeting `builder` and `foo` groups runs. In this way an existing image can be extended with site-specific code, without modifying the part of the image which has already been tested in the StackHPC CI. - - - The playbook `ansible/fatimage.yml` is run which is only a subset of `ansible/site.yml`. This allows restricting the code - which runs during build for cases where setting `builder` groupvars is not sufficient (e.g. a role always attempts to configure or start services). This may eventually be removed. - -There are some things to be aware of when developing Ansible to run in a Packer build VM: - - Only some tasks make sense. E.g. any services with a reliance on the network cannot be started, and may not be able to be enabled if when creating an instance with the resulting image the remote service will not be immediately present. - - Nothing should be written to the persistent state directory `appliances_state_dir`, as this is on the root filesystem rather than an OpenStack volume. - - Care should be taken not to leave data on the root filesystem which is not wanted in the final image, (e.g secrets). - - Build VM hostnames are not the same as for equivalent "real" hosts and do not contain `login`, `control` etc. Therefore variables used by the build VM must be defined as groupvars not hostvars. - - Ansible may need to proxy to real compute nodes. If Packer should not use the same proxy to connect to the - build VMs (e.g. build happens on a different network), proxy configuration should not be added to the `all` group. - - Currently two fat image "sources" are defined, with and without CUDA. This simplifies CI configuration by allowing the - default source images to be defined in the `openstack.pkr.hcl` definition. diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl index fe922c78e..fae0bf7b2 100644 --- a/packer/openstack.pkr.hcl +++ b/packer/openstack.pkr.hcl @@ -48,6 +48,7 @@ variable "os_version" { # Must supply either source_image_name or source_image_id variable "source_image_name" { type = string + default = null description = "name of source image" } @@ -132,6 +133,11 @@ variable "volume_size" { } } +variable "extra_build_volume_size" { + type = number + default = 15 # same as default non-CUDA build +} + variable "image_disk_format" { type = string default = "qcow2" @@ -154,12 +160,23 @@ variable "groups" { } } +variable "extra_build_groups" { + type = list(string) + default = [] +} + +variable "extra_build_image_name" { + type = string + description = "Infix for 'extra' build image name" + default = "extra" +} + source "openstack" "openhpc" { # Build VM: flavor = var.flavor use_blockstorage_volume = var.use_blockstorage_volume volume_type = var.volume_type - volume_size = var.volume_size[source.name] + volume_size = lookup(var.volume_size, source.name, var.extra_build_volume_size) metadata = var.metadata instance_metadata = {ansible_init_disable = "true"} networks = var.networks @@ -214,12 +231,12 @@ build { # Extended site-specific image, built on fat image: source "source.openstack.openhpc" { name = "openhpc-extra" - image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}" + image_name = "openhpc-${var.extra_build_image_name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}" } provisioner "ansible" { playbook_file = "${var.repo_root}/ansible/fatimage.yml" - groups = concat(["builder"], var.groups[source.name]) + groups = concat(["builder"], lookup(var.groups, source.name, var.extra_build_groups)) keep_inventory_file = true # for debugging use_proxy = false # see https://www.packer.io/docs/provisioners/ansible#troubleshooting extra_arguments = [