Merge pull request #33 from stackhpc/feat/image-build

m-bull · web-flow · commit d06f4157f21c · 2023-06-26T14:51:51.000+01:00
Add an OpenHPC image build playbook and role
diff --git a/.gitignore b/.gitignore
@@ -7,4 +7,6 @@
 inventory.*
 __pycache__
 terraform/backend.tf
-inventory
+inventory
+venv
+bin
diff --git a/ansible.cfg b/ansible.cfg
@@ -6,7 +6,8 @@ host_key_checking = False
 remote_tmp = /tmp
 # Enable our custom vars plugin that parses variables from the current working directory
 vars_plugins_enabled = host_group_vars,cwd_host_group_vars
-roles_path = roles:vendor/stackhpc/ansible-slurm-appliance/ansible/roles
+roles_path = vendor/stackhpc/ansible-slurm-appliance/ansible/roles:roles
+callbacks_enabled = ansible.posix.profile_tasks
 
 [ssh_connection]
 ssh_args = -o ControlMaster=auto -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null
diff --git a/image-build.yml b/image-build.yml
@@ -0,0 +1,24 @@
+---
+- name: Build image
+  hosts: openstack
+  tasks:
+    - name: Manage image build infra
+      include_role:
+        name: image_build_infra
+      when:
+        - image_build_manage_infra is defined
+        - image_build_manage_infra
+
+    - block:
+      - name: Build fat image
+        include_role:
+          name: image_build
+
+      - name: Set cluster_image fact
+        set_fact:
+          cluster_image: "{{ image_build_data.artifact_id }}"
+
+      - name: Print cluster_image UUID
+        debug:
+          msg: "{{ cluster_image }}"
+      when: cluster_state is not defined or (cluster_state is defined and cluster_state != "absent")
diff --git a/image-build/hosts b/image-build/hosts
@@ -0,0 +1,2 @@
+[openstack]
+localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}"
diff --git a/requirements.yml b/requirements.yml
@@ -3,7 +3,7 @@ roles:
   - src: stackhpc.nfs
     version: v22.9.1
   - src: https://github.com/stackhpc/ansible-role-openhpc.git
-    version: v0.18.0 # requires/uses openhpc v2.6.1
+    version: v0.20.0 # Allow multiple empty partitions by @sjpb in #156
     name: stackhpc.openhpc
   - src: https://github.com/stackhpc/ansible-node-exporter.git
     version: feature/no-install
@@ -33,11 +33,9 @@ collections:
     version: 4.5.0  # https://github.com/ansible-collections/community.general/pull/4281
   - name: community.crypto
     version: 2.10.0
-  - name: community.mysql
-
   - name: openstack.cloud
     version: 1.10.0
   - name: https://github.com/stackhpc/ansible-collection-terraform
     type: git
-    version: 75fb75132bbc77e3e78a05ba674458131da2b1dd
-    
+    version: dadf2e81b78cfd267821c568f92d3a8da6541a41
+
diff --git a/roles/cluster_infra/tasks/main.yml b/roles/cluster_infra/tasks/main.yml
@@ -73,8 +73,9 @@
               else 'vd'
            }}
   # Only run when block_device_prefix isn't set as an extravar
-  when: block_device_prefix is not defined
-
+  when:
+    - block_device_prefix is not defined
+    - cluster_image is defined
 
 - name: Template Terraform files into project directory
   template:
diff --git a/roles/image_build/defaults/main.yml b/roles/image_build/defaults/main.yml
@@ -0,0 +1,38 @@
+---
+# Attach a floating IP to the Packer build instance
+image_build_attach_floating_ip: false
+
+# Use a volume for the root disk of the Packer build instance
+image_build_use_blockstorage_volume: false
+
+# Packer image format (only used when image_build_use_blockstorage_volume: true
+image_build_image_disk_format: "qcow2"
+
+# Metadata items to set on the Packer image
+image_build_metadata: {}
+
+# The directory that contains the openstack.pkr.hcl to build the Slurm image
+image_build_packer_root_path: "{{ playbook_dir }}/vendor/stackhpc/ansible-slurm-appliance/packer"
+
+# The appliances_environment_root directory. This may contain a hooks directory
+# optionally containing pre.yml, post-bootstrap.yml and post.yml playbooks, to
+# run during the image-build process
+image_build_appliances_environment_root: "{{ playbook_dir }}/image-build"
+
+# Vars to apply to the builder group
+image_build_builder_group_vars:
+  update_log_path: /tmp/update_log
+  appliances_repository_root: "{{ playbook_dir }}/vendor/stackhpc/ansible-slurm-appliance"
+
+# ansible_ssh_common_args for Packer build
+image_build_ansible_ssh_common_args: >-
+    {% if image_build_ssh_bastion_host is defined %}
+    '-o ProxyCommand="ssh -W %h:%p -q
+    {% if image_build_ssh_bastion_private_key_file is defined %}
+    -i {{ image_build_ssh_bastion_private_key_file }}
+    {% endif %}
+    -l {{ image_build_ssh_bastion_username }}
+    {{ image_build_ssh_bastion_host }}"'
+    {% else %}
+    ""
+    {% endif %}
diff --git a/roles/image_build/tasks/main.yml b/roles/image_build/tasks/main.yml
@@ -0,0 +1,84 @@
+---
+
+- name: Run prechecks
+  include_tasks: prechecks.yml
+
+- name: Create temporary file for pkrvars.hcl
+  ansible.builtin.tempfile:
+    state: file
+    suffix: .pkrvars.hcl
+  register: pkrvars_hcl_file
+
+- name: Make Packer vars file
+  template:
+    src: builder.pkrvars.hcl.j2
+    dest: "{{ pkrvars_hcl_file.path }}"
+
+- name: Create temporary image-build inventory directory
+  ansible.builtin.tempfile:
+    state: directory
+    prefix: image-build.
+  register: image_build_inventory
+
+- name: Symlink "everything" layout to image-build inventory
+  file:
+    state: link
+    src: "{{ playbook_dir }}/vendor/stackhpc/ansible-slurm-appliance/environments/common/layouts/everything"
+    dest: "{{ image_build_inventory.path }}/groups"
+
+- name: Symlink CAAS group_vars to image-build inventory
+  file:
+    state: link
+    src: "{{ playbook_dir }}/group_vars"
+    dest: "{{ image_build_inventory.path }}/group_vars"
+
+- name: Add builder vars to image-build inventory hosts file
+  copy:
+    dest: "{{ image_build_inventory.path }}/hosts"
+    content: |
+      {% raw %}
+      localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}"
+      {% endraw %}
+      [builder:vars]
+      {% if image_build_ssh_bastion_host is defined %}
+      ansible_ssh_common_args={{ image_build_ansible_ssh_common_args }}
+      {% endif %}
+      {% for k,v in image_build_builder_group_vars.items() -%}
+      {{ k }}={{ v }}
+      {% endfor -%}
+
+- name: Create temporary file for ansible.cfg
+  ansible.builtin.tempfile:
+    state: file
+    suffix: ansible.cfg
+  register: ansible_cfg_file
+
+- name: Template image-build ansible.cfg
+  template:
+    src: ansible.cfg.j2
+    dest: "{{ ansible_cfg_file.path }}"
+
+- name: Packer init
+  command:
+    cmd: |
+      packer init .
+    chdir: "{{ image_build_packer_root_path }}"
+
+- name: Build image with packer
+  command:
+    cmd: |
+      packer build -only openstack.openhpc -var-file={{ pkrvars_hcl_file.path }} openstack.pkr.hcl
+    chdir: "{{ image_build_packer_root_path }}"
+  environment:
+    APPLIANCES_ENVIRONMENT_ROOT: "{{ image_build_appliances_environment_root }}"
+    ANSIBLE_CONFIG: "{{ ansible_cfg_file.path }}"
+    PACKER_LOG: "1"
+    PACKER_LOG_PATH: "{{ lookup('ansible.builtin.env', 'PACKER_LOG_PATH', default='/tmp/packer-build.log') }}"
+
+- name: Parse packer-manifest.json
+  set_fact:
+    packer_manifest: "{{ lookup('file', '/tmp/builder.manifest.json') | from_json }}"
+
+- name: Extract image-build data
+  set_fact:
+    image_build_data: "{{ packer_manifest.builds | selectattr('packer_run_uuid', 'eq', packer_manifest.last_run_uuid) | first }}"
diff --git a/roles/image_build/tasks/prechecks.yml b/roles/image_build/tasks/prechecks.yml
@@ -0,0 +1,22 @@
+---
+
+- name: Check required vars are defined
+  assert:
+    that:
+      - "{{ item }} is defined"
+    fail_msg: "{{ item }} is not defined"
+  loop:
+    - image_build_network_id
+    - image_build_floating_ip_network
+    - image_build_source_image_id
+    - image_build_security_group_id
+
+- name: Ensure builder access mode
+  fail:
+    msg: >- 
+      Set either image_build_ssh_bastion_host or 
+      image_build_attach_floating_ip to access the image
+      build instance via a bastion or directly
+  when:
+    - image_build_ssh_bastion_host is defined 
+    - image_build_attach_floating_ip is defined and image_build_attach_floating_ip
diff --git a/roles/image_build/templates/ansible.cfg.j2 b/roles/image_build/templates/ansible.cfg.j2
@@ -0,0 +1,15 @@
+[defaults]
+any_errors_fatal = True
+gathering = smart
+host_key_checking = False
+remote_tmp = /tmp
+roles_path = {{ playbook_dir }}/vendor/stackhpc/ansible-slurm-appliance/ansible/roles
+inventory = {{ playbook_dir }}/vendor/stackhpc/ansible-slurm-appliance/environments/common/inventory,{{ image_build_inventory.path }}
+
+[ssh_connection]
+ssh_args = -o ControlMaster=auto -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null
+pipelining = True
+# This is important because we are using one of the hosts in the play as a jump host
+# This ensures that if the proxy connection is interrupted, rendering the other hosts
+# unreachable, the connection is retried instead of failing the entire play
+retries = 10
diff --git a/roles/image_build/templates/builder.pkrvars.hcl.j2 b/roles/image_build/templates/builder.pkrvars.hcl.j2
@@ -0,0 +1,38 @@
+repo_root = "{{ playbook_dir }}/vendor/stackhpc/ansible-slurm-appliance"
+environment_root = "{{ playbook_dir }}/image_build"
+networks = ["{{ image_build_network_id }}"]
+{% if image_build_ssh_username is defined %}
+ssh_username = "{{ image_build_ssh_username }}"
+{% endif %}
+{% if image_build_ssh_bastion_host is defined %}
+ssh_bastion_host = "{{ image_build_ssh_bastion_host }}"
+{% endif %}
+{% if image_build_ssh_bastion_username is defined %}
+ssh_bastion_username = "{{ image_build_ssh_bastion_username }}"
+{% endif %}
+{% if image_build_ssh_bastion_private_key_file is defined %}
+ssh_bastion_private_key_file = "{{ image_build_ssh_bastion_private_key_file }}"
+{% endif %}
+{% if image_build_attach_floating_ip %}
+floating_ip_network = "{{ image_build_floating_ip_network }}"
+{% endif %}
+security_groups = ["{{ image_build_security_group_id }}"]
+fatimage_source_image = "{{ image_build_source_image_id }}"
+{% if image_build_ssh_keypair_name is defined %}
+ssh_keypair_name = "{{ image_build_ssh_keypair_name }}"
+{% endif %}
+{% if image_build_ssh_private_key_file is defined %}
+ssh_private_key_file = "{{ image_build_ssh_private_key_file }}"
+{% endif %}
+flavor = "{{ image_build_flavor_name }}"
+metadata = {
+{% for k,v in image_build_metadata.items() %}
+  "{{ k }}" = "{{ v }}"
+{% endfor %}
+}
+use_blockstorage_volume = {{ image_build_use_blockstorage_volume | string | lower }}
+{% if image_build_use_blockstorage_volume %}
+volume_size = {{ image_build_volume_size }}
+image_disk_format = "{{ image_build_image_disk_format }}"
+{% endif %}
+manifest_output_path = "/tmp/builder.manifest.json"
diff --git a/roles/image_build_infra/defaults/main.yml b/roles/image_build_infra/defaults/main.yml
@@ -0,0 +1,12 @@
+---
+
+image_build_terraform_project_path: "{{ playbook_dir }}/terraform-caas-image-build"
+image_build_cluster_name: "caas-image-build"
+
+# Regex to capture existing cloud image names to use as the
+# OpenHPC Slurm base-image
+image_build_existing_image_regex: "^Rocky-8-GenericCloud-Base-8.8-.*"
+# Attributes to sort the list of existing base images returned by
+# image_build_existing_image_regex. See
+# https://registry.terraform.io/providers/terraform-provider-openstack/openstack/latest/docs/data-sources/images_image_ids_v2#sort
+image_build_existing_image_sort_attributes: "name,updated_at"
diff --git a/roles/image_build_infra/tasks/main.yml b/roles/image_build_infra/tasks/main.yml
@@ -0,0 +1,45 @@
+---
+- name: Install Terraform binary
+  include_role:
+    name: stackhpc.terraform.install
+
+- name: Make Terraform project directory
+  file:
+    path: "{{ image_build_terraform_project_path }}"
+    state: directory
+
+- name: Write backend configuration
+  copy:
+    content: |
+      terraform {
+        backend "{{ terraform_backend_type }}" { }
+      }
+    dest: "{{ image_build_terraform_project_path }}/backend.tf"
+
+- name: Template Terraform files into project directory
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ image_build_terraform_project_path }}/{{ item }}"
+  loop:
+    - outputs.tf
+    - providers.tf
+    - resources.tf
+
+- name: Provision infrastructure using Terraform
+  terraform:
+    binary_path: "{{ terraform_binary_path or omit }}"
+    project_path: "{{ image_build_terraform_project_path }}"
+    state: "{{ terraform_state }}"
+    backend_config: "{{ terraform_backend_config }}"
+    force_init: yes
+    init_reconfigure: yes
+    variables: "{{ image_build_terraform_variables | default(omit) }}"
+  register: image_build_terraform_provision
+
+- name: Set image build infrastructure facts
+  set_fact:
+    image_build_network_id: "{{ image_build_terraform_provision.outputs.network_id.value }}"
+    image_build_floating_ip_network: "{{ image_build_terraform_provision.outputs.floating_ip_network_id.value }}"
+    image_build_source_image_id: "{{ image_build_terraform_provision.outputs.source_image_name.value }}"
+    image_build_security_group_id: "{{ image_build_terraform_provision.outputs.security_group_id.value }}"
+  when: cluster_state is not defined or (cluster_state is defined and cluster_state != "absent")
diff --git a/roles/image_build_infra/templates/outputs.tf.j2 b/roles/image_build_infra/templates/outputs.tf.j2
@@ -0,0 +1,27 @@
+output "network_id" {
+  description = "The image build network ID"
+  value       = data.openstack_networking_network_v2.caas_image_build_network.id
+}
+
+output "source_image_name" {
+  description = "The id of the image used to build the cluster nodes"
+  {% if image_build_source_image_id is defined %}
+  value       = "{{ image_build_source_image_id }}"
+  {% else %}
+  value       = data.openstack_images_image_ids_v2.image_build_source_image.ids[0]
+  {% endif %}
+}
+
+output "floating_ip_network_id" {
+  description = "Network to allocate floating IPs from"
+  value = data.openstack_networking_network_v2.caas_image_build_external_network.id
+}
+
+output "security_group_id" {
+  description = "Security group ID to associate with the builder instance"
+  {% if image_build_security_group_id is defined %}
+  value = "{{ image_build_security_group_id }}"
+  {% else %}
+  value = openstack_networking_secgroup_v2.caas_image_build_secgroup.id
+  {% endif %}
+}
diff --git a/roles/image_build_infra/templates/providers.tf.j2 b/roles/image_build_infra/templates/providers.tf.j2
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 0.14"
+
+  # We need the OpenStack provider
+  required_providers {
+    openstack = {
+      source = "terraform-provider-openstack/openstack"
+    }
+  }
+}
diff --git a/roles/image_build_infra/templates/resources.tf.j2 b/roles/image_build_infra/templates/resources.tf.j2
diff --git a/vendor/stackhpc/ansible-slurm-appliance b/vendor/stackhpc/ansible-slurm-appliance

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+[openstack]`
	`2`	`+localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}"`