Merge branch 'main' into feat/ephemeral-ssh-keys

Author: m-bull

.github/workflows/stackhpc.yml

@@ -66,24 +66,13 @@ jobs:
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform apply -auto-approve
 
-      - name: Get server provisioning failure messages
-        id: provision_failure
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
-          echo TF failure messages: $TF_FAIL_MSGS
-          echo "::set-output name=messages::${TF_FAIL_MSGS}"
-        if: always() && steps.provision_servers.outcome == 'failure'
-        
       - name: Delete infrastructure if failed due to lack of hosts
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
-        if: ${{ always() && steps.provision_servers.outcome == 'failure' && contains(steps.provision_failure.messages, 'not enough hosts available') }}
+        if: ${{ steps.provision_servers.outcome == 'failure' }}
 
       - name: Configure cluster
         run: |

ansible/roles/filebeat/defaults/main.yml

@@ -3,3 +3,4 @@
 #filebeat_config_path: undefined # REQUIRED. Path to filebeat.yml configuration file template
 filebeat_podman_user: "{{ ansible_user }}" # User that runs the filebeat container
 filebeat_version: 7.12.1 # latest usable with opensearch - see https://opensearch.org/docs/2.4/tools/index/#compatibility-matrix-for-beats
+filebeat_debug: false

ansible/roles/opensearch/defaults/main.yml

@@ -9,3 +9,4 @@ opensearch_data_path: /usr/share/opensearch/data
 opensearch_state: started # will be restarted if required
 opensearch_systemd_service_enabled: true
 opensearch_certs_duration: "{{ 365 * 10 }}" # days validity for self-signed certs
+opensearch_debug: false

ansible/roles/opensearch/templates/opensearch.yml.j2

@@ -32,3 +32,5 @@ plugins.security.system_indices.enabled: true
 
 # Fake version for filebeat: https://opensearch.org/docs/2.4/tools/index/#agents-and-ingestion-tools
 compatibility.override_main_response_version: true
+
+{% if opensearch_debug | default(false) | bool %}logger.level: debug{% endif %}

ansible/roles/proxy/defaults/main.yml

@@ -1,5 +1,5 @@
 # proxy_http_proxy: 
 proxy_https_proxy: "{{ proxy_http_proxy }}"
-proxy_no_proxy: "{{ (groups['all'] + hostvars.values() | map(attribute='ansible_host')) | sort | join(',') }}"
+proxy_no_proxy: "{{ (['localhost', '127.0.0.1'] + groups['all'] + hostvars.values() | map(attribute='ansible_host')) | sort | join(',') }}"
 proxy_dnf: true
 proxy_systemd: true

environments/common/files/filebeat/filebeat.yml

@@ -54,3 +54,5 @@ output.elasticsearch:
   ssl.verification_mode: none
   username: "admin"
   password: "{{ vault_elasticsearch_admin_password }}"
+
+{% if filebeat_debug | default(false) | bool %}logging.level: debug{% endif %}

environments/skeleton/{{cookiecutter.environment}}/terraform/getfaults.py

@@ -102,8 +102,37 @@ variable "floating_ip_network" {
   default = null
 }
 
+variable "manifest_output_path" {
+  type = string
+  default = "packer-manifest.json"
+}
+
+variable "use_blockstorage_volume" {
+  type = bool
+  default = false
+}
+
+variable "volume_size" {
+  type = number
+  default = null # When not specified use the size of the builder instance root disk
+}
+
+variable "image_disk_format" {
+  type = string
+  default = null # When not specified use the image default
+}
+
+variable "metadata" {
+  type = map(string)
+  default = {}
+}
+
 source "openstack" "openhpc" {
   flavor = "${var.flavor}"
+  volume_size = "${var.volume_size}"
+  use_blockstorage_volume = "${var.use_blockstorage_volume}"
+  image_disk_format = "${var.image_disk_format}"
+  metadata = "${var.metadata}"
   networks = "${var.networks}"
   ssh_username = "${var.ssh_username}"
   ssh_timeout = "20m"
@@ -122,7 +151,7 @@ build {
     name = "compute"
     source_image = "${var.source_image}"
     source_image_name = "${var.source_image_name}" # NB: must already exist in OpenStack
-    image_name = "ohpc-${source.name}-${local.timestamp}-${substr(local.git_commit, 0, 8)}.qcow2" # also provides a unique legal instance hostname (in case of parallel packer builds)
+    image_name = "ohpc-${source.name}-${local.timestamp}-${substr(local.git_commit, 0, 8)}" # also provides a unique legal instance hostname (in case of parallel packer builds)
   }
 
   provisioner "ansible" {
@@ -134,6 +163,7 @@ build {
   }
 
   post-processor "manifest" {
+    output = "${var.manifest_output_path}"
     custom_data  = {
       source = "${source.name}"
     }
@@ -146,7 +176,7 @@ build {
     floating_ip_network = "${var.floating_ip_network}"
     source_image = "${var.fatimage_source_image}"
     source_image_name = "${var.fatimage_source_image_name}" # NB: must already exist in OpenStack
-    image_name = "${source.name}-${local.timestamp}-${substr(local.git_commit, 0, 8)}.qcow2" # similar to name from slurm_image_builder
+    image_name = "${source.name}-${local.timestamp}-${substr(local.git_commit, 0, 8)}" # similar to name from slurm_image_builder
   }
 
   provisioner "ansible" {
@@ -158,6 +188,7 @@ build {
   }
 
   post-processor "manifest" {
+    output = "${var.manifest_output_path}"
     custom_data  = {
       source = "${source.name}"
     }