Merge pull request #1 from converged-computing/test-google-builds

wip: testing packer for google cloud

Author: vsoch

tutorial/aws/tf-hpc6a/build/build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 ################################################################
 #
-# Flux, Singularity, and EFA
+# Flux, Singularity, and ORAS
 #
 
 /usr/bin/cloud-init status --wait

tutorial/google/README.md

@@ -0,0 +1,40 @@
+# Flux on Google Cloud
+
+This is a setup akin to Cluster Toolkit (using Terraform) to run Flux on Google Cloud.
+Google Cloud does not support any kind of special networking, so we will rely on ethernet. This setup comes also with Singularity and ORAS. You'll need to build from [build-images](build-images). Since we can change the instance on the fly (generally speaking) we just have one build and terraform directory. 
+
+## Usage
+
+### 1. Create Google Service Accounts
+
+Create default application credentials (just once):
+
+```bash
+gcloud auth application-default login
+```
+
+### 2. Build Base Image
+
+You can build the base VM with [build-images](build-images). This is working with packer (!) so you should look at the main packer HCL files, see if you want to customize anything, and then just:
+
+```bash
+make
+```
+
+The install script is in [build-images/build.sh](build-images/build.sh), and you can customize it as you like. Note that the Makefile has a setting so that when a command fails, it waits for your response. I recommend that you shell into the VM if this happens to debug, and then do it again when you've found and fixed the issue. If you don't change anything, it should work as is.
+
+### 3. Terraform
+
+Next, cd into [tf](tf) and again open [tf/basic.tfvars](tf/basic.tfvars) to look at the metadata and update anything as needed. I recommend starting at a small scale first.  Then bring it up!
+
+```bash
+make
+```
+
+When you are done:
+
+```bash
+make destroy
+```
+
+Note that I had issues with a fully terraform teardown, so I wrote a script that asks for the number of instances, and uses gcloud to supplement.

tutorial/google/base/main.tf

@@ -0,0 +1,44 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module "compute_nodes" {
+    source          = "./modules/compute"
+
+    for_each = {
+        for index, node in var.compute_node_specs:
+        node.name_prefix => node
+    }
+    project_id        = var.project_id
+    region            = var.region
+
+    family            = var.family
+    
+    name_prefix       = each.value.name_prefix
+    subnetwork        = var.subnetwork
+    machine_arch      = each.value.machine_arch
+    machine_type      = each.value.machine_type
+    num_instances     = each.value.instances
+
+    boot_script       = lookup(each.value, "boot_script", null)
+    compact_placement = lookup(each.value, "compact", false)
+    gpu               = lookup(each.value, "gpu_type", null) == null || lookup(each.value, "gpu_count", 0) <= 0 ? null : {
+        type  = each.value.gpu_type
+        count = each.value.gpu_count
+    }
+    service_account   = {
+        email  = var.service_account_emails["compute"]
+        scopes = var.compute_scopes
+    }
+    nfs_mounts         = var.cluster_storage
+}

tutorial/google/base/modules/compute/main.tf

@@ -0,0 +1,82 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data "google_compute_image" "fluxfw_compute_x86_64_image" {
+    project = var.project_id
+    family  = var.family
+}
+
+data "google_compute_zones" "available" {
+    project = var.project_id
+    region  = var.region
+}
+
+locals {
+    automatic_restart = var.compact_placement ? false : var.automatic_restart
+    compute_images = {
+        "x86-64" = {
+            image   = data.google_compute_image.fluxfw_compute_x86_64_image.self_link
+            project = data.google_compute_image.fluxfw_compute_x86_64_image.project
+        }
+    }
+    on_host_maintenance = var.compact_placement ? "TERMINATE" : var.on_host_maintenance
+}
+
+resource "google_compute_resource_policy" "collocated" {
+    count   = var.compact_placement ? 1 : 0
+    name    = "${var.name_prefix}-collocated-policy"
+    project = var.project_id
+    region  = var.region
+    group_placement_policy {
+      vm_count = var.num_instances
+      collocation = "COLLOCATED"
+    }
+}
+
+module "flux_compute_instance_template" {
+    source               = "github.com/terraform-google-modules/terraform-google-vm/modules/instance_template"
+    region               = var.region
+    project_id           = var.project_id
+    name_prefix          = var.name_prefix
+    subnetwork           = var.subnetwork
+    gpu                  = var.gpu
+    service_account      = var.service_account
+    tags                 = ["ssh", "flux", "compute"]
+    machine_type         = var.machine_type
+    disk_size_gb         = 256
+    source_image         = local.compute_images["${var.machine_arch}"].image
+    source_image_project = local.compute_images["${var.machine_arch}"].project
+    automatic_restart    = local.automatic_restart
+    on_host_maintenance  = local.on_host_maintenance
+    startup_script       = var.boot_script
+
+    metadata             = { 
+        "enable-oslogin"   : "TRUE",
+        "VmDnsSetting"     : "GlobalDefault",
+        "nfs-mounts"       : jsonencode(var.nfs_mounts),
+        "gpus-attached"    : var.gpu != null ? "TRUE" : "FALSE"
+    }
+}
+
+module "flux_compute_instances" {
+    source              = "github.com/terraform-google-modules/terraform-google-vm/modules/compute_instance"
+    region              = var.region
+    zone                = data.google_compute_zones.available.names[0]
+    hostname            = var.name_prefix
+    add_hostname_suffix = true
+    num_instances       = var.num_instances
+    resource_policies   = var.compact_placement ? [ google_compute_resource_policy.collocated[0].self_link ] : []
+    instance_template   = module.flux_compute_instance_template.self_link
+    subnetwork          = var.subnetwork
+}

tutorial/google/base/modules/compute/variables.tf

@@ -0,0 +1,88 @@
+variable "automatic_restart" {
+  type        = bool
+  description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)."
+  default     = true
+}
+
+variable "boot_script" {
+    description = "(Optional) the name of a file containing a script to be executed on compute nodes at boot time"
+    type        = string
+    default     = null
+}
+
+variable "compact_placement" {
+    description = "(Optional) a boolean which determines whether a set of compute nodes has a compact placement resource policy attached to them."
+    type        = bool
+    default     = false
+}
+
+variable "family" {
+    description = "The source X86 image family prefix to use"
+    type        = string
+    default     = "flux-framework"
+}
+
+variable "gpu" {
+    description = "The type and count of GPU(s) to attach to a compute node"
+    type        = object({
+        type  = string
+        count = number
+    })
+    default     = null
+}
+
+variable "machine_arch" {
+    description = "The instruction set architecture, usually x86_64, used by the compute node"
+    type        = string
+}
+
+variable "machine_type" {
+    description = "The Compute Engine machine type to be used for the compute node"
+    type        = string
+}
+
+variable "name_prefix" {
+    description = "The name prefix for the compute node instances, the full instances names will be this prefix followed by a node number"
+    type        = string
+}
+
+variable "nfs_mounts" {
+    description = "A map with keys 'share' and 'mountpoint' describing an NFS export and its intended mount point"
+    type        = map(string)
+    default     = {}
+}
+
+variable "num_instances" {
+    description = "The number of compute node instances to create"
+    type        = number
+    default     = 1
+}
+
+variable "on_host_maintenance" {
+  type        = string
+  description = "Instance availability Policy"
+  default     = "MIGRATE"
+}
+
+variable "project_id" {
+    description = "The GCP project ID"
+    type        = string
+}
+
+variable "region" {
+    description = "The GCP region where the cluster resides"
+    type        = string
+}
+
+variable "service_account" {
+    description = "The GCP service account used by the compute node"
+    type        = object({
+        email  = string
+        scopes = set(string)
+    })
+}
+
+variable "subnetwork" {
+    description = "Subnetwork to deploy to"
+    type        = string
+}

tutorial/google/base/variables.tf

@@ -0,0 +1,53 @@
+
+variable "cluster_storage" { 
+    description = "A map with keys 'share' and 'mountpoint' describing an NFS export and its intended mount point"
+    type        = map(string)
+}
+
+variable "family" {
+    description = "The source image x86 prefix to be used by the compute node(s)"
+    type        = string
+    default     = "global"
+}
+
+variable "compute_node_specs" {
+    description = "A list of compute node specifications"
+    type = list(object({
+       name_prefix  = string
+       machine_arch = string
+       machine_type = string
+       gpu_type     = string
+       gpu_count    = number
+       compact      = bool
+       instances    = number
+       properties   = set(string)
+       boot_script  = string
+    }))
+    default = []
+}
+
+variable "compute_scopes" {
+    description = "The set of access scopes for compute node instances"
+    default     = [ "cloud-platform" ]
+    type        = set(string)
+}
+
+variable "project_id" {
+    description = "The GCP project ID"
+    type        = string
+}
+
+variable "region" {
+    description = "The GCP region where the cluster resides"
+    type = string
+}
+
+variable "service_account_emails" {
+    description = "A map with keys: 'compute', 'login', 'manager' that map to the service account to be used by the respective nodes"
+    type        = map(string)
+}
+
+variable "subnetwork" {
+    description = "Subnetwork to deploy to"
+    type        = string
+}

tutorial/google/build-images/Makefile

@@ -0,0 +1,20 @@
+.PHONY: all
+all: init fmt validate build
+
+.PHONY: init
+init: 
+	packer init .
+	
+.PHONY: fmt
+fmt: 
+	packer fmt .
+
+.PHONY: validate
+validate: 
+	packer validate .
+	
+.PHONY: build
+build: 
+	# On error, the best strategy I've found is to ssh in, figure out what isn't there
+	# (where it failed) and debug
+	packer build --on-error=ask flux-build.pkr.hcl