Terraform love

.envrc

@@ -1 +1,2 @@
-which nix &>/dev/null && use nix
+has nix && use nix
+dotenv_if_exists

.gitignore

@@ -1,21 +1,23 @@
+# hidden files/dirs
+.*
+!deploy/compose/state/webroot/misc/osie/current/.keep
+!deploy/compose/state/webroot/workflow/.keep
+!deploy/.env
+!.gitignore
+
 # Local .terraform directories
-**/.terraform/*
 .terraform*
+!.terraform.lock.hcl
 
 # .tfstate files
 *.tfstate
 *.tfstate.*
 
-!.terraform.lock.hcl
-envrc
-out
-!deploy/.env
-.vagrant
+compose.tar.gz
+compose.zip
+deploy/compose/state/webroot/*.gz
 deploy/compose/state/webroot/misc/osie/current/*
 deploy/compose/state/webroot/workflow/*
-!deploy/compose/state/webroot/misc/osie/current/.keep
-!deploy/compose/state/webroot/workflow/.keep
-deploy/compose/state/webroot/*.gz
+envrc
+out
 workflow_id.txt
-compose.tar.gz
-compose.zip

deploy/terraform/cloud-config.cfg

@@ -6,7 +6,11 @@ write_files:
   content: ${COMPOSE_ZIP}
   path: /root/compose.zip
 
-runcmd:
-- cd /root/sandbox/compose && unzip /root/compose.zip
-- cd /root/sandbox/compose && TINKERBELL_CLIENT_MAC=${WORKER_MAC} TINKERBELL_TEMPLATE_MANIFEST=/manifests/template/ubuntu-equinix-metal.yaml TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json docker-compose up -d
+- encoding: b64
+  content: ${SETUPSH}
+  path: /root/setup.sh
+  owner: root:root
+  permissions: "0755"
 
+runcmd:
+- /root/setup.sh ${WORKER_MAC}

deploy/terraform/main.tf

@@ -20,7 +20,7 @@ provider "metal" {
   auth_token = var.metal_api_token
 }
 
-# Create a new VLAN in datacenter "ewr1"
+# Create a new VLAN in datacenter
 resource "metal_vlan" "provisioning_vlan" {
   description = "provisioning_vlan"
   metro       = var.metro
@@ -85,6 +85,7 @@ data "archive_file" "compose" {
 
 locals {
   compose_zip = data.archive_file.compose.output_size > 0 ? filebase64("${path.module}/compose.zip") : ""
+  worker_macs = flatten([for wp in metal_device.tink_worker[*].ports[*] : [for p in wp : p.mac if p.name == "eth0"]])
 }
 
 data "cloudinit_config" "setup" {
@@ -94,15 +95,12 @@ data "cloudinit_config" "setup" {
   gzip          = false # not supported on Equinix Metal
   base64_encode = false # not supported on Equinix Metal
 
-  part {
-    content_type = "text/x-shellscript"
-    content      = file("${path.module}/setup.sh")
-  }
   part {
     content_type = "text/cloud-config"
     content = templatefile("${path.module}/cloud-config.cfg", {
       COMPOSE_ZIP = local.compose_zip
-      WORKER_MAC  = metal_device.tink_worker.ports[1].mac
+      SETUPSH     = filebase64("${path.module}/setup.sh")
+      WORKER_MAC  = local.worker_macs[0]
     })
   }
 }

deploy/terraform/outputs.tf

@@ -2,6 +2,22 @@ output "provisioner_ip" {
   value = metal_device.tink_provisioner.network[0].address
 }
 
+output "provisioner_id" {
+  value = metal_device.tink_provisioner.id
+}
+
+output "provisioner_ssh" {
+  value = format("%s.packethost.net", split("-", metal_device.tink_provisioner.id)[0])
+}
+
+output "worker_id" {
+  value = metal_device.tink_worker.id
+}
+
+output "worker_macs" {
+  value = local.worker_macs
+}
+
 output "worker_sos" {
-  value = formatlist("%s@sos.%s.platformequinix.com", metal_device.tink_worker[*].id, metal_device.tink_worker.deployed_facility)
+  value = format("%s@sos.%s.platformequinix.com", metal_device.tink_worker.id, metal_device.tink_worker.deployed_facility)
 }

deploy/terraform/setup.sh

@@ -1,66 +1,174 @@
 #!/usr/bin/env bash
 
-set -xo pipefail
-
 install_docker() {
 	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-	add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+	add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
 	update_apt
-	DEBIAN_FRONTEND=noninteractive apt install -y apt-transport-https ca-certificates curl gnupg-agent gnupg2 software-properties-common docker-ce docker-ce-cli containerd.io
+	apt-get install --no-install-recommends containerd.io docker-ce docker-ce-cli
 }
 
 install_docker_compose() {
-	curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-	chmod +x /usr/local/bin/docker-compose
+	apt-get install --no-install-recommends python3-pip
+	pip install docker-compose
 }
 
-update_apt() (
-	$APT update
-	DEBIAN_FRONTEND=noninteractive $APT --yes --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade
-)
+install_iptables_persistent() {
+	apt-get install --no-install-recommends iptables-persistent
+}
 
-restart_docker_service() (
-	service docker restart
-)
+apt-get() {
+	DEBIAN_FRONTEND=noninteractive command apt-get \
+		--allow-change-held-packages \
+		--allow-downgrades \
+		--allow-remove-essential \
+		--allow-unauthenticated \
+		--option Dpkg::Options::=--force-confdef \
+		--option Dpkg::Options::=--force-confold \
+		--yes \
+		"$@"
+}
+
+update_apt() {
+	apt-get update
+	apt-get upgrade
+}
 
 # get_second_interface_from_bond0 returns the second interface of the bond0 interface
 get_second_interface_from_bond0() {
-	local return_value
-	return_value=$(cut -d' ' -f2 /sys/class/net/bond0/bonding/slaves | xargs)
-	echo "${return_value}"
+	local addr=$1
+
+	# if the ip is in a file in interfaces.d then lets assume this is a re-run and we can just
+	# return the basename of the file (which should be named same as the interface)
+	f=$(grep -lr "${addr}" /etc/network/interfaces.d)
+	[[ -n ${f:-} ]] && basename "$f" && return
+
+	# sometimes the interfaces aren't sorted as expected in the /slaves file
+	#
+	# seeing as how this function is named *second* I figured its best to be
+	# precise (via head -n2) when choosing the iface instead of choosing the last
+	# iface and hoping there are only 2
+	tr ' ' '\n' </sys/class/net/bond0/bonding/slaves | sort | head -n2 | tail -n1
 }
 
 # setup_layer2_network removes the second interface from bond0 and uses it for the layer2 network
 # https://metal.equinix.com/developers/docs/layer2-networking/hybrid-unbonded-mode/
 setup_layer2_network() {
-	local layer2_interface="$1"
-	#local ip_addr="$2"
-	ifenslave -d bond0 "${layer2_interface}"
-	#ip addr add ${ip_addr}/24 dev "${layer2_interface}"
-	ip addr add 192.168.56.4/24 dev "${layer2_interface}"
-	ip link set dev "${layer2_interface}" up
+	local interface=$1
+	local addr=$2
+
+	# I tried getting rid of the following "manual" commands in favor of
+	# persisting the network config and then restarting the network but that
+	# didn't always work and was hard to recover from without a reboot so we're
+	# stuck doing it once imperatively and also persisting the config
+	ifenslave -d bond0 "${interface}"
+	ip addr add "${addr}/24" dev "${interface}"
+	ip link set dev "${interface}" up
+
+	# persist the new network settings
+	# gets rid of the auto ${interface} block
+	# "/^auto ${interface}/,/^\s*$/ d"
+	# gets rid of ${interface} in bond config
+	# "s|${interface}||" \
+	# gets rid empty lines
+	# 's|\s*$||' \
+	# gets rid of source lines from previous runs, having this here helps in debugging/developing
+	# '/^source / d' \
+	# appends a source line to the end of the file that will pick up iface-conf file we generate
+	# '$ s|$|\n\nsource /etc/network/interfaces.d/*|' \
+	sed -i \
+		-e "/^auto ${interface}/,/^\s*$/ d" \
+		-e "s|${interface}||" \
+		-e 's|\s*$||' \
+		-e '/^source / d' \
+		-e '$ s|$|\n\nsource /etc/network/interfaces.d/*|' \
+		/etc/network/interfaces
+
+	cat >"/etc/network/interfaces.d/${interface}" <<-EOF
+		auto ${interface}
+		iface ${interface} inet static
+		    address ${addr}
+	EOF
 }
 
 # make_host_gw_server makes the host a gateway server
 make_host_gw_server() {
-	local incoming_interface="$1"
-	local outgoing_interface="$2"
+	local incoming_interface=$1
+	local outgoing_interface=$2
+
+	# drop all rules, especially interested in droppin docker's we don't want to persist docker's rules
+	# docker will re-create them when starting back up
+	systemctl stop docker
+	netfilter-persistent flush
+
 	iptables -t nat -A POSTROUTING -o "${outgoing_interface}" -j MASQUERADE
 	iptables -A FORWARD -i "${outgoing_interface}" -o "${incoming_interface}" -m state --state RELATED,ESTABLISHED -j ACCEPT
 	iptables -A FORWARD -i "${incoming_interface}" -o "${outgoing_interface}" -j ACCEPT
+
+	netfilter-persistent save
+	systemctl start docker
+}
+
+extract_compose_files() {
+	mkdir -p /sandbox
+	unzip /root/compose.zip -d /sandbox/compose
+}
+
+setup_compose_env_overrides() {
+	local worker_mac=$1
+	readarray -t lines <<-EOF
+		TINKERBELL_CLIENT_MAC=$worker_mac
+		TINKERBELL_TEMPLATE_MANIFEST=/manifests/template/ubuntu-equinix-metal.yaml
+		TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json
+	EOF
+	for line in "${lines[@]}"; do
+		grep -q "$line" /sandbox/compose/.env && continue
+		echo "$line" >>/sandbox/compose/.env
+	done
+}
+
+create_tink_helper_script() {
+	cat >/usr/local/bin/tink <<-'EOF'
+		#!/usr/bin/env bash
+
+		exec docker-compose -f /sandbox/compose/docker-compose.yml exec tink-cli tink "$@"
+	EOF
+	chmod +x /usr/local/bin/tink
+}
+
+tweak_bash_interactive_settings() {
+	grep -q 'cd /sandbox/compose' ~root/.bashrc || echo 'cd /sandbox/compose' >>~root/.bashrc
+	readarray -t aliases <<-EOF
+		dc=docker-compose
+	EOF
+	for alias in "${aliases[@]}"; do
+		grep -q "$alias" ~root/.bash_aliases || echo "alias $alias" >>~root/.bash_aliases
+	done
 }
 
-main() (
-	#local provisioner_ip="$1"
+main() {
+	worker_mac=$1
+	layer2_ip=192.168.56.4
 
+	update_apt
 	install_docker
 	install_docker_compose
-	restart_docker_service
-	mkdir -p /root/sandbox/compose
+	install_iptables_persistent
+
 	local layer2_interface
-	layer2_interface="$(get_second_interface_from_bond0)"
-	setup_layer2_network "${layer2_interface}" #"${provisioner_ip}"
-	make_host_gw_server "${layer2_interface}" "bond0"
-)
+	layer2_interface=$(get_second_interface_from_bond0 ${layer2_ip})
+	setup_layer2_network "${layer2_interface}" ${layer2_ip}
+	make_host_gw_server "${layer2_interface}" bond0
+
+	extract_compose_files
+	setup_compose_env_overrides "$worker_mac"
+	docker-compose -f /sandbox/compose/docker-compose.yml up -d
+
+	create_tink_helper_script
+	tweak_bash_interactive_settings
+}
+
+if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
+	set -euxo pipefail
 
-main #"$1"
+	main "$@"
+fi