From be2bb16b9f0d78b5b19f61ea57c3ee61af2b73c8 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 30 Mar 2022 15:28:40 -0400
Subject: [PATCH 01/25] direnv: use `has` from stdlib

More ergonomic.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 .envrc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.envrc b/.envrc
index b8b78324..0fc3eabb 100644
--- a/.envrc
+++ b/.envrc
@@ -1 +1,2 @@
-which nix &>/dev/null && use nix
+has nix && use nix
+dotenv_if_exists

From c22b926b75fb80aa4f3f6139d099b9c375d0231e Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 30 Mar 2022 14:18:14 -0400
Subject: [PATCH 02/25] git: Manage git ignores in just one .gitignore file

Lots of dupes here and it was confusing, better to just have one place.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 .gitignore                              | 24 +++++++++++++-----------
 deploy/compose/state/webroot/.gitignore |  1 -
 deploy/terraform/.gitignore             |  5 -----
 3 files changed, 13 insertions(+), 17 deletions(-)
 delete mode 100644 deploy/compose/state/webroot/.gitignore
 delete mode 100644 deploy/terraform/.gitignore

diff --git a/.gitignore b/.gitignore
index 091a8317..7491f518 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,21 +1,23 @@
+# hidden files/dirs
+.*
+!deploy/compose/state/webroot/misc/osie/current/.keep
+!deploy/compose/state/webroot/workflow/.keep
+!deploy/.env
+!.gitignore
+
 # Local .terraform directories
-**/.terraform/*
 .terraform*
+!.terraform.lock.hcl
 
 # .tfstate files
 *.tfstate
 *.tfstate.*
 
-!.terraform.lock.hcl
-envrc
-out
-!deploy/.env
-.vagrant
+compose.tar.gz
+compose.zip
+deploy/compose/state/webroot/*.gz
 deploy/compose/state/webroot/misc/osie/current/*
 deploy/compose/state/webroot/workflow/*
-!deploy/compose/state/webroot/misc/osie/current/.keep
-!deploy/compose/state/webroot/workflow/.keep
-deploy/compose/state/webroot/*.gz
+envrc
+out
 workflow_id.txt
-compose.tar.gz
-compose.zip
diff --git a/deploy/compose/state/webroot/.gitignore b/deploy/compose/state/webroot/.gitignore
deleted file mode 100644
index 10d00b57..00000000
--- a/deploy/compose/state/webroot/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.gz
diff --git a/deploy/terraform/.gitignore b/deploy/terraform/.gitignore
deleted file mode 100644
index 1918c7f0..00000000
--- a/deploy/terraform/.gitignore
+++ /dev/null
@@ -1,5 +0,0 @@
-.terraform
-terraform.tfstate
-terraform.tfstate.backup
-terraform.tfvars
-.terraform.lock.hcl

From 6d25367ddb431651f65c1589ed640f05c8ee6610 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 15:34:27 -0400
Subject: [PATCH 03/25] tf: Get rid of mention of ewr1 in comment

It's incorrect, surprise surprise...

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/terraform/main.tf b/deploy/terraform/main.tf
index 81fc872e..86f04464 100644
--- a/deploy/terraform/main.tf
+++ b/deploy/terraform/main.tf
@@ -20,7 +20,7 @@ provider "metal" {
   auth_token = var.metal_api_token
 }
 
-# Create a new VLAN in datacenter "ewr1"
+# Create a new VLAN in datacenter
 resource "metal_vlan" "provisioning_vlan" {
   description = "provisioning_vlan"
   metro       = var.metro

From 3b5e18197e1e0f6c7ab395927e51ecddeb9594c0 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 15:38:33 -0400
Subject: [PATCH 04/25] tf: Add output value for the provisioner ssh hostname

Makes it nice and easy to connect to the provisioner using cli.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/outputs.tf     | 4 ++++
 docs/quickstarts/TERRAFORMEM.md | 3 ++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/deploy/terraform/outputs.tf b/deploy/terraform/outputs.tf
index d27a9a41..305929b0 100644
--- a/deploy/terraform/outputs.tf
+++ b/deploy/terraform/outputs.tf
@@ -2,6 +2,10 @@ output "provisioner_ip" {
   value = metal_device.tink_provisioner.network[0].address
 }
 
+output "provisioner_ssh" {
+  value = format("%s.packethost.net", split("-", metal_device.tink_provisioner.id)[0])
+}
+
 output "worker_sos" {
   value = formatlist("%s@sos.%s.platformequinix.com", metal_device.tink_worker[*].id, metal_device.tink_worker.deployed_facility)
 }
diff --git a/docs/quickstarts/TERRAFORMEM.md b/docs/quickstarts/TERRAFORMEM.md
index 187bbf7d..62b00456 100644
--- a/docs/quickstarts/TERRAFORMEM.md
+++ b/docs/quickstarts/TERRAFORMEM.md
@@ -49,7 +49,8 @@ This option will also show you how to create a machine to provision.
 
    ```bash
    # log in to the provisioner
-   ssh root@139.178.69.231
+   ssh root@$(terraform output -raw provisioner_ssh)
+
    # watch the workflow events and status for workflow completion
    # once the workflow is complete (see the expected output below for completion), move on to the next step
    wid=$(docker exec -it compose_tink-cli_1 tink workflow get --no-headers | awk '/^\|/ {print $2}'); docker exec -it compose_tink-cli_1 watch -n1 "tink workflow events ${wid}; tink workflow state ${wid}"

From 64cb0aeeb5c4798c5a6dffec85b0899150bc8b60 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 15:39:28 -0400
Subject: [PATCH 05/25] tf/setup: Configure bash to be stricter/safer

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index 69a33bc0..98a6ad80 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -xo pipefail
+set -euxo pipefail
 
 install_docker() {
 	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

From eb41f998f84bf71f02d43eb8dce22081a25da6fc Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 15:46:28 -0400
Subject: [PATCH 06/25] tf/setup: Ensure all functions use same execution mode

func() is not the same as func{}, the latter is run in a new subshell. There
does not seem to be a need for the subshell behavior so I'm changing to the
more common (in this file, and also generally speaking) {} form.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index 98a6ad80..4cd76574 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -14,14 +14,14 @@ install_docker_compose() {
 	chmod +x /usr/local/bin/docker-compose
 }
 
-update_apt() (
+update_apt() {
 	$APT update
 	DEBIAN_FRONTEND=noninteractive $APT --yes --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade
-)
+}
 
-restart_docker_service() (
+restart_docker_service() {
 	service docker restart
-)
+}
 
 # get_second_interface_from_bond0 returns the second interface of the bond0 interface
 get_second_interface_from_bond0() {
@@ -50,7 +50,7 @@ make_host_gw_server() {
 	iptables -A FORWARD -i "${incoming_interface}" -o "${outgoing_interface}" -j ACCEPT
 }
 
-main() (
+main() {
 	#local provisioner_ip="$1"
 
 	install_docker
@@ -61,6 +61,6 @@ main() (
 	layer2_interface="$(get_second_interface_from_bond0)"
 	setup_layer2_network "${layer2_interface}" #"${provisioner_ip}"
 	make_host_gw_server "${layer2_interface}" "bond0"
-)
+}
 
 main #"$1"

From 08714f90995f880c62bcd82f1fd888114c91d80f Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 15:49:18 -0400
Subject: [PATCH 07/25] tf/setup: Use apt-get helper function

This helper function lets us avoid the need to specify the env and extra args
every single time we want to interact with packages. This uses apt-get and not apt
because apt prints the following warning message:

> # WARNING: apt does not have a stable CLI interface. Use with caution in scripts.

and there's no overwhelming reason to favor apt > apt-get, so lets not risk it.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index 4cd76574..2337b43f 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -6,7 +6,7 @@ install_docker() {
 	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
 	add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
 	update_apt
-	DEBIAN_FRONTEND=noninteractive apt install -y apt-transport-https ca-certificates curl gnupg-agent gnupg2 software-properties-common docker-ce docker-ce-cli containerd.io
+	apt-get install apt-transport-https ca-certificates curl gnupg-agent gnupg2 software-properties-common docker-ce docker-ce-cli containerd.io
 }
 
 install_docker_compose() {
@@ -14,9 +14,21 @@ install_docker_compose() {
 	chmod +x /usr/local/bin/docker-compose
 }
 
+apt-get() {
+	DEBIAN_FRONTEND=noninteractive command apt-get \
+		--allow-change-held-packages \
+		--allow-downgrades \
+		--allow-remove-essential \
+		--allow-unauthenticated \
+		--option Dpkg::Options::=--force-confdef \
+		--option Dpkg::Options::=--force-confold \
+		--yes \
+		"$@"
+}
+
 update_apt() {
-	$APT update
-	DEBIAN_FRONTEND=noninteractive $APT --yes --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade
+	apt-get update
+	apt-get upgrade
 }
 
 restart_docker_service() {

From 6123d9efd72726e956a81db3e2e45d2da2105eb4 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 16:00:46 -0400
Subject: [PATCH 08/25] tf/setup: Only explicitly install the docker packages

Let dpkg package dependencies and apt-get figure out dependencies.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index 2337b43f..038985fa 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -6,7 +6,7 @@ install_docker() {
 	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
 	add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
 	update_apt
-	apt-get install apt-transport-https ca-certificates curl gnupg-agent gnupg2 software-properties-common docker-ce docker-ce-cli containerd.io
+	apt-get install --no-install-recommends containerd.io docker-ce docker-ce-cli
 }
 
 install_docker_compose() {

From f4c4605855e7a7dc433562e2bd9f2bc7548465d1 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 15:58:52 -0400
Subject: [PATCH 09/25] tf/setup: Don't hard code the arch when adding docker
 apt-repository

This is not needed and would mess up someone trying to run the tf stack on
aarch64 (.../me whistles innocently...) or other arches.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index 038985fa..8d90f106 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -4,7 +4,7 @@ set -euxo pipefail
 
 install_docker() {
 	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-	add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+	add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
 	update_apt
 	apt-get install --no-install-recommends containerd.io docker-ce docker-ce-cli
 }

From 95e3caef4b7a92306c52178f0c92f99a2a1eca69 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 16:01:44 -0400
Subject: [PATCH 10/25] tf/setup: Install docker-compose using pip

GitHub releases only has binaries for x86, which sort of don't work on other
architectures. Luckily the cheese shop has all the flavors of docker-compose
any one would be interested in.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index 8d90f106..925b2930 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -10,8 +10,8 @@ install_docker() {
 }
 
 install_docker_compose() {
-	curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-	chmod +x /usr/local/bin/docker-compose
+	apt-get install --no-install-recommends python3-pip
+	pip install docker-compose
 }
 
 apt-get() {

From 2a4b18ee08c99aa503547d1cce00ce79aefc97f5 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 16:12:34 -0400
Subject: [PATCH 11/25] tf/setup: Make main function actually functional

Previuosly main didn't really offer much over just doing the same work outside
of a function, main would always run. With this new setup we can source
setup.sh and have nothing run or just have a full run if executed instead.

Very much like Python's `if __name__ == '__main__': # call main func` pattern.

Also add some spaces to group functionality together for easier
reading/scanning.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index 925b2930..e8305250 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 
-set -euxo pipefail
-
 install_docker() {
 	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
 	add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
@@ -63,16 +61,20 @@ make_host_gw_server() {
 }
 
 main() {
-	#local provisioner_ip="$1"
-
 	install_docker
 	install_docker_compose
 	restart_docker_service
-	mkdir -p /root/sandbox/compose
+
 	local layer2_interface
 	layer2_interface="$(get_second_interface_from_bond0)"
 	setup_layer2_network "${layer2_interface}" #"${provisioner_ip}"
 	make_host_gw_server "${layer2_interface}" "bond0"
+
+	mkdir -p /root/sandbox/compose
 }
 
-main #"$1"
+if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
+	set -euxo pipefail
+
+	main "$@"
+fi

From 1625f906a74115be3344d6681fdec3d99b7b452f Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 16:14:34 -0400
Subject: [PATCH 12/25] tf/setup: Do not restart docker service

Its not necessary.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index e8305250..73ebef81 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -29,10 +29,6 @@ update_apt() {
 	apt-get upgrade
 }
 
-restart_docker_service() {
-	service docker restart
-}
-
 # get_second_interface_from_bond0 returns the second interface of the bond0 interface
 get_second_interface_from_bond0() {
 	local return_value
@@ -63,7 +59,6 @@ make_host_gw_server() {
 main() {
 	install_docker
 	install_docker_compose
-	restart_docker_service
 
 	local layer2_interface
 	layer2_interface="$(get_second_interface_from_bond0)"

From 0dbe3dd428b2d4936d5a403b52398897593f087b Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 16:49:16 -0400
Subject: [PATCH 13/25] tf/setup: Persist 2 separate network config

Without this the network settings are lost on reboot, which doesn't make for a
very useful development machine. I put the layer2 iface config within a
file in interfaces.d to make reruns of this script work correctly after the next
commit.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 48 +++++++++++++++++++++++++++++++--------
 1 file changed, 39 insertions(+), 9 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index 73ebef81..f89230c9 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -39,12 +39,41 @@ get_second_interface_from_bond0() {
 # setup_layer2_network removes the second interface from bond0 and uses it for the layer2 network
 # https://metal.equinix.com/developers/docs/layer2-networking/hybrid-unbonded-mode/
 setup_layer2_network() {
-	local layer2_interface="$1"
-	#local ip_addr="$2"
-	ifenslave -d bond0 "${layer2_interface}"
-	#ip addr add ${ip_addr}/24 dev "${layer2_interface}"
-	ip addr add 192.168.56.4/24 dev "${layer2_interface}"
-	ip link set dev "${layer2_interface}" up
+	local interface=$1
+	local addr=$2
+
+	# I tried getting rid of the following "manual" commands in favor of
+	# persisting the network config and then restarting the network but that
+	# didn't always work and was hard to recover from without a reboot so we're
+	# stuck doing it once imperatively and also persisting the config
+	ifenslave -d bond0 "${interface}"
+	ip addr add "${addr}/24" dev "${interface}"
+	ip link set dev "${interface}" up
+
+	# persist the new network settings
+	# gets rid of the auto ${interface} block
+	# "/^auto ${interface}/,/^\s*$/ d"
+	# gets rid of ${interface} in bond config
+	# "s|${interface}||" \
+	# gets rid empty lines
+	# 's|\s*$||' \
+	# gets rid of source lines from previous runs, having this here helps in debugging/developing
+	# '/^source / d' \
+	# appends a source line to the end of the file that will pick up iface-conf file we generate
+	# '$ s|$|\n\nsource /etc/network/interfaces.d/*|' \
+	sed -i \
+		-e "/^auto ${interface}/,/^\s*$/ d" \
+		-e "s|${interface}||" \
+		-e 's|\s*$||' \
+		-e '/^source / d' \
+		-e '$ s|$|\n\nsource /etc/network/interfaces.d/*|' \
+		/etc/network/interfaces
+
+	cat >"/etc/network/interfaces.d/${interface}" <<-EOF
+		auto ${interface}
+		iface ${interface} inet static
+		    address ${addr}
+	EOF
 }
 
 # make_host_gw_server makes the host a gateway server
@@ -60,10 +89,11 @@ main() {
 	install_docker
 	install_docker_compose
 
+	local layer2_ip=192.168.56.4
 	local layer2_interface
-	layer2_interface="$(get_second_interface_from_bond0)"
-	setup_layer2_network "${layer2_interface}" #"${provisioner_ip}"
-	make_host_gw_server "${layer2_interface}" "bond0"
+	layer2_interface=$(get_second_interface_from_bond0 ${layer2_ip})
+	setup_layer2_network "${layer2_interface}" ${layer2_ip}
+	make_host_gw_server "${layer2_interface}" bond0
 
 	mkdir -p /root/sandbox/compose
 }

From 220c65cf1b3924a8119d23dfc89dd815bb2fa6e3 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 16:59:24 -0400
Subject: [PATCH 14/25] tf/setup: Improve correctness of
 get_second_interface_from_bond0

I noticed that the slaves names in the bonding/slaves file was not guaranteed
to be in any specific order so added a sort call. I also wanted to future proof
the function in case there's ever a type that has more than 2 nics in the bond
by default by ensuring we use the second nic. Finally, I made the function
behave correctly if the script is being re-run and the bond has already been
broken up, this was useful while debugging breakages and fixing this script.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index f89230c9..d1721c7c 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -31,9 +31,19 @@ update_apt() {
 
 # get_second_interface_from_bond0 returns the second interface of the bond0 interface
 get_second_interface_from_bond0() {
-	local return_value
-	return_value=$(cut -d' ' -f2 /sys/class/net/bond0/bonding/slaves | xargs)
-	echo "${return_value}"
+	local addr=$1
+
+	# if the ip is in a file in interfaces.d then lets assume this is a re-run and we can just
+	# return the basename of the file (which should be named same as the interface)
+	f=$(grep -lr "${addr}" /etc/network/interfaces.d)
+	[[ -n ${f:-} ]] && basename "$f" && return
+
+	# sometimes the interfaces aren't sorted as expected in the /slaves file
+	#
+	# seeing as how this function is named *second* I figured its best to be
+	# precise (via head -n2) when choosing the iface instead of choosing the last
+	# iface and hoping there are only 2
+	tr ' ' '\n' </sys/class/net/bond0/bonding/slaves | sort | head -n2 | tail -n1
 }
 
 # setup_layer2_network removes the second interface from bond0 and uses it for the layer2 network

From ef11293231971089ce0b4a57eb4f0c201af6fe02 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 17:04:16 -0400
Subject: [PATCH 15/25] tf/setup: Persist iptables gw rules

Otherwise gateway functionality is broken after a reboot.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index d1721c7c..a18c6232 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -12,6 +12,10 @@ install_docker_compose() {
 	pip install docker-compose
 }
 
+install_iptables_persistent() {
+	apt-get install --no-install-recommends iptables-persistent
+}
+
 apt-get() {
 	DEBIAN_FRONTEND=noninteractive command apt-get \
 		--allow-change-held-packages \
@@ -88,16 +92,27 @@ setup_layer2_network() {
 
 # make_host_gw_server makes the host a gateway server
 make_host_gw_server() {
-	local incoming_interface="$1"
-	local outgoing_interface="$2"
+	local incoming_interface=$1
+	local outgoing_interface=$2
+
+	# drop all rules, especially interested in droppin docker's we don't want to persist docker's rules
+	# docker will re-create them when starting back up
+	systemctl stop docker
+	netfilter-persistent flush
+
 	iptables -t nat -A POSTROUTING -o "${outgoing_interface}" -j MASQUERADE
 	iptables -A FORWARD -i "${outgoing_interface}" -o "${incoming_interface}" -m state --state RELATED,ESTABLISHED -j ACCEPT
 	iptables -A FORWARD -i "${incoming_interface}" -o "${outgoing_interface}" -j ACCEPT
+
+	netfilter-persistent save
+	systemctl start docker
 }
 
 main() {
+	update_apt
 	install_docker
 	install_docker_compose
+	install_iptables_persistent
 
 	local layer2_ip=192.168.56.4
 	local layer2_interface

From d2b768af2092d353c3c3be9ebe3d702f65b77317 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 17:06:05 -0400
Subject: [PATCH 16/25] tf: Add local variable for worker_macs

This variable doesn't assume that port[1] == "eth0", instead it grabs the mac
for the port whos  name is eth0. It also keeps track of the mac(s) in an array
because I was messing with multiple workers at one point and the logic to
figure this out was tricky and I don't want to lose it.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/main.tf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/deploy/terraform/main.tf b/deploy/terraform/main.tf
index 86f04464..ad124e9c 100644
--- a/deploy/terraform/main.tf
+++ b/deploy/terraform/main.tf
@@ -85,6 +85,7 @@ data "archive_file" "compose" {
 
 locals {
   compose_zip = data.archive_file.compose.output_size > 0 ? filebase64("${path.module}/compose.zip") : ""
+  worker_macs = flatten([for wp in metal_device.tink_worker[*].ports[*] : [for p in wp : p.mac if p.name == "eth0"]])
 }
 
 data "cloudinit_config" "setup" {
@@ -102,7 +103,7 @@ data "cloudinit_config" "setup" {
     content_type = "text/cloud-config"
     content = templatefile("${path.module}/cloud-config.cfg", {
       COMPOSE_ZIP = local.compose_zip
-      WORKER_MAC  = metal_device.tink_worker.ports[1].mac
+      WORKER_MAC  = local.worker_macs[0]
     })
   }
 }

From f4d3a3e058d294bc9725f6a72566a5eb3f4b8e24 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 17:15:38 -0400
Subject: [PATCH 17/25] tf: Add output for worker_macs

Makes it easy to get the macs and make use of, for example look at boots logs.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/outputs.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/deploy/terraform/outputs.tf b/deploy/terraform/outputs.tf
index 305929b0..3093f961 100644
--- a/deploy/terraform/outputs.tf
+++ b/deploy/terraform/outputs.tf
@@ -6,6 +6,10 @@ output "provisioner_ssh" {
   value = format("%s.packethost.net", split("-", metal_device.tink_provisioner.id)[0])
 }
 
+output "worker_macs" {
+  value = local.worker_macs
+}
+
 output "worker_sos" {
   value = formatlist("%s@sos.%s.platformequinix.com", metal_device.tink_worker[*].id, metal_device.tink_worker.deployed_facility)
 }

From d9614020f3c40c158f8f0a0edef2af1ca8712a97 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Fri, 18 Mar 2022 15:52:43 -0400
Subject: [PATCH 18/25] tf: Add outputs for provisioner and worker ids

Makes using them much easier.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/outputs.tf     | 8 ++++++++
 docs/quickstarts/TERRAFORMEM.md | 4 ++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/deploy/terraform/outputs.tf b/deploy/terraform/outputs.tf
index 3093f961..beaf483a 100644
--- a/deploy/terraform/outputs.tf
+++ b/deploy/terraform/outputs.tf
@@ -2,10 +2,18 @@ output "provisioner_ip" {
   value = metal_device.tink_provisioner.network[0].address
 }
 
+output "provisioner_id" {
+  value = metal_device.tink_provisioner.id
+}
+
 output "provisioner_ssh" {
   value = format("%s.packethost.net", split("-", metal_device.tink_provisioner.id)[0])
 }
 
+output "worker_id" {
+  value = metal_device.tink_worker.id
+}
+
 output "worker_macs" {
   value = local.worker_macs
 }
diff --git a/docs/quickstarts/TERRAFORMEM.md b/docs/quickstarts/TERRAFORMEM.md
index 62b00456..d9841b39 100644
--- a/docs/quickstarts/TERRAFORMEM.md
+++ b/docs/quickstarts/TERRAFORMEM.md
@@ -42,7 +42,7 @@ This option will also show you how to create a machine to provision.
    Or if you have the [Equinix Metal CLI](https://github.com/equinix/metal-cli) installed run the following:
 
    ```bash
-   metal device reboot -i $(terraform show -json | jq -r '.values.root_module.resources[3].values.id')
+   metal device reboot -i $(terraform output -raw worker_id)
    ```
 
 5. Watch the provision complete
@@ -95,7 +95,7 @@ This option will also show you how to create a machine to provision.
    Now reboot the `tink-worker` via the [Equinix Metal Web UI](https://console.equinix.com), or if you have the [Equinix Metal CLI](https://github.com/equinix/metal-cli) installed run the following:
 
    ```bash
-   metal device reboot -i $(terraform show -json | jq -r '.values.root_module.resources[3].values.id')
+   metal device reboot -i $(terraform output -raw worker_id)
    ```
 
 7. Login to the machine

From 81bbe41259bedf7fccf5ee218c293d41e0de79af Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 16 Mar 2022 17:12:46 -0400
Subject: [PATCH 19/25] tf: Modify compose/.env file for repeat docker-compose
 runs

Later/manual runs of docker-compose will do the wrong thing without this
because the env settings won't be populated for those runs. This fixes that
error by first modifying the .env file which docker-compose will use.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/cloud-config.cfg | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/deploy/terraform/cloud-config.cfg b/deploy/terraform/cloud-config.cfg
index 9ce7a7a1..f7994db0 100644
--- a/deploy/terraform/cloud-config.cfg
+++ b/deploy/terraform/cloud-config.cfg
@@ -7,6 +7,8 @@ write_files:
   path: /root/compose.zip
 
 runcmd:
-- cd /root/sandbox/compose && unzip /root/compose.zip
-- cd /root/sandbox/compose && TINKERBELL_CLIENT_MAC=${WORKER_MAC} TINKERBELL_TEMPLATE_MANIFEST=/manifests/template/ubuntu-equinix-metal.yaml TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json docker-compose up -d
-
+- unzip -v /root/compose.zip -d /root/sandbox
+- echo TINKERBELL_CLIENT_MAC=${WORKER_MAC} >>/root/sandbox/compose/.env
+- echo TINKERBELL_TEMPLATE_MANIFEST=/manifests/template/ubuntu-equinix-metal.yaml >>/root/sandbox/compose/.env
+- echo TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json >>/root/sandbox/compose/.env
+- docker-compose -f /root/sandbox/compose/docker-compose.yml up

From 9bf8aee837a120dd3e28863a1181291642091853 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Fri, 18 Mar 2022 15:19:56 -0400
Subject: [PATCH 20/25] tf: Put all setup logic in setup.sh

No sense in splitting up the setup logic into 2 different files.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/cloud-config.cfg | 12 +++++++-----
 deploy/terraform/main.tf          |  5 +----
 deploy/terraform/setup.sh         | 26 ++++++++++++++++++++++++--
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/deploy/terraform/cloud-config.cfg b/deploy/terraform/cloud-config.cfg
index f7994db0..c9562c9c 100644
--- a/deploy/terraform/cloud-config.cfg
+++ b/deploy/terraform/cloud-config.cfg
@@ -6,9 +6,11 @@ write_files:
   content: ${COMPOSE_ZIP}
   path: /root/compose.zip
 
+- encoding: b64
+  content: ${SETUPSH}
+  path: /root/setup.sh
+  owner: root:root
+  permissions: "0755"
+
 runcmd:
-- unzip -v /root/compose.zip -d /root/sandbox
-- echo TINKERBELL_CLIENT_MAC=${WORKER_MAC} >>/root/sandbox/compose/.env
-- echo TINKERBELL_TEMPLATE_MANIFEST=/manifests/template/ubuntu-equinix-metal.yaml >>/root/sandbox/compose/.env
-- echo TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json >>/root/sandbox/compose/.env
-- docker-compose -f /root/sandbox/compose/docker-compose.yml up
+- /root/setup.sh ${WORKER_MAC}
diff --git a/deploy/terraform/main.tf b/deploy/terraform/main.tf
index ad124e9c..7643df7b 100644
--- a/deploy/terraform/main.tf
+++ b/deploy/terraform/main.tf
@@ -95,14 +95,11 @@ data "cloudinit_config" "setup" {
   gzip          = false # not supported on Equinix Metal
   base64_encode = false # not supported on Equinix Metal
 
-  part {
-    content_type = "text/x-shellscript"
-    content      = file("${path.module}/setup.sh")
-  }
   part {
     content_type = "text/cloud-config"
     content = templatefile("${path.module}/cloud-config.cfg", {
       COMPOSE_ZIP = local.compose_zip
+      SETUPSH     = filebase64("${path.module}/setup.sh")
       WORKER_MAC  = local.worker_macs[0]
     })
   }
diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index a18c6232..eb0d5cfd 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -108,19 +108,41 @@ make_host_gw_server() {
 	systemctl start docker
 }
 
+extract_compose_files() {
+	mkdir -p /root/sandbox
+	unzip /root/compose.zip -d /root/sandbox/compose
+}
+
+setup_compose_env_overrides() {
+	local worker_mac=$1
+	readarray -t lines <<-EOF
+		TINKERBELL_CLIENT_MAC=$worker_mac
+		TINKERBELL_TEMPLATE_MANIFEST=/manifests/template/ubuntu-equinix-metal.yaml
+		TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json
+	EOF
+	for line in "${lines[@]}"; do
+		grep -q "$line" /root/sandbox/compose/.env && continue
+		echo "$line" >>/root/sandbox/compose/.env
+	done
+}
+
 main() {
+	worker_mac=$1
+	layer2_ip=192.168.56.4
+
 	update_apt
 	install_docker
 	install_docker_compose
 	install_iptables_persistent
 
-	local layer2_ip=192.168.56.4
 	local layer2_interface
 	layer2_interface=$(get_second_interface_from_bond0 ${layer2_ip})
 	setup_layer2_network "${layer2_interface}" ${layer2_ip}
 	make_host_gw_server "${layer2_interface}" bond0
 
-	mkdir -p /root/sandbox/compose
+	extract_compose_files
+	setup_compose_env_overrides "$worker_mac"
+	docker-compose -f /root/sandbox/compose/docker-compose.yml up -d
 }
 
 if [[ ${BASH_SOURCE[0]} == "$0" ]]; then

From 7b9728f414e287900bb27f67a4d600bcac321968 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Thu, 10 Mar 2022 10:18:50 -0500
Subject: [PATCH 21/25] tf: Add some interactive user goodies

Alias of dc=docker-compose alias for 700% effeciency gains. Alias tink for
even more gains!

Automatically cd'ing to /root/sandbox/compose for interactive logins because
that probably makes a lot of sense.

And a `tink` wrapper/helper script that calls docker-compose underneath. Much
nicer than `dc` calls everywhere and will even work with `watch`.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh       | 22 ++++++++++++++++++++++
 docs/quickstarts/TERRAFORMEM.md |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index eb0d5cfd..d20c75ec 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -126,6 +126,25 @@ setup_compose_env_overrides() {
 	done
 }
 
+create_tink_helper_script() {
+	cat >/usr/local/bin/tink <<-'EOF'
+		#!/usr/bin/env bash
+
+		exec docker-compose -f /root/sandbox/compose/docker-compose.yml exec tink-cli tink "$@"
+	EOF
+	chmod +x /usr/local/bin/tink
+}
+
+tweak_bash_interactive_settings() {
+	grep -q 'cd /root/sandbox/compose' ~root/.bashrc || echo 'cd /root/sandbox/compose' >>~root/.bashrc
+	readarray -t aliases <<-EOF
+		dc=docker-compose
+	EOF
+	for alias in "${aliases[@]}"; do
+		grep -q "$alias" ~root/.bash_aliases || echo "alias $alias" >>~root/.bash_aliases
+	done
+}
+
 main() {
 	worker_mac=$1
 	layer2_ip=192.168.56.4
@@ -143,6 +162,9 @@ main() {
 	extract_compose_files
 	setup_compose_env_overrides "$worker_mac"
 	docker-compose -f /root/sandbox/compose/docker-compose.yml up -d
+
+	create_tink_helper_script
+	tweak_bash_interactive_settings
 }
 
 if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
diff --git a/docs/quickstarts/TERRAFORMEM.md b/docs/quickstarts/TERRAFORMEM.md
index d9841b39..2d5ebbc8 100644
--- a/docs/quickstarts/TERRAFORMEM.md
+++ b/docs/quickstarts/TERRAFORMEM.md
@@ -53,7 +53,7 @@ This option will also show you how to create a machine to provision.
 
    # watch the workflow events and status for workflow completion
    # once the workflow is complete (see the expected output below for completion), move on to the next step
-   wid=$(docker exec -it compose_tink-cli_1 tink workflow get --no-headers | awk '/^\|/ {print $2}'); docker exec -it compose_tink-cli_1 watch -n1 "tink workflow events ${wid}; tink workflow state ${wid}"
+   wid=$(tink workflow get --no-headers | awk '/^\|/ {print $2}'); watch -n1 "tink workflow events ${wid}; tink workflow state ${wid}"
    ```
 
    <details>

From d0010f5fde1c17e36f4e6d50d6102891d1b40ddd Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Wed, 30 Mar 2022 15:26:31 -0400
Subject: [PATCH 22/25] tf: Use format instead of formatlist for worker_sos
 output

Makes it easier to use since we can avoid jq.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/outputs.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/terraform/outputs.tf b/deploy/terraform/outputs.tf
index beaf483a..d3e9e657 100644
--- a/deploy/terraform/outputs.tf
+++ b/deploy/terraform/outputs.tf
@@ -19,5 +19,5 @@ output "worker_macs" {
 }
 
 output "worker_sos" {
-  value = formatlist("%s@sos.%s.platformequinix.com", metal_device.tink_worker[*].id, metal_device.tink_worker.deployed_facility)
+  value = format("%s@sos.%s.platformequinix.com", metal_device.tink_worker.id, metal_device.tink_worker.deployed_facility)
 }

From 0f9d3c7976d31ec8f729f2181e7ad81d39fcd0c9 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Fri, 18 Mar 2022 16:01:44 -0400
Subject: [PATCH 23/25] vagrant: Move all provisioner code into just one script

This just seems easier to manage one script instead of many ruby blocks. Bonus
points for having it look similar to terraform/setup.sh so common things look
the same(ish).

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/vagrant/Vagrantfile | 41 ++-----------------------
 deploy/vagrant/setup.sh    | 63 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 39 deletions(-)
 create mode 100755 deploy/vagrant/setup.sh

diff --git a/deploy/vagrant/Vagrantfile b/deploy/vagrant/Vagrantfile
index 52a0883a..1464d26c 100644
--- a/deploy/vagrant/Vagrantfile
+++ b/deploy/vagrant/Vagrantfile
@@ -37,46 +37,9 @@ Vagrant.configure("2") do |config|
       override.vm.synced_folder "../compose/", "/vagrant/compose/", type: "rsync"
     end
 
-    provisioner.vm.provision "shell", name: "Setup interactive goodies", inline: <<-SCRIPT.gsub(/^ {6}/, "")
-      set -x
-      grep -q 'cd /vagrant/compose' ~vagrant/.bashrc || echo 'cd /vagrant/compose' >>~vagrant/.bashrc
-      read -r -d '' aliases <<-'EOF'
-        dc=docker-compose
-      EOF
-      while read -r alias; do
-        grep -q "$alias" ~vagrant/.bash_aliases || echo alias "$alias" >>~vagrant/.bash_aliases
-      done <<<"$aliases"
-    SCRIPT
-
-    provisioner.vm.provision "file", source: "tink", destination: "~/.local/bin/tink"
-
-    provisioner.vm.provision "shell", name: "Setup eth1", inline: <<-SCRIPT.gsub(/^ {6}/, "")
-      set -x
-      ip addr show dev eth1 | grep -q #{PROVISIONER_IP} && exit 0
-      ip addr add #{PROVISIONER_IP}/24 dev eth1
-      ip link set dev eth1 up
-    SCRIPT
-
-    provisioner.vm.provision "shell", name: "Setup provider specific overrides", inline: <<-SCRIPT.gsub(/^ {6}/, "")
-      set -x
-      if lsblk | grep -q vda; then
-        sed -i 's|sda|vda|g' /vagrant/compose/create-tink-records/manifests/template/ubuntu.yaml
-      fi
-      read -r -d '' lines <<-'EOF'
-      	TINKERBELL_HOST_IP=#{PROVISIONER_IP}
-      	TINKERBELL_CLIENT_IP=#{MACHINE1_IP}
-      EOF
-      while read -r line; do
-      	grep -q "$line" /vagrant/compose/.env && continue
-      	echo "$line" >>/vagrant/compose/.env
-      done <<<"$lines"
-    SCRIPT
-
     provisioner.vm.provision :docker
-    provisioner.vm.provision :docker_compose,
-      compose_version: "1.29.2",
-      yml: "/vagrant/compose/docker-compose.yml",
-      run: "always"
+    provisioner.vm.provision :docker_compose, compose_version: "1.29.2"
+    provisioner.vm.provision :shell, path: "setup.sh", args: [PROVISIONER_IP, MACHINE1_IP]
   end
 
   config.vm.define :machine1, autostart: false do |machine1|
diff --git a/deploy/vagrant/setup.sh b/deploy/vagrant/setup.sh
new file mode 100755
index 00000000..e650c055
--- /dev/null
+++ b/deploy/vagrant/setup.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+setup_layer2_network() {
+	host_addr=$1
+	ip addr show dev eth1 | grep -q "$host_addr" && return 0
+	ip addr add "$host_addr/24" dev eth1
+	ip link set dev eth1 up
+}
+
+setup_compose_env_overrides() {
+	local host_addr=$1
+	local worker_addr=$2
+	if lsblk | grep -q vda; then
+		sed -i 's|sda|vda|g' /vagrant/compose/create-tink-records/manifests/template/ubuntu.yaml
+	fi
+	readarray -t lines <<-EOF
+		TINKERBELL_HOST_IP="$host_addr"
+		TINKERBELL_CLIENT_IP="$worker_addr"
+	EOF
+	for line in "${lines[@]}"; do
+		grep -q "$line" /vagrant/compose/.env && continue
+		echo "$line" >>/vagrant/compose/.env
+	done
+}
+
+create_tink_helper_script() {
+	mkdir -p ~vagrant/.local/bin
+	cat >~vagrant/.local/bin/tink <<-'EOF'
+		#!/usr/bin/env bash
+
+		exec docker-compose -f /vagrant/compose/docker-compose.yml exec tink-cli tink "$@"
+	EOF
+	chmod +x ~vagrant/.local/bin/tink
+}
+
+tweak_bash_interactive_settings() {
+	grep -q 'cd /vagrant/compose' ~vagrant/.bashrc || echo 'cd /vagrant/compose' >>~vagrant/.bashrc
+	readarray -t aliases <<-EOF
+		dc=docker-compose
+	EOF
+	for alias in "${aliases[@]}"; do
+		grep -q "$alias" ~vagrant/.bash_aliases || echo "alias $alias" >>~vagrant/.bash_aliases
+	done
+}
+
+main() {
+	local host_addr=$1
+	local worker_addr=$2
+
+	setup_layer2_network "$host_addr"
+
+	setup_compose_env_overrides "$host_addr" "$worker_addr"
+	docker-compose -f /vagrant/compose/docker-compose.yml up -d
+
+	create_tink_helper_script
+	tweak_bash_interactive_settings
+}
+
+if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
+	set -euxo pipefail
+
+	main "$@"
+fi

From 0e5f93b351e8b9660e45ec2e98bea51c78b7b9ac Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Fri, 18 Mar 2022 16:15:35 -0400
Subject: [PATCH 24/25] vagrant: Install docker and docker-compose via setup.sh

Without this a user needs to run vagrant more than once the first time they
try this out, once to fetch the vagrant-docker-compose plugin and then again
to actually run through. Why go through that when we already have code for
install docker and docker-compose that we have to maintain seprately anyway
and leads to a better first experience too?

Another minor benefit is that we increase the code sharing between terraform
and vagrant's setup.sh, which is nice.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/vagrant/Vagrantfile |  8 --------
 deploy/vagrant/setup.sh    | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/deploy/vagrant/Vagrantfile b/deploy/vagrant/Vagrantfile
index 1464d26c..73f8a384 100644
--- a/deploy/vagrant/Vagrantfile
+++ b/deploy/vagrant/Vagrantfile
@@ -4,12 +4,6 @@
 PROVISIONER_IP = "192.168.56.4"
 MACHINE1_IP = "192.168.56.43"
 
-unless Vagrant.has_plugin?("vagrant-docker-compose")
-  system("vagrant plugin install vagrant-docker-compose")
-  puts "Dependencies installed, please try the command again."
-  exit
-end
-
 Vagrant.configure("2") do |config|
   config.vm.provider :libvirt do |libvirt|
     libvirt.qemu_use_session = false
@@ -37,8 +31,6 @@ Vagrant.configure("2") do |config|
       override.vm.synced_folder "../compose/", "/vagrant/compose/", type: "rsync"
     end
 
-    provisioner.vm.provision :docker
-    provisioner.vm.provision :docker_compose, compose_version: "1.29.2"
     provisioner.vm.provision :shell, path: "setup.sh", args: [PROVISIONER_IP, MACHINE1_IP]
   end
 
diff --git a/deploy/vagrant/setup.sh b/deploy/vagrant/setup.sh
index e650c055..72b5ee17 100755
--- a/deploy/vagrant/setup.sh
+++ b/deploy/vagrant/setup.sh
@@ -1,5 +1,34 @@
 #!/usr/bin/env bash
 
+install_docker() {
+	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+	add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+	update_apt
+	apt-get install --no-install-recommends containerd.io docker-ce docker-ce-cli
+	gpasswd -a vagrant docker
+}
+
+install_docker_compose() {
+	apt-get install --no-install-recommends python3-pip
+	pip install docker-compose
+}
+
+apt-get() {
+	DEBIAN_FRONTEND=noninteractive command apt-get \
+		--allow-change-held-packages \
+		--allow-downgrades \
+		--allow-remove-essential \
+		--allow-unauthenticated \
+		--option Dpkg::Options::=--force-confdef \
+		--option Dpkg::Options::=--force-confold \
+		--yes \
+		"$@"
+}
+
+update_apt() {
+	apt-get update
+}
+
 setup_layer2_network() {
 	host_addr=$1
 	ip addr show dev eth1 | grep -q "$host_addr" && return 0
@@ -47,6 +76,10 @@ main() {
 	local host_addr=$1
 	local worker_addr=$2
 
+	update_apt
+	install_docker
+	install_docker_compose
+
 	setup_layer2_network "$host_addr"
 
 	setup_compose_env_overrides "$host_addr" "$worker_addr"

From 33cac79e12b77882c460de5ca9dda2533863fd57 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <mmendez@equinix.com>
Date: Fri, 18 Mar 2022 16:05:41 -0400
Subject: [PATCH 25/25] deploy: Use the same folder path on both terraform and
 vagrant

Better to have everything look the same for easier diffing/comparing
vs having the tool name in the path.

Signed-off-by: Manuel Mendez <mmendez@equinix.com>
---
 deploy/terraform/setup.sh  | 14 +++++++-------
 deploy/vagrant/Vagrantfile |  6 +++---
 deploy/vagrant/setup.sh    | 12 ++++++------
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh
index d20c75ec..2e643304 100755
--- a/deploy/terraform/setup.sh
+++ b/deploy/terraform/setup.sh
@@ -109,8 +109,8 @@ make_host_gw_server() {
 }
 
 extract_compose_files() {
-	mkdir -p /root/sandbox
-	unzip /root/compose.zip -d /root/sandbox/compose
+	mkdir -p /sandbox
+	unzip /root/compose.zip -d /sandbox/compose
 }
 
 setup_compose_env_overrides() {
@@ -121,8 +121,8 @@ setup_compose_env_overrides() {
 		TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json
 	EOF
 	for line in "${lines[@]}"; do
-		grep -q "$line" /root/sandbox/compose/.env && continue
-		echo "$line" >>/root/sandbox/compose/.env
+		grep -q "$line" /sandbox/compose/.env && continue
+		echo "$line" >>/sandbox/compose/.env
 	done
 }
 
@@ -130,13 +130,13 @@ create_tink_helper_script() {
 	cat >/usr/local/bin/tink <<-'EOF'
 		#!/usr/bin/env bash
 
-		exec docker-compose -f /root/sandbox/compose/docker-compose.yml exec tink-cli tink "$@"
+		exec docker-compose -f /sandbox/compose/docker-compose.yml exec tink-cli tink "$@"
 	EOF
 	chmod +x /usr/local/bin/tink
 }
 
 tweak_bash_interactive_settings() {
-	grep -q 'cd /root/sandbox/compose' ~root/.bashrc || echo 'cd /root/sandbox/compose' >>~root/.bashrc
+	grep -q 'cd /sandbox/compose' ~root/.bashrc || echo 'cd /sandbox/compose' >>~root/.bashrc
 	readarray -t aliases <<-EOF
 		dc=docker-compose
 	EOF
@@ -161,7 +161,7 @@ main() {
 
 	extract_compose_files
 	setup_compose_env_overrides "$worker_mac"
-	docker-compose -f /root/sandbox/compose/docker-compose.yml up -d
+	docker-compose -f /sandbox/compose/docker-compose.yml up -d
 
 	create_tink_helper_script
 	tweak_bash_interactive_settings
diff --git a/deploy/vagrant/Vagrantfile b/deploy/vagrant/Vagrantfile
index 73f8a384..75180ff5 100644
--- a/deploy/vagrant/Vagrantfile
+++ b/deploy/vagrant/Vagrantfile
@@ -11,7 +11,7 @@ Vagrant.configure("2") do |config|
 
   config.vm.define "provisioner" do |provisioner|
     provisioner.vm.box = "generic/ubuntu2004"
-    provisioner.vm.synced_folder "../compose/", "/vagrant/compose/"
+    provisioner.vm.synced_folder "../compose/", "/sandbox/compose/"
     provisioner.vm.network "private_network", ip: PROVISIONER_IP,
                                               libvirt__network_name: "tink_network",
                                               libvirt__host_ip: "192.168.56.1",
@@ -22,13 +22,13 @@ Vagrant.configure("2") do |config|
     provisioner.vm.provider "virtualbox" do |v, override|
       v.memory = 2048
       v.cpus = 2
-      override.vm.synced_folder "../compose/", "/vagrant/compose/"
+      override.vm.synced_folder "../compose/", "/sandbox/compose/"
     end
 
     provisioner.vm.provider "libvirt" do |l, override|
       l.memory = 2048
       l.cpus = 2
-      override.vm.synced_folder "../compose/", "/vagrant/compose/", type: "rsync"
+      override.vm.synced_folder "../compose/", "/sandbox/compose/", type: "rsync"
     end
 
     provisioner.vm.provision :shell, path: "setup.sh", args: [PROVISIONER_IP, MACHINE1_IP]
diff --git a/deploy/vagrant/setup.sh b/deploy/vagrant/setup.sh
index 72b5ee17..2c1398f0 100755
--- a/deploy/vagrant/setup.sh
+++ b/deploy/vagrant/setup.sh
@@ -40,15 +40,15 @@ setup_compose_env_overrides() {
 	local host_addr=$1
 	local worker_addr=$2
 	if lsblk | grep -q vda; then
-		sed -i 's|sda|vda|g' /vagrant/compose/create-tink-records/manifests/template/ubuntu.yaml
+		sed -i 's|sda|vda|g' /sandbox/compose/create-tink-records/manifests/template/ubuntu.yaml
 	fi
 	readarray -t lines <<-EOF
 		TINKERBELL_HOST_IP="$host_addr"
 		TINKERBELL_CLIENT_IP="$worker_addr"
 	EOF
 	for line in "${lines[@]}"; do
-		grep -q "$line" /vagrant/compose/.env && continue
-		echo "$line" >>/vagrant/compose/.env
+		grep -q "$line" /sandbox/compose/.env && continue
+		echo "$line" >>/sandbox/compose/.env
 	done
 }
 
@@ -57,13 +57,13 @@ create_tink_helper_script() {
 	cat >~vagrant/.local/bin/tink <<-'EOF'
 		#!/usr/bin/env bash
 
-		exec docker-compose -f /vagrant/compose/docker-compose.yml exec tink-cli tink "$@"
+		exec docker-compose -f /sandbox/compose/docker-compose.yml exec tink-cli tink "$@"
 	EOF
 	chmod +x ~vagrant/.local/bin/tink
 }
 
 tweak_bash_interactive_settings() {
-	grep -q 'cd /vagrant/compose' ~vagrant/.bashrc || echo 'cd /vagrant/compose' >>~vagrant/.bashrc
+	grep -q 'cd /sandbox/compose' ~vagrant/.bashrc || echo 'cd /sandbox/compose' >>~vagrant/.bashrc
 	readarray -t aliases <<-EOF
 		dc=docker-compose
 	EOF
@@ -83,7 +83,7 @@ main() {
 	setup_layer2_network "$host_addr"
 
 	setup_compose_env_overrides "$host_addr" "$worker_addr"
-	docker-compose -f /vagrant/compose/docker-compose.yml up -d
+	docker-compose -f /sandbox/compose/docker-compose.yml up -d
 
 	create_tink_helper_script
 	tweak_bash_interactive_settings