diff --git a/.envrc b/.envrc index b8b78324..0fc3eabb 100644 --- a/.envrc +++ b/.envrc @@ -1 +1,2 @@ -which nix &>/dev/null && use nix +has nix && use nix +dotenv_if_exists diff --git a/.gitignore b/.gitignore index 091a8317..7491f518 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,23 @@ +# hidden files/dirs +.* +!deploy/compose/state/webroot/misc/osie/current/.keep +!deploy/compose/state/webroot/workflow/.keep +!deploy/.env +!.gitignore + # Local .terraform directories -**/.terraform/* .terraform* +!.terraform.lock.hcl # .tfstate files *.tfstate *.tfstate.* -!.terraform.lock.hcl -envrc -out -!deploy/.env -.vagrant +compose.tar.gz +compose.zip +deploy/compose/state/webroot/*.gz deploy/compose/state/webroot/misc/osie/current/* deploy/compose/state/webroot/workflow/* -!deploy/compose/state/webroot/misc/osie/current/.keep -!deploy/compose/state/webroot/workflow/.keep -deploy/compose/state/webroot/*.gz +envrc +out workflow_id.txt -compose.tar.gz -compose.zip diff --git a/deploy/compose/state/webroot/.gitignore b/deploy/compose/state/webroot/.gitignore deleted file mode 100644 index 10d00b57..00000000 --- a/deploy/compose/state/webroot/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.gz diff --git a/deploy/terraform/.gitignore b/deploy/terraform/.gitignore deleted file mode 100644 index 1918c7f0..00000000 --- a/deploy/terraform/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.terraform -terraform.tfstate -terraform.tfstate.backup -terraform.tfvars -.terraform.lock.hcl diff --git a/deploy/terraform/cloud-config.cfg b/deploy/terraform/cloud-config.cfg index 9ce7a7a1..c9562c9c 100644 --- a/deploy/terraform/cloud-config.cfg +++ b/deploy/terraform/cloud-config.cfg @@ -6,7 +6,11 @@ write_files: content: ${COMPOSE_ZIP} path: /root/compose.zip -runcmd: -- cd /root/sandbox/compose && unzip /root/compose.zip -- cd /root/sandbox/compose && TINKERBELL_CLIENT_MAC=${WORKER_MAC} TINKERBELL_TEMPLATE_MANIFEST=/manifests/template/ubuntu-equinix-metal.yaml TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json docker-compose up -d +- encoding: b64 + content: ${SETUPSH} + path: /root/setup.sh + owner: root:root + permissions: "0755" +runcmd: +- /root/setup.sh ${WORKER_MAC} diff --git a/deploy/terraform/main.tf b/deploy/terraform/main.tf index 81fc872e..7643df7b 100644 --- a/deploy/terraform/main.tf +++ b/deploy/terraform/main.tf @@ -20,7 +20,7 @@ provider "metal" { auth_token = var.metal_api_token } -# Create a new VLAN in datacenter "ewr1" +# Create a new VLAN in datacenter resource "metal_vlan" "provisioning_vlan" { description = "provisioning_vlan" metro = var.metro @@ -85,6 +85,7 @@ data "archive_file" "compose" { locals { compose_zip = data.archive_file.compose.output_size > 0 ? filebase64("${path.module}/compose.zip") : "" + worker_macs = flatten([for wp in metal_device.tink_worker[*].ports[*] : [for p in wp : p.mac if p.name == "eth0"]]) } data "cloudinit_config" "setup" { @@ -94,15 +95,12 @@ data "cloudinit_config" "setup" { gzip = false # not supported on Equinix Metal base64_encode = false # not supported on Equinix Metal - part { - content_type = "text/x-shellscript" - content = file("${path.module}/setup.sh") - } part { content_type = "text/cloud-config" content = templatefile("${path.module}/cloud-config.cfg", { COMPOSE_ZIP = local.compose_zip - WORKER_MAC = metal_device.tink_worker.ports[1].mac + SETUPSH = filebase64("${path.module}/setup.sh") + WORKER_MAC = local.worker_macs[0] }) } } diff --git a/deploy/terraform/outputs.tf b/deploy/terraform/outputs.tf index d27a9a41..d3e9e657 100644 --- a/deploy/terraform/outputs.tf +++ b/deploy/terraform/outputs.tf @@ -2,6 +2,22 @@ output "provisioner_ip" { value = metal_device.tink_provisioner.network[0].address } +output "provisioner_id" { + value = metal_device.tink_provisioner.id +} + +output "provisioner_ssh" { + value = format("%s.packethost.net", split("-", metal_device.tink_provisioner.id)[0]) +} + +output "worker_id" { + value = metal_device.tink_worker.id +} + +output "worker_macs" { + value = local.worker_macs +} + output "worker_sos" { - value = formatlist("%s@sos.%s.platformequinix.com", metal_device.tink_worker[*].id, metal_device.tink_worker.deployed_facility) + value = format("%s@sos.%s.platformequinix.com", metal_device.tink_worker.id, metal_device.tink_worker.deployed_facility) } diff --git a/deploy/terraform/setup.sh b/deploy/terraform/setup.sh index 69a33bc0..2e643304 100755 --- a/deploy/terraform/setup.sh +++ b/deploy/terraform/setup.sh @@ -1,66 +1,174 @@ #!/usr/bin/env bash -set -xo pipefail - install_docker() { curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" update_apt - DEBIAN_FRONTEND=noninteractive apt install -y apt-transport-https ca-certificates curl gnupg-agent gnupg2 software-properties-common docker-ce docker-ce-cli containerd.io + apt-get install --no-install-recommends containerd.io docker-ce docker-ce-cli } install_docker_compose() { - curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose + apt-get install --no-install-recommends python3-pip + pip install docker-compose } -update_apt() ( - $APT update - DEBIAN_FRONTEND=noninteractive $APT --yes --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade -) +install_iptables_persistent() { + apt-get install --no-install-recommends iptables-persistent +} -restart_docker_service() ( - service docker restart -) +apt-get() { + DEBIAN_FRONTEND=noninteractive command apt-get \ + --allow-change-held-packages \ + --allow-downgrades \ + --allow-remove-essential \ + --allow-unauthenticated \ + --option Dpkg::Options::=--force-confdef \ + --option Dpkg::Options::=--force-confold \ + --yes \ + "$@" +} + +update_apt() { + apt-get update + apt-get upgrade +} # get_second_interface_from_bond0 returns the second interface of the bond0 interface get_second_interface_from_bond0() { - local return_value - return_value=$(cut -d' ' -f2 /sys/class/net/bond0/bonding/slaves | xargs) - echo "${return_value}" + local addr=$1 + + # if the ip is in a file in interfaces.d then lets assume this is a re-run and we can just + # return the basename of the file (which should be named same as the interface) + f=$(grep -lr "${addr}" /etc/network/interfaces.d) + [[ -n ${f:-} ]] && basename "$f" && return + + # sometimes the interfaces aren't sorted as expected in the /slaves file + # + # seeing as how this function is named *second* I figured its best to be + # precise (via head -n2) when choosing the iface instead of choosing the last + # iface and hoping there are only 2 + tr ' ' '\n' "/etc/network/interfaces.d/${interface}" <<-EOF + auto ${interface} + iface ${interface} inet static + address ${addr} + EOF } # make_host_gw_server makes the host a gateway server make_host_gw_server() { - local incoming_interface="$1" - local outgoing_interface="$2" + local incoming_interface=$1 + local outgoing_interface=$2 + + # drop all rules, especially interested in droppin docker's we don't want to persist docker's rules + # docker will re-create them when starting back up + systemctl stop docker + netfilter-persistent flush + iptables -t nat -A POSTROUTING -o "${outgoing_interface}" -j MASQUERADE iptables -A FORWARD -i "${outgoing_interface}" -o "${incoming_interface}" -m state --state RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -i "${incoming_interface}" -o "${outgoing_interface}" -j ACCEPT + + netfilter-persistent save + systemctl start docker +} + +extract_compose_files() { + mkdir -p /sandbox + unzip /root/compose.zip -d /sandbox/compose +} + +setup_compose_env_overrides() { + local worker_mac=$1 + readarray -t lines <<-EOF + TINKERBELL_CLIENT_MAC=$worker_mac + TINKERBELL_TEMPLATE_MANIFEST=/manifests/template/ubuntu-equinix-metal.yaml + TINKERBELL_HARDWARE_MANIFEST=/manifests/hardware/hardware-equinix-metal.json + EOF + for line in "${lines[@]}"; do + grep -q "$line" /sandbox/compose/.env && continue + echo "$line" >>/sandbox/compose/.env + done +} + +create_tink_helper_script() { + cat >/usr/local/bin/tink <<-'EOF' + #!/usr/bin/env bash + + exec docker-compose -f /sandbox/compose/docker-compose.yml exec tink-cli tink "$@" + EOF + chmod +x /usr/local/bin/tink +} + +tweak_bash_interactive_settings() { + grep -q 'cd /sandbox/compose' ~root/.bashrc || echo 'cd /sandbox/compose' >>~root/.bashrc + readarray -t aliases <<-EOF + dc=docker-compose + EOF + for alias in "${aliases[@]}"; do + grep -q "$alias" ~root/.bash_aliases || echo "alias $alias" >>~root/.bash_aliases + done } -main() ( - #local provisioner_ip="$1" +main() { + worker_mac=$1 + layer2_ip=192.168.56.4 + update_apt install_docker install_docker_compose - restart_docker_service - mkdir -p /root/sandbox/compose + install_iptables_persistent + local layer2_interface - layer2_interface="$(get_second_interface_from_bond0)" - setup_layer2_network "${layer2_interface}" #"${provisioner_ip}" - make_host_gw_server "${layer2_interface}" "bond0" -) + layer2_interface=$(get_second_interface_from_bond0 ${layer2_ip}) + setup_layer2_network "${layer2_interface}" ${layer2_ip} + make_host_gw_server "${layer2_interface}" bond0 + + extract_compose_files + setup_compose_env_overrides "$worker_mac" + docker-compose -f /sandbox/compose/docker-compose.yml up -d + + create_tink_helper_script + tweak_bash_interactive_settings +} + +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + set -euxo pipefail -main #"$1" + main "$@" +fi diff --git a/deploy/vagrant/Vagrantfile b/deploy/vagrant/Vagrantfile index 52a0883a..75180ff5 100644 --- a/deploy/vagrant/Vagrantfile +++ b/deploy/vagrant/Vagrantfile @@ -4,12 +4,6 @@ PROVISIONER_IP = "192.168.56.4" MACHINE1_IP = "192.168.56.43" -unless Vagrant.has_plugin?("vagrant-docker-compose") - system("vagrant plugin install vagrant-docker-compose") - puts "Dependencies installed, please try the command again." - exit -end - Vagrant.configure("2") do |config| config.vm.provider :libvirt do |libvirt| libvirt.qemu_use_session = false @@ -17,7 +11,7 @@ Vagrant.configure("2") do |config| config.vm.define "provisioner" do |provisioner| provisioner.vm.box = "generic/ubuntu2004" - provisioner.vm.synced_folder "../compose/", "/vagrant/compose/" + provisioner.vm.synced_folder "../compose/", "/sandbox/compose/" provisioner.vm.network "private_network", ip: PROVISIONER_IP, libvirt__network_name: "tink_network", libvirt__host_ip: "192.168.56.1", @@ -28,55 +22,16 @@ Vagrant.configure("2") do |config| provisioner.vm.provider "virtualbox" do |v, override| v.memory = 2048 v.cpus = 2 - override.vm.synced_folder "../compose/", "/vagrant/compose/" + override.vm.synced_folder "../compose/", "/sandbox/compose/" end provisioner.vm.provider "libvirt" do |l, override| l.memory = 2048 l.cpus = 2 - override.vm.synced_folder "../compose/", "/vagrant/compose/", type: "rsync" + override.vm.synced_folder "../compose/", "/sandbox/compose/", type: "rsync" end - provisioner.vm.provision "shell", name: "Setup interactive goodies", inline: <<-SCRIPT.gsub(/^ {6}/, "") - set -x - grep -q 'cd /vagrant/compose' ~vagrant/.bashrc || echo 'cd /vagrant/compose' >>~vagrant/.bashrc - read -r -d '' aliases <<-'EOF' - dc=docker-compose - EOF - while read -r alias; do - grep -q "$alias" ~vagrant/.bash_aliases || echo alias "$alias" >>~vagrant/.bash_aliases - done <<<"$aliases" - SCRIPT - - provisioner.vm.provision "file", source: "tink", destination: "~/.local/bin/tink" - - provisioner.vm.provision "shell", name: "Setup eth1", inline: <<-SCRIPT.gsub(/^ {6}/, "") - set -x - ip addr show dev eth1 | grep -q #{PROVISIONER_IP} && exit 0 - ip addr add #{PROVISIONER_IP}/24 dev eth1 - ip link set dev eth1 up - SCRIPT - - provisioner.vm.provision "shell", name: "Setup provider specific overrides", inline: <<-SCRIPT.gsub(/^ {6}/, "") - set -x - if lsblk | grep -q vda; then - sed -i 's|sda|vda|g' /vagrant/compose/create-tink-records/manifests/template/ubuntu.yaml - fi - read -r -d '' lines <<-'EOF' - TINKERBELL_HOST_IP=#{PROVISIONER_IP} - TINKERBELL_CLIENT_IP=#{MACHINE1_IP} - EOF - while read -r line; do - grep -q "$line" /vagrant/compose/.env && continue - echo "$line" >>/vagrant/compose/.env - done <<<"$lines" - SCRIPT - - provisioner.vm.provision :docker - provisioner.vm.provision :docker_compose, - compose_version: "1.29.2", - yml: "/vagrant/compose/docker-compose.yml", - run: "always" + provisioner.vm.provision :shell, path: "setup.sh", args: [PROVISIONER_IP, MACHINE1_IP] end config.vm.define :machine1, autostart: false do |machine1| diff --git a/deploy/vagrant/setup.sh b/deploy/vagrant/setup.sh new file mode 100755 index 00000000..2c1398f0 --- /dev/null +++ b/deploy/vagrant/setup.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +install_docker() { + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + update_apt + apt-get install --no-install-recommends containerd.io docker-ce docker-ce-cli + gpasswd -a vagrant docker +} + +install_docker_compose() { + apt-get install --no-install-recommends python3-pip + pip install docker-compose +} + +apt-get() { + DEBIAN_FRONTEND=noninteractive command apt-get \ + --allow-change-held-packages \ + --allow-downgrades \ + --allow-remove-essential \ + --allow-unauthenticated \ + --option Dpkg::Options::=--force-confdef \ + --option Dpkg::Options::=--force-confold \ + --yes \ + "$@" +} + +update_apt() { + apt-get update +} + +setup_layer2_network() { + host_addr=$1 + ip addr show dev eth1 | grep -q "$host_addr" && return 0 + ip addr add "$host_addr/24" dev eth1 + ip link set dev eth1 up +} + +setup_compose_env_overrides() { + local host_addr=$1 + local worker_addr=$2 + if lsblk | grep -q vda; then + sed -i 's|sda|vda|g' /sandbox/compose/create-tink-records/manifests/template/ubuntu.yaml + fi + readarray -t lines <<-EOF + TINKERBELL_HOST_IP="$host_addr" + TINKERBELL_CLIENT_IP="$worker_addr" + EOF + for line in "${lines[@]}"; do + grep -q "$line" /sandbox/compose/.env && continue + echo "$line" >>/sandbox/compose/.env + done +} + +create_tink_helper_script() { + mkdir -p ~vagrant/.local/bin + cat >~vagrant/.local/bin/tink <<-'EOF' + #!/usr/bin/env bash + + exec docker-compose -f /sandbox/compose/docker-compose.yml exec tink-cli tink "$@" + EOF + chmod +x ~vagrant/.local/bin/tink +} + +tweak_bash_interactive_settings() { + grep -q 'cd /sandbox/compose' ~vagrant/.bashrc || echo 'cd /sandbox/compose' >>~vagrant/.bashrc + readarray -t aliases <<-EOF + dc=docker-compose + EOF + for alias in "${aliases[@]}"; do + grep -q "$alias" ~vagrant/.bash_aliases || echo "alias $alias" >>~vagrant/.bash_aliases + done +} + +main() { + local host_addr=$1 + local worker_addr=$2 + + update_apt + install_docker + install_docker_compose + + setup_layer2_network "$host_addr" + + setup_compose_env_overrides "$host_addr" "$worker_addr" + docker-compose -f /sandbox/compose/docker-compose.yml up -d + + create_tink_helper_script + tweak_bash_interactive_settings +} + +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + set -euxo pipefail + + main "$@" +fi diff --git a/docs/quickstarts/TERRAFORMEM.md b/docs/quickstarts/TERRAFORMEM.md index 187bbf7d..2d5ebbc8 100644 --- a/docs/quickstarts/TERRAFORMEM.md +++ b/docs/quickstarts/TERRAFORMEM.md @@ -42,17 +42,18 @@ This option will also show you how to create a machine to provision. Or if you have the [Equinix Metal CLI](https://github.com/equinix/metal-cli) installed run the following: ```bash - metal device reboot -i $(terraform show -json | jq -r '.values.root_module.resources[3].values.id') + metal device reboot -i $(terraform output -raw worker_id) ``` 5. Watch the provision complete ```bash # log in to the provisioner - ssh root@139.178.69.231 + ssh root@$(terraform output -raw provisioner_ssh) + # watch the workflow events and status for workflow completion # once the workflow is complete (see the expected output below for completion), move on to the next step - wid=$(docker exec -it compose_tink-cli_1 tink workflow get --no-headers | awk '/^\|/ {print $2}'); docker exec -it compose_tink-cli_1 watch -n1 "tink workflow events ${wid}; tink workflow state ${wid}" + wid=$(tink workflow get --no-headers | awk '/^\|/ {print $2}'); watch -n1 "tink workflow events ${wid}; tink workflow state ${wid}" ```
@@ -94,7 +95,7 @@ This option will also show you how to create a machine to provision. Now reboot the `tink-worker` via the [Equinix Metal Web UI](https://console.equinix.com), or if you have the [Equinix Metal CLI](https://github.com/equinix/metal-cli) installed run the following: ```bash - metal device reboot -i $(terraform show -json | jq -r '.values.root_module.resources[3].values.id') + metal device reboot -i $(terraform output -raw worker_id) ``` 7. Login to the machine