Create multiple supernodes

README.md

@@ -10,3 +10,8 @@
   - each supernode gets one IP from this prefix
 - Assign a management IP from the site management net
   - Statically set on the VM config
+
+## Running Ansible
+
+- Set `NETBOX_TOKEN` to your token
+- Run `ansible-playbook site.yml --diff --check`

ansible.cfg

@@ -6,6 +6,7 @@ pipelining = true
 timeout = 15
 roles_path = roles
 remote_user = admin
+host_key_checking = False
 
 [ssh_connection]
 scp_if_ssh = true

group_vars/all.yaml

@@ -26,7 +26,18 @@ gateway_ipv6_address: >-
       first
   }}
 
+client_ipv4_start_address_int: "{{ gateway_ipv4_address | ansible.utils.ipaddr('network') | ansible.utils.ipaddr('int') }}"
+dhcp_range_start_address_int: "{{ (config_context|first).dhcp_range.start_address|ansible.utils.ipaddr('int') }}"
+dhcp_range_end_address_int: "{{ (config_context|first).dhcp_range.end_address|ansible.utils.ipaddr('int') }}"
+dhcp_pool_offset: "{{ (dhcp_range_start_address_int|int) - (client_ipv4_start_address_int|int) - 1 }}"
+dhcp_pool_size: "{{ (dhcp_range_end_address_int|int) - (dhcp_range_start_address_int|int) }}"
+
+# Private Anycast Address in the domain used for reaching DHCP & DNS
+# todo: replace with data from netbox
+anycast_service_address: "{{ gateway_ipv4_address | ansible.utils.ipaddr('network/prefix') | ansible.utils.ipaddr('-2') }}"
+
 wan_interface: "{{ interfaces | selectattr('name', 'equalto', 'eth0') | first }}"
+# Outside IP used for connecting to the supernode from the Internet
 service_ipv4_address: >-
   {{ wan_interface.ip_addresses |
       map(attribute='address') |
@@ -39,3 +50,5 @@ loopback_interface: "{{ interfaces | selectattr('name', 'equalto', 'lo') | first
 service_ipv6_address: "{{ gateway_ipv6_address }}"
 
 domain_ipv6_subnet: "{{ gateway_ipv6_address | ansible.utils.ipaddr('network/prefix') }}"
+
+batbone_interface: "{{ interfaces | selectattr('name', 'equalto', 'eth1') | first }}"

roles/batman/tasks/main.yaml

@@ -15,26 +15,15 @@
     line: batman-adv
     create: true
 
-- name: Create Batman Interface
+- name: Configure Networking
   ansible.builtin.template:
-    src: batman.netdev.j2
-    dest: /etc/systemd/network/batman.netdev
-  notify: reload network
-
-- name: Configure Batman Interface
-  ansible.builtin.template:
-    src: batman.network.j2
-    dest: /etc/systemd/network/batman.network
-  notify: reload network
-
-- name: Create Batman Bridge
-  ansible.builtin.template:
-    src: batman-bridge.netdev.j2
-    dest: /etc/systemd/network/batman-bridge.netdev
-  notify: reload network
-
-- name: Configure Batman Bridge
-  ansible.builtin.template:
-    src: batman-bridge.network.j2
-    dest: /etc/systemd/network/batman-bridge.network
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/network/{{ item }}"
+  loop:
+    - batman.netdev
+    - batman.network
+    - batman-bridge.netdev
+    - batman-bridge.network
+    - batbone.link
+    - batbone.network
   notify: reload network

roles/batman/templates/batbone.link.j2

@@ -0,0 +1,5 @@
+[Match]
+MACAddress={{ (batbone_interface.mac_addresses | first).mac_address }}
+
+[Link]
+Name=eth1

roles/batman/templates/batbone.network.j2

@@ -0,0 +1,8 @@
+[Match]
+MACAddress={{ (batbone_interface.mac_addresses | first).mac_address }}
+
+[Link]
+RequiredForOnline=no
+
+[Network]
+Bridge=br1

roles/gateway/templates/client-bridge.network.j2

@@ -12,11 +12,11 @@ DHCPServer=yes
 IPv6SendRA=yes
 
 [DHCPServer]
-PoolOffset=10
-PoolSize=64000
+PoolOffset={{ dhcp_pool_offset }}
+PoolSize={{ dhcp_pool_size }}
 EmitDNS=yes
-ServerAddress=10.12.255.254/16
-DNS={{ service_ipv4_address | ansible.utils.ipaddr('address') }}
+ServerAddress={{ anycast_service_address }}
+DNS={{ anycast_service_address | ansible.utils.ipaddr('address') }}
 
 {% for addr in client_bridge_interface.ip_addresses | map(attribute='address') | ansible.utils.ipv6 %}
 [IPv6Prefix]

roles/gateway/templates/dnsmasq.conf.j2

@@ -1,6 +1,6 @@
 interface=br0
 except-interface=lo
-listen-address={{ gateway_ipv4_address | ansible.utils.ipaddr('address') }}
+listen-address={{ anycast_service_address | ansible.utils.ipaddr('address') }}
 listen-address={{ gateway_ipv6_address | ansible.utils.ipaddr('address') }}
 bind-interfaces
 

terraform/batbone.tf

@@ -0,0 +1,8 @@
+resource "netbox_available_vlan" "batbone" {
+  name        = "batbone ${var.domain_name}"
+  status      = "active"
+  description = "Batbone VLAN for ${var.domain_name}"
+
+  group_id = var.batbone_vlan_group_id
+  site_id  = data.netbox_site.local.id
+}

terraform/domains/dev.tfvars

@@ -1,2 +1,4 @@
 domain_name = "dev"
 domain_id   = 12
+
+supernode_count = 2

terraform/main.tf

@@ -1,14 +1,21 @@
+data "netbox_site" "local" {
+  name = var.site_name
+}
+
 module "supernode" {
   count = var.supernode_count
 
   source = "./modules/supernode"
 
-  supernode_name = "${var.domain_name}-${count.index}"
+  supernode_name  = "${var.domain_name}-${count.index}"
+  supernode_index = count.index
 
   public_ipv4_prefix_id = data.netbox_prefix.primary_ipv4.id
-  domain_ipv4_id        = netbox_prefix.domain_ipv4.id
-  domain_ipv6_id        = netbox_prefix.domain_ipv6.id
+  domain_ipv4_prefix    = netbox_prefix.domain_ipv4.prefix
+  domain_ipv6_prefix    = netbox_prefix.domain_ipv6.prefix
   domain_vrf_id         = data.netbox_vrf.mesh.id
+  site_id               = data.netbox_site.local.id
+  batbone_vlan          = netbox_available_vlan.batbone.vid
 
   vm_ssh_keys = local.ssh_keys
 }

terraform/modules/supernode/interfaces.tf

@@ -1,14 +1,42 @@
-resource "macaddress" "eth0" {}
+resource "macaddress" "vm" {
+  for_each = toset(["eth0", "eth1"])
+}
 
 resource "netbox_interface" "eth0" {
   virtual_machine_id = netbox_virtual_machine.supernode.id
 
-  name        = "eth0"
-  mac_address = macaddress.eth0.address
+  name = "eth0"
+  tags = toset(var.tags)
+}
+
+resource "netbox_mac_address" "eth0" {
+  mac_address  = macaddress.vm["eth0"].address
+  interface_id = netbox_interface.eth0.id
+  object_type  = "virtualization.vminterface"
+}
+
+data "netbox_vlan" "batbone" {
+  vid = var.batbone_vlan
+}
+
+resource "netbox_interface" "eth1" {
+  virtual_machine_id = netbox_virtual_machine.supernode.id
+
+  name        = "eth1"
+  description = "Batbone"
+
+  mode          = "access"
+  untagged_vlan = data.netbox_vlan.batbone.id
 
   tags = toset(var.tags)
 }
 
+resource "netbox_mac_address" "eth1" {
+  mac_address  = macaddress.vm["eth1"].address
+  interface_id = netbox_interface.eth1.id
+  object_type  = "virtualization.vminterface"
+}
+
 resource "netbox_interface" "lo" {
   virtual_machine_id = netbox_virtual_machine.supernode.id
   name               = "lo"

terraform/modules/supernode/management-ipv6.tf

@@ -4,7 +4,7 @@ data "netbox_prefix" "management_net" {
 }
 
 data "iphelpers_eui64_address" "supernode_management" {
-  mac_address = macaddress.eth0.address
+  mac_address = macaddress.vm["eth0"].address
   prefix      = trimsuffix(data.netbox_prefix.management_net.prefix, "/64")
 }