Use a single router process in attachment points (#381)

Use a single router process in attachment points, vs previously
distributing the interfaces over many routers with max 12 interfaces
each. This was a workaround for some issue in an old version of the
router. Now, this is no longer necessary and running many instances is
harmful for the system's performance.

Remove the automatic splitting of interfaces over many routers.
Once this change is deployed, the script `merge_ap_routers` should be
run to merge and clean up the left router instances.

One notable caveat: the router, as it is implemented now, uses a
blocking write, which can in this configuration potentially slow down
processing of the downstream traffic to the slowest User AS link speed.
This should be fixed in the router implementation, hopefully soon.

Author: matzf

scionlab/fixtures/testdata.yaml

@@ -4219,7 +4219,7 @@
     AS: 4
     interface_id: 2
     host: 4
-    border_router: 17
+    border_router: 4
     public_ip: 10.0.8.1
     public_port: 50001
     bind_ip: null
@@ -4239,7 +4239,7 @@
     AS: 7
     interface_id: 3
     host: 7
-    border_router: 19
+    border_router: 7
     public_ip: null
     public_port: 50002
     bind_ip: null
@@ -4259,7 +4259,7 @@
     AS: 13
     interface_id: 3
     host: 13
-    border_router: 21
+    border_router: 13
     public_ip: null
     public_port: 50002
     bind_ip: null
@@ -4279,7 +4279,7 @@
     AS: 14
     interface_id: 3
     host: 14
-    border_router: 23
+    border_router: 14
     public_ip: 10.8.0.1
     public_port: 50002
     bind_ip: null
@@ -4299,7 +4299,7 @@
     AS: 4
     interface_id: 3
     host: 4
-    border_router: 17
+    border_router: 4
     public_ip: null
     public_port: 50002
     bind_ip: null
@@ -4600,41 +4600,21 @@
   fields:
     AS: 16
     host: 16
-- model: scionlab.borderrouter
-  pk: 17
-  fields:
-    AS: 4
-    host: 4
 - model: scionlab.borderrouter
   pk: 18
   fields:
     AS: 17
     host: 17
-- model: scionlab.borderrouter
-  pk: 19
-  fields:
-    AS: 7
-    host: 7
 - model: scionlab.borderrouter
   pk: 20
   fields:
     AS: 18
     host: 18
-- model: scionlab.borderrouter
-  pk: 21
-  fields:
-    AS: 13
-    host: 13
 - model: scionlab.borderrouter
   pk: 22
   fields:
     AS: 19
     host: 19
-- model: scionlab.borderrouter
-  pk: 23
-  fields:
-    AS: 14
-    host: 14
 - model: scionlab.borderrouter
   pk: 24
   fields:

scionlab/models/user_as.py

@@ -211,8 +211,6 @@ def update_attachments(self,
             self._create_or_update_attachment(att_conf)
         for deleted_link in deleted_links:
             self._delete_attachment(deleted_link)
-        for ap in aps_set:
-            ap.split_border_routers()
         self._deactivate_unused_vpn_clients(att_confs)
 
         self.save()
@@ -482,8 +480,6 @@ def get_border_router_for_useras_interface(self) -> BorderRouter:
         """
         Selects the preferred border router on which the Interfaces to UserASes should be configured
 
-        Note: the border router effectively used will be be overwritten by `split_border_routers`.
-
         :returns: a `BorderRouter` of the related `AS`
         """
         host = self._get_host_for_useras_attachment()
@@ -515,51 +511,6 @@ def _get_host_for_useras_attachment(self) -> Host:
             host = self.AS.hosts.filter(public_ip__isnull=False)[0]
         return host
 
-    def split_border_routers(self, max_ifaces=10):
-        """
-        This is a workaround for an (apparent) issue with the border router, that cannot handle more
-        than ~12 interfaces per process; this problem seemed to be fixed but is apparently still
-        here. This is a hopefully temporary patch.
-
-        Will create / remove border routers so no one of them has more than
-        the specified limit of interfaces. The links to parent ASes will
-        always remain in a different border router.
-        :param int max_ifaces The maximum number of interfaces per BR
-        """
-        host = self._get_host_for_useras_attachment()
-        # find the *active* interfaces attaching for UserASes (attaching_ifaces) and the rest
-        # (infra_ifaces)
-        ifaces = host.interfaces.active().order_by('interface_id')
-        attaching_ifaces = ifaces.filter(
-            link_as_interfaceA__type=Link.PROVIDER,
-            link_as_interfaceA__interfaceB__AS__owner__isnull=False)
-        infra_ifaces = ifaces.exclude(pk__in=attaching_ifaces)
-
-        # attaching non children all to one BR:
-        infra_br = BorderRouter.objects.first_or_create(host)
-        brs_to_delete = list(
-            host.border_routers.order_by('pk').exclude(pk=infra_br.pk).values_list('pk', flat=True))
-        brs_to_delete.reverse()
-        infra_ifaces.update(border_router=infra_br)
-        # attaching children to several BRs:
-        attaching_ifaces = attaching_ifaces.all()
-        for i in range(0, len(attaching_ifaces), max_ifaces):
-            if brs_to_delete:
-                br = BorderRouter.objects.get(pk=brs_to_delete.pop())
-            else:
-                br = BorderRouter.objects.create(host=host)
-            for j in range(i, min(len(attaching_ifaces), i + max_ifaces)):
-                iface = attaching_ifaces[j]
-                iface.border_router = br
-                iface.save()
-            br.save()
-        # squirrel away the *inactive* interfaces somewhere...
-        host.interfaces.inactive().update(border_router=infra_br)
-
-        # delete old BRs
-        if brs_to_delete:
-            BorderRouter.objects.filter(pk__in=brs_to_delete).delete()
-
     @staticmethod
     def from_link(link: Link) -> 'AttachmentPoint':
         """

scionlab/tests/data/test_config_tar/host_4.yml

@@ -139,17 +139,6 @@ etc/scion/br-1.toml: |
   [metrics]
   prometheus = "127.0.0.1:30401"
 
-  [log.console]
-  level = "info"
-etc/scion/br-2.toml: |
-  [general]
-  config_dir = "/etc/scion"
-  id = "br-2"
-  reconnect_to_dispatcher = true
-
-  [metrics]
-  prometheus = "127.0.0.1:30402"
-
   [log.console]
   level = "info"
 etc/scion/certs/ISD17-B1-S1.trc: !!binary |
@@ -513,13 +502,7 @@ etc/scion/topology.json: |-
               "public": "192.0.2.17:50000",
               "remote": "192.0.2.12:50001"
             }
-          }
-        },
-        "internal_addr": "127.0.0.1:30001"
-      },
-      "br-2": {
-        "ctrl_addr": "127.0.0.1:30202",
-        "interfaces": {
+          },
           "2": {
             "bandwidth": 1000,
             "isd_as": "17-ffaa:1:1",
@@ -541,7 +524,7 @@ etc/scion/topology.json: |-
             }
           }
         },
-        "internal_addr": "127.0.0.1:30002"
+        "internal_addr": "127.0.0.1:30001"
       }
     },
     "control_service": {
@@ -564,7 +547,6 @@ scionlab-config.json: |-
       "etc/openvpn/server.conf": "4da81a8209247858c583fcda94c7b617381d64ab",
       "etc/scion/beacon_policy.yaml": "90c69ea879ca52546774c3007e7d29104766fc07",
       "etc/scion/br-1.toml": "93b34c926178d1ea7c422d7d9fa2d8d7dd770d3b",
-      "etc/scion/br-2.toml": "3e4c3c9c65f706f6792830ce68e3ae0d6b394a1f",
       "etc/scion/certs/ISD17-B1-S1.trc": "6e20b26b9831223ff8557b88cf7e2a3a441619b0",
       "etc/scion/certs/ISD19-B1-S1.trc": "b91dfb74d7f9d24d7cd81b53fdc2ffd820b1f46c",
       "etc/scion/certs/ISD20-B1-S1.trc": "66a6da3efd14c19c51c88dacd3de1534caa9b92c",
@@ -573,13 +555,12 @@ scionlab-config.json: |-
       "etc/scion/cs-1.toml": "84467cd10682d975caeccba3a7eaaec1bd9ea858",
       "etc/scion/keys/master0.key": "73184546f40ab4bc57870965e4a5896105a8ca2a",
       "etc/scion/keys/master1.key": "73184546f40ab4bc57870965e4a5896105a8ca2a",
-      "etc/scion/topology.json": "919e85f60aa61648a1d45d792fda85581c0537f4"
+      "etc/scion/topology.json": "6febb6e9910b8291d46bbc2767ae6f75ee820457"
     },
     "host_id": "ab0189ab83c940f88b0e16d51136e441",
     "host_secret": "8a6244bc807f481092cca4b70e8be75b",
     "systemd_units": [
       "[email protected]",
-      "[email protected]",
       "[email protected]",
       "scion-bwtestserver.service",
       "scion-daemon.service",

scionlab/tests/test_user_as_models.py

@@ -231,25 +231,8 @@ def _check_attachment_point(testcase, attachment_point: AttachmentPoint):
     """
     Check the assignment of interfaces to border routers in the attachment point.
     """
-
     host = attachment_point._get_host_for_useras_attachment()
-    border_routers = list(host.border_routers.all())
-
-    # The first BR is for the infrastructure links and also contains the inactive interfaces.
-    infra_br = border_routers.pop(0)
-    for iface in infra_br.interfaces.iterator():
-        testcase.assertTrue(iface.remote_as().owner is None or not iface.link().active)
-
-    # The other BRs contain up to 10 interfaces each.
-    MAX_IFACES = 10
-    for br in border_routers:
-        # Expecting only active interfaces in these BRs
-        testcase.assertTrue(all(interface.link().active for interface in br.interfaces.iterator()))
-        c = br.interfaces.count()
-        if br == border_routers[-1]:  # only last one can have less than max
-            testcase.assertLessEqual(c, MAX_IFACES)
-        else:
-            testcase.assertEqual(c, MAX_IFACES)
+    testcase.assertEqual(host.border_routers.count(), 1)
 
 
 def update_useras(testcase,

scripts/deactivate_dormant.py

@@ -30,7 +30,7 @@
 from django.db import transaction
 from django.db.models import Q
 
-from scionlab.models.user_as import UserAS, AttachmentPoint
+from scionlab.models.user_as import UserAS
 
 nologin_threshold = datetime.timedelta(days=365)
 
@@ -54,7 +54,3 @@ def deactivate_dormant(as_ids):
     for user_as in q:
         print(user_as.as_id)
         user_as.update_active(False)
-
-    # Fix-up distribution of user AS interfaces to router instances.
-    for ap in AttachmentPoint.objects.all():
-        ap.split_border_routers()

scripts/merge_ap_routers.py

@@ -0,0 +1,46 @@
+# Copyright 2021 ETH Zurich
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+:mod: scripts.merge_ap_routers
+=========================
+
+Run with python manage.py runscript merge_ap_routers
+
+For each attachment point, merge all border router instances such that only a single router is used.
+This is a cleanup step.
+Previously, we've automatically distributed the interfaces for attachment points over many router
+instances with at most ~12 interfaces each (`AttachmentPoint.split_border_routers`). This was a
+workaround for some issue in an old version of the router. Now, this is no longer necessary and
+running many instances is harmful for the system's performance.
+"""
+
+from django.db import transaction
+
+from scionlab.models.user_as import AttachmentPoint
+
+
+@transaction.atomic
+def run():
+    for ap in AttachmentPoint.objects.all():
+        merge_router_instances(ap.AS.hosts.get())
+
+
+def merge_router_instances(host):
+    chosen = host.border_routers.first()
+    for iface in host.interfaces.all():
+        iface.border_router = chosen
+        iface.save()
+    host.border_routers.exclude(pk=chosen.pk).delete()