[feature] Cache VPN server checksum #1064

Closes #1064

Author: pandafy

openwisp_controller/config/admin.py

@@ -312,7 +312,8 @@ def download_view(self, request, pk):
             config = instance.config
         else:
             raise Http404()
-        config_archive = config.generate()
+        get_configuration = getattr(config, "get_cached_configuration", config.generate)
+        config_archive = get_configuration()
         return send_file(
             filename="{0}.tar.gz".format(config.name),
             contents=config_archive.getvalue(),

openwisp_controller/config/apps.py

@@ -264,7 +264,7 @@ def enable_cache_invalidation(self):
         Triggers the cache invalidation for the
         device config checksum (view and model method)
         """
-        from .controller.views import DeviceChecksumView
+        from .controller.views import DeviceChecksumView, GetVpnView
         from .handlers import (
             device_cache_invalidation_handler,
             devicegroup_change_handler,
@@ -293,6 +293,21 @@ def enable_cache_invalidation(self):
             DeviceChecksumView.invalidate_checksum_cache,
             dispatch_uid="invalidate_checksum_cache",
         )
+        # VPN cache invalidation
+        post_save.connect(
+            GetVpnView.invalidate_get_vpn_cache,
+            sender=self.vpn_model,
+            dispatch_uid="invalidate_get_vpn_cache",
+        )
+        pre_delete.connect(
+            GetVpnView.invalidate_get_vpn_cache,
+            sender=self.vpn_model,
+            dispatch_uid="vpn_server_pre_delete_invalidate_get_vpn_cache",
+        )
+        vpn_server_modified.connect(
+            GetVpnView.invalidate_get_vpn_cache,
+            dispatch_uid="vpn_server_modified_invalidate_get_vpn_cache",
+        )
         device_group_changed.connect(
             devicegroup_change_handler,
             sender=self.device_model,

openwisp_controller/config/base/base.py

@@ -1,8 +1,10 @@
 import collections
 import hashlib
 import json
+import logging
 from copy import deepcopy
 
+from cache_memoize import cache_memoize
 from django.core.exceptions import ValidationError
 from django.db import models
 from django.utils.functional import cached_property
@@ -15,6 +17,82 @@
 
 from .. import settings as app_settings
 
+logger = logging.getLogger(__name__)
+
+
+def get_cached_args_rewrite(instance):
+    """
+    Use only the PK parameter for calculating the cache key
+    """
+    return instance.pk.hex
+
+
+class ChecksumCacheMixin:
+    """
+    Mixin that provides caching for checksum.
+    """
+
+    _CHECKSUM_CACHE_TIMEOUT = 60 * 60 * 24 * 30  # 30 days
+
+    @cache_memoize(
+        timeout=_CHECKSUM_CACHE_TIMEOUT, args_rewrite=get_cached_args_rewrite
+    )
+    def get_cached_checksum(self):
+        """
+        Handles caching,
+        timeout=None means value is cached indefinitely
+        (invalidation handled on post_save/post_delete signal)
+        """
+        logger.debug(f"calculating checksum for {self.__class__.__name__} ID {self.pk}")
+        return self.checksum
+
+    @classmethod
+    def bulk_invalidate_get_cached_checksum(cls, query_params):
+        """
+        Bulk invalidate checksum cache for multiple instances
+        """
+        for instance in cls.objects.only("id").filter(**query_params).iterator():
+            instance.get_cached_checksum.invalidate(instance)
+
+    def invalidate_checksum_cache(self):
+        """
+        Invalidate the checksum cache for this instance
+        """
+        self.get_cached_checksum.invalidate(self)
+        logger.debug(
+            f"invalidated checksum cache for {self.__class__.__name__} ID {self.pk}"
+        )
+
+
+class ConfigChecksumCacheMixin(ChecksumCacheMixin):
+    """
+    Mixin that provides caching for both checksum and configuration.
+    """
+
+    @cache_memoize(
+        timeout=ChecksumCacheMixin._CHECKSUM_CACHE_TIMEOUT,
+        args_rewrite=get_cached_args_rewrite,
+    )
+    def get_cached_configuration(self):
+        """
+        Returns cached configuration
+        """
+        return self.generate()
+
+    def invalidate_configuration_cache(self):
+        """
+        Invalidate the configuration cache for this instance
+        """
+        self.get_cached_configuration.invalidate(self)
+        logger.debug(
+            f"invalidated configuration cache for {self.__class__.__name__}"
+            f" ID {self.pk}"
+        )
+
+    def invalidate_checksum_cache(self):
+        super().invalidate_checksum_cache()
+        self.invalidate_configuration_cache()
+
 
 class BaseModel(TimeStampedEditableModel):
     """

openwisp_controller/config/base/config.py

@@ -3,7 +3,6 @@
 import re
 from collections import defaultdict
 
-from cache_memoize import cache_memoize
 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied, ValidationError
 from django.db import models, transaction
 from django.utils.translation import gettext_lazy as _
@@ -24,11 +23,20 @@
 )
 from ..sortedm2m.fields import SortedManyToManyField
 from ..utils import get_default_templates_queryset
-from .base import BaseConfig
+from .base import BaseConfig, ChecksumCacheMixin, get_cached_args_rewrite
 
 logger = logging.getLogger(__name__)
 
 
+def get_cached_checksum_args_rewrite(config):
+    """
+    DEPRECATED: Use get_cached_args_rewrite instead.
+
+    TODO: Remove this in 1.2.0 release.
+    """
+    return get_cached_args_rewrite(config)
+
+
 class TemplatesThrough(object):
     """
     Improves string representation of m2m relationship objects
@@ -38,14 +46,7 @@ def __str__(self):
         return _("Relationship with {0}").format(self.template.name)
 
 
-def get_cached_checksum_args_rewrite(config):
-    """
-    Use only the PK parameter for calculating the cache key
-    """
-    return config.pk.hex
-
-
-class AbstractConfig(BaseConfig):
+class AbstractConfig(ChecksumCacheMixin, BaseConfig):
     """
     Abstract model implementing the
     NetJSON DeviceConfiguration object
@@ -150,23 +151,6 @@ def key(self):
         """
         return self.device.key
 
-    @cache_memoize(
-        timeout=_CHECKSUM_CACHE_TIMEOUT, args_rewrite=get_cached_checksum_args_rewrite
-    )
-    def get_cached_checksum(self):
-        """
-        Handles caching,
-        timeout=None means value is cached indefinitely
-        (invalidation handled on post_save/post_delete signal)
-        """
-        logger.debug(f"calculating checksum for config ID {self.pk}")
-        return self.checksum
-
-    @classmethod
-    def bulk_invalidate_get_cached_checksum(cls, query_params):
-        for config in cls.objects.only("id").filter(**query_params).iterator():
-            config.get_cached_checksum.invalidate(config)
-
     @classmethod
     def get_template_model(cls):
         return cls.templates.rel.model

openwisp_controller/config/base/vpn.py

@@ -33,7 +33,7 @@
     trigger_zerotier_server_update,
     trigger_zerotier_server_update_member,
 )
-from .base import BaseConfig
+from .base import BaseConfig, ConfigChecksumCacheMixin
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +43,7 @@ def _peer_cache_key(vpn):
     return str(vpn.pk)
 
 
-class AbstractVpn(ShareableOrgMixinUniqueName, BaseConfig):
+class AbstractVpn(ConfigChecksumCacheMixin, ShareableOrgMixinUniqueName, BaseConfig):
     """
     Abstract VPN model
     """
@@ -281,6 +281,7 @@ def save(self, *args, **kwargs):
         if create_dh:
             transaction.on_commit(lambda: create_vpn_dh.delay(self.id))
         if not created and self._send_vpn_modified_after_save:
+            self.invalidate_checksum_cache()
             self._send_vpn_modified_signal()
             self._send_vpn_modified_after_save = False
         # For ZeroTier VPN server, if the
@@ -308,10 +309,9 @@ def _check_changes(self):
         ]
         current = self._meta.model.objects.only(*attrs).get(pk=self.pk)
         for attr in attrs:
-            if getattr(self, attr) == getattr(current, attr):
-                continue
-            self._send_vpn_modified_after_save = True
-            break
+            if getattr(self, attr) != getattr(current, attr):
+                self._send_vpn_modified_after_save = True
+                break
 
     def _send_vpn_modified_signal(self):
         vpn_server_modified.send(sender=self.__class__, instance=self)
@@ -687,8 +687,10 @@ def _invalidate_peer_cache(self, update=False):
         for backend in ["wireguard", "vxlan"]:
             if self._is_backend_type(backend):
                 getattr(self, f"_get_{backend}_peers").invalidate(self)
+                self.invalidate_checksum_cache()
                 if update:
                     getattr(self, f"_get_{backend}_peers")()
+                    self.get_cached_configuration()
                 # Send signal for peers changed
                 vpn_peers_changed.send(sender=self.__class__, instance=self)
 

openwisp_controller/config/controller/views.py

@@ -127,6 +127,18 @@ def get_device_args_rewrite(view):
     return pk.hex
 
 
+def get_vpn_args_rewrite(view):
+    """
+    Use only the PK parameter for calculating the cache key for VPN
+    """
+    pk = view.kwargs["pk"]
+    try:
+        pk = uuid.UUID(pk)
+    except ValueError:
+        return pk
+    return pk.hex
+
+
 class DeviceChecksumView(UpdateLastIpMixin, GetDeviceView):
     """
     returns device's configuration checksum
@@ -463,24 +475,43 @@ class GetVpnView(SingleObjectMixin, View):
     model = Vpn
 
     def get_object(self, *args, **kwargs):
-        queryset = self.model.objects.select_related("organization").filter(
-            Q(organization__is_active=True) | Q(organization__isnull=True)
-        )
+        queryset = self.model.objects.select_related(
+            "organization", "ca", "cert", "subnet", "ip"
+        ).filter(Q(organization__is_active=True) | Q(organization__isnull=True))
         return get_object_or_404(queryset, *args, **kwargs)
 
+    @cache_memoize(
+        timeout=Vpn._CHECKSUM_CACHE_TIMEOUT, args_rewrite=get_vpn_args_rewrite
+    )
+    def get_vpn(self):
+        pk = self.kwargs["pk"]
+        logger.debug(f"retrieving VPN ID {pk} from DB")
+        return self.get_object(pk=pk)
+
+    @classmethod
+    def invalidate_get_vpn_cache(cls, instance, **kwargs):
+        """
+        Called from signal receiver which performs cache invalidation
+        """
+        view = cls()
+        pk = str(instance.pk.hex)
+        view.kwargs = {"pk": pk}
+        view.get_vpn.invalidate(view)
+        logger.debug(f"invalidated view cache for VPN ID {pk}")
+
 
 class VpnChecksumView(GetVpnView):
     """
     returns vpn's configuration checksum
     """
 
     def get(self, request, *args, **kwargs):
-        vpn = self.get_object(*args, **kwargs)
+        vpn = self.get_vpn()
         bad_request = forbid_unallowed(request, "GET", "key", vpn.key)
         if bad_request:
             return bad_request
         checksum_requested.send(sender=vpn.__class__, instance=vpn, request=request)
-        return ControllerResponse(vpn.checksum, content_type="text/plain")
+        return ControllerResponse(vpn.get_cached_checksum(), content_type="text/plain")
 
 
 class VpnDownloadConfigView(GetVpnView):
@@ -489,7 +520,7 @@ class VpnDownloadConfigView(GetVpnView):
     """
 
     def get(self, request, *args, **kwargs):
-        vpn = self.get_object(*args, **kwargs)
+        vpn = self.get_vpn()
         bad_request = forbid_unallowed(request, "GET", "key", vpn.key)
         if bad_request:
             return bad_request

openwisp_controller/config/handlers.py

@@ -140,7 +140,7 @@ def devicegroup_templates_change_handler(instance, **kwargs):
 
 def organization_disabled_handler(instance, **kwargs):
     """
-    Asynchronously invalidates DeviceCheckView.get_device cache
+    Asynchronously invalidates device and VPN controller views cache
     """
     if instance.is_active:
         return
@@ -151,4 +151,4 @@ def organization_disabled_handler(instance, **kwargs):
     if instance.is_active == db_instance.is_active:
         # No change in is_active
         return
-    tasks.invalidate_device_checksum_view_cache.delay(str(instance.id))
+    tasks.invalidate_controller_views_cache.delay(str(instance.id))

openwisp_controller/config/tasks.py

@@ -99,6 +99,15 @@ def invalidate_devicegroup_cache_delete(instance_id, model_name, **kwargs):
 
 @shared_task(base=OpenwispCeleryTask)
 def trigger_vpn_server_endpoint(endpoint, auth_token, vpn_id):
+    Vpn = load_model("config", "Vpn")
+    try:
+        vpn = Vpn.objects.get(pk=vpn_id)
+    except Vpn.DoesNotExist:
+        logger.error(f"VPN Server UUID: {vpn_id} does not exist.")
+        return
+
+    # Cache the configuration here makes downloading the configuration faster.
+    vpn.get_cached_configuration()
     response = requests.post(
         endpoint,
         params={"key": auth_token},
@@ -149,11 +158,31 @@ def bulk_invalidate_config_get_cached_checksum(query_params):
 
 
 @shared_task(base=OpenwispCeleryTask)
-def invalidate_device_checksum_view_cache(organization_id):
-    from .controller.views import DeviceChecksumView
+def invalidate_controller_views_cache(organization_id):
+    """
+    Invalidates the cache of DeviceChecksumView, GetVpnView
+    """
+    from .controller.views import DeviceChecksumView, GetVpnView
 
     Device = load_model("config", "Device")
+    Vpn = load_model("config", "Vpn")
+
     for device in (
         Device.objects.filter(organization_id=organization_id).only("id").iterator()
     ):
         DeviceChecksumView.invalidate_get_device_cache(device)
+
+    for vpn in (
+        Vpn.objects.filter(organization_id=organization_id).only("id").iterator()
+    ):
+        GetVpnView.invalidate_get_vpn_cache(vpn)
+
+
+@shared_task(base=OpenwispCeleryTask)
+def invalidate_device_checksum_view_cache(organization_id):
+    """
+    DEPRECATED: Use invalidate_controller_views_cache instead.
+
+    TODO: Remove this in 1.2.0 release.
+    """
+    return invalidate_controller_views_cache(organization_id)