[enhancement] Added updating older WHOIS records

DragnEmperor · DragnEmperor · commit 25da61a0c7c3 · 2025-08-30T00:51:55.000+05:30
Added threshold setting for checking older WHOIS records. Added test cases

Signed-off-by: DragnEmperor &lt;dragnemperor@gmail.com&gt;
diff --git a/docs/user/estimated-location.rst b/docs/user/estimated-location.rst
@@ -78,3 +78,10 @@ In REST API, the field will be visible in the :ref:`Device Location
 <location_geojson_estimated>` if the feature is **enabled**. The field can
 also be used for filtering in the location list (including geojson)
 endpoints and in the :ref:`Device List <device_list_estimated_filters>`.
+
+Managing Older Estimated Locations
+----------------------------------
+
+Whenever location related fields in WHOIS record are updated as per
+:ref:`Managing WHOIS Older Records <whois_older_records>`; the location
+will also be updated automatically.
diff --git a/docs/user/settings.rst b/docs/user/settings.rst
@@ -812,3 +812,16 @@ Allows enabling the optional :doc:`Estimated Location feature
 .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png
     :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png
     :alt: Estimated Location setting
+
+.. _openwisp_controller_whois_refresh_threshold_days:
+
+``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS``
+----------------------------------------------------
+
+============ =======
+**type**:    ``int``
+**default**: ``14``
+============ =======
+
+Specifies the number of days after which the WHOIS information for a
+device is considered stale and eligible for refresh.
diff --git a/docs/user/whois.rst b/docs/user/whois.rst
@@ -30,12 +30,14 @@ associated with the device's public IP address and includes:
 - Timezone of the ASN's registered location
 - Coordinates (Latitude and Longitude)
 
+.. _whois_trigger_conditions:
+
 Trigger Conditions
 ------------------
 
 A WHOIS lookup is triggered automatically when:
 
-- A new device is registered.
+- A new device is registered or its last IP address is changed.
 - A device fetches its checksum.
 
 However, the lookup will only run if **all** the following conditions are
@@ -114,3 +116,16 @@ retrieved details can be viewed in the following locations:
 - **Device REST API**: See WHOIS details in the :ref:`Device List
   <device_list_whois>` and :ref:`Device Detail <device_detail_whois>`
   responses.
+
+.. _whois_older_records:
+
+Managing Older WHOIS Records
+----------------------------
+
+If a record is older than :ref:`Threshold
+<openwisp_controller_whois_refresh_threshold_days>`, it will be refreshed
+automatically.
+
+This will be triggered for the same scenarios defined in `trigger
+conditions <whois_trigger_conditions_>`_ but among the conditions **only
+WHOIS enablement is required**.
diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py
@@ -519,4 +519,6 @@ def _check_last_ip(self, creating=False):
             return
         if creating or self.last_ip != self._initial_last_ip:
             self.whois_service.process_ip_data_and_location()
+        else:
+            self.whois_service.update_whois_info()
         self._initial_last_ip = self.last_ip
diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py
@@ -153,6 +153,9 @@ def get(self, request, pk):
         # updates cache if ip addresses changed
         if updated:
             self.update_device_cache(device)
+        # check if WHOIS Info of device requires update
+        else:
+            device.whois_service.update_whois_info()
         checksum_requested.send(
             sender=device.__class__, instance=device, request=request
         )
diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py
@@ -68,6 +68,7 @@ def get_setting(option, default):
     "API_TASK_RETRY_OPTIONS",
     dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True),
 )
+WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 14)
 WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None)
 WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None)
 WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False)
diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py
@@ -1,7 +1,9 @@
+from datetime import timedelta
 from ipaddress import ip_address as ip_addr
 
 from django.core.cache import cache
 from django.db import transaction
+from django.utils import timezone
 from swapper import load_model
 
 from openwisp_controller.config import settings as app_settings
@@ -43,6 +45,15 @@ def _get_whois_info_from_db(ip_address):
 
         return WHOISInfo.objects.filter(ip_address=ip_address)
 
+    @staticmethod
+    def is_older(datetime):
+        """
+        Check if given datetime is older than the refresh threshold.
+        """
+        return (timezone.now() - datetime) >= timedelta(
+            days=app_settings.WHOIS_REFRESH_THRESHOLD_DAYS
+        )
+
     @staticmethod
     def get_org_config_settings(org_id):
         """
@@ -90,6 +101,8 @@ def is_whois_enabled(self):
         """
         Check if the WHOIS lookup feature is enabled.
         """
+        if not app_settings.WHOIS_CONFIGURED:
+            return False
         org_settings = self.get_org_config_settings(org_id=self.device.organization.pk)
         return org_settings.whois_enabled
 
@@ -98,6 +111,8 @@ def is_estimated_location_enabled(self):
         """
         Check if the Estimated location feature is enabled.
         """
+        if not app_settings.WHOIS_CONFIGURED:
+            return False
         org_settings = self.get_org_config_settings(org_id=self.device.organization.pk)
         return org_settings.estimated_location_enabled
 
@@ -108,15 +123,17 @@ def _need_whois_lookup(self, new_ip):
 
         The lookup is not triggered if:
             - The new IP address is None or it is a private IP address.
-            - The WHOIS information of new ip is already present.
+            - The WHOIS information of new ip is present and is not older than
+              14 days.
             - WHOIS is disabled in the organization settings. (query from db)
         """
 
         # Check cheap conditions first before hitting the database
         if not self.is_valid_public_ip_address(new_ip):
             return False
 
-        if self._get_whois_info_from_db(new_ip).exists():
+        whois_obj = self._get_whois_info_from_db(ip_address=new_ip).first()
+        if whois_obj and not self.is_older(whois_obj.modified):
             return False
 
         return self.is_whois_enabled
@@ -145,20 +162,19 @@ def get_device_whois_info(self):
 
         return self._get_whois_info_from_db(ip_address=ip_address).first()
 
-    def process_ip_data_and_location(self):
+    def process_ip_data_and_location(self, force_lookup=False):
         """
         Trigger WHOIS lookup based on the conditions of `_need_whois_lookup`
         and also manage estimated locations based on the conditions of
         `_need_estimated_location_management`.
         Tasks are triggered on commit to ensure redundant data is not created.
         """
         new_ip = self.device.last_ip
-        if self._need_whois_lookup(new_ip):
+        if force_lookup or self._need_whois_lookup(new_ip):
             transaction.on_commit(
                 lambda: fetch_whois_details.delay(
                     device_pk=self.device.pk,
                     initial_ip_address=self.device._initial_last_ip,
-                    new_ip_address=new_ip,
                 )
             )
         # To handle the case when WHOIS already exists as in that case
@@ -170,3 +186,17 @@ def process_ip_data_and_location(self):
                     device_pk=self.device.pk, ip_address=new_ip
                 )
             )
+
+    def update_whois_info(self):
+        """
+        Update the WHOIS information for the device.
+        """
+        ip_address = self.device.last_ip
+        if not self.is_valid_public_ip_address(ip_address):
+            return
+
+        if not self.is_whois_enabled:
+            return
+        whois_obj = WHOISService._get_whois_info_from_db(ip_address=ip_address).first()
+        if whois_obj and self.is_older(whois_obj.modified):
+            fetch_whois_details.delay(device_pk=self.device.pk, initial_ip_address=None)
diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py
@@ -1,8 +1,11 @@
 import logging
+from datetime import timedelta
 
 import requests
 from celery import shared_task
 from django.contrib.gis.geos import Point
+from django.db import transaction
+from django.utils import timezone
 from django.utils.translation import gettext as _
 from geoip2 import errors
 from geoip2 import webservice as geoip2_webservice
@@ -59,84 +62,104 @@ def on_failure(self, exc, task_id, args, kwargs, einfo):
     base=WHOISCeleryRetryTask,
     **app_settings.API_TASK_RETRY_OPTIONS,
 )
-def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address):
+def fetch_whois_details(self, device_pk, initial_ip_address):
     """
     Fetches the WHOIS details of the given IP address
     and creates/updates the WHOIS record.
     """
     Device = load_model("config", "Device")
     WHOISInfo = load_model("config", "WHOISInfo")
 
-    # The task can be triggered for same ip address multiple times
-    # so we need to return early if WHOIS is already created.
-    if WHOISInfo.objects.filter(ip_address=new_ip_address).exists():
-        return
-
-    device = Device.objects.get(pk=device_pk)
-    # Host is based on the db that is used to fetch the details.
-    # As we are using GeoLite2, 'geolite.info' host is used.
-    # Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example
-    ip_client = geoip2_webservice.Client(
-        account_id=app_settings.WHOIS_GEOIP_ACCOUNT,
-        license_key=app_settings.WHOIS_GEOIP_KEY,
-        host="geolite.info",
-    )
-
-    try:
-        data = ip_client.city(ip_address=new_ip_address)
-
-    # Catching all possible exceptions raised by the geoip2 client
-    # and raising them with appropriate messages to be handled by the task
-    # retry mechanism.
-    except (
-        errors.AddressNotFoundError,
-        errors.AuthenticationError,
-        errors.OutOfQueriesError,
-        errors.PermissionRequiredError,
-    ) as e:
-        exc_type = type(e)
-        message = EXCEPTION_MESSAGES.get(exc_type)
-        if exc_type is errors.AddressNotFoundError:
-            message = message.format(ip_address=new_ip_address)
-        raise exc_type(message)
-    except requests.RequestException as e:
-        raise e
-
-    else:
-        # The attributes are always present in the response,
-        # but they can be None, so added fallbacks.
-        address = {
-            "city": data.city.name or "",
-            "country": data.country.name or "",
-            "continent": data.continent.name or "",
-            "postal": str(data.postal.code or ""),
-        }
-        coordinates = Point(data.location.longitude, data.location.latitude, srid=4326)
-        whois_obj = WHOISInfo(
-            isp=data.traits.autonomous_system_organization,
-            asn=data.traits.autonomous_system_number,
-            timezone=data.location.time_zone,
-            address=address,
-            cidr=data.traits.network,
-            ip_address=new_ip_address,
-            coordinates=coordinates,
+    with transaction.atomic():
+        device = Device.objects.get(pk=device_pk)
+        new_ip_address = device.last_ip
+        # If there is existing WHOIS older record then it needs to be updated
+        whois_obj = WHOISInfo.objects.filter(ip_address=new_ip_address).first()
+        if whois_obj and (timezone.now() - whois_obj.modified) < timedelta(days=14):
+            return
+
+        # Host is based on the db that is used to fetch the details.
+        # As we are using GeoLite2, 'geolite.info' host is used.
+        # Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example
+        ip_client = geoip2_webservice.Client(
+            account_id=app_settings.WHOIS_GEOIP_ACCOUNT,
+            license_key=app_settings.WHOIS_GEOIP_KEY,
+            host="geolite.info",
         )
-        whois_obj.full_clean()
-        whois_obj.save()
-        logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.")
 
-        if device._get_organization__config_settings().estimated_location_enabled:
-            manage_estimated_locations.delay(
-                device_pk=device_pk, ip_address=new_ip_address
+        try:
+            data = ip_client.city(ip_address=new_ip_address)
+
+        # Catching all possible exceptions raised by the geoip2 client
+        # and raising them with appropriate messages to be handled by the task
+        # retry mechanism.
+        except (
+            errors.AddressNotFoundError,
+            errors.AuthenticationError,
+            errors.OutOfQueriesError,
+            errors.PermissionRequiredError,
+        ) as e:
+            exc_type = type(e)
+            message = EXCEPTION_MESSAGES.get(exc_type)
+            if exc_type is errors.AddressNotFoundError:
+                message = message.format(ip_address=new_ip_address)
+            raise exc_type(message)
+        except requests.RequestException as e:
+            raise e
+
+        else:
+            # The attributes are always present in the response,
+            # but they can be None, so added fallbacks.
+            address = {
+                "city": data.city.name or "",
+                "country": data.country.name or "",
+                "continent": data.continent.name or "",
+                "postal": str(data.postal.code or ""),
+            }
+            coordinates = Point(
+                data.location.longitude, data.location.latitude, srid=4326
             )
-
-        # delete WHOIS record for initial IP if no devices are linked to it
-        if (
-            not Device.objects.filter(_is_deactivated=False)
-            .filter(last_ip=initial_ip_address)
-            .exists()
-        ):
-            delete_whois_record(ip_address=initial_ip_address)
+            details = {
+                "isp": data.traits.autonomous_system_organization,
+                "asn": data.traits.autonomous_system_number,
+                "timezone": data.location.time_zone,
+                "address": address,
+                "coordinates": coordinates,
+                "cidr": data.traits.network,
+                "ip_address": new_ip_address,
+            }
+            update_fields = []
+            if whois_obj:
+                for attr, value in details.items():
+                    if getattr(whois_obj, attr) != value:
+                        update_fields.append(attr)
+                        setattr(whois_obj, attr, value)
+                if update_fields:
+                    whois_obj.save(update_fields=update_fields)
+            else:
+                whois_obj = WHOISInfo(**details)
+                whois_obj.full_clean()
+                whois_obj.save()
+            logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.")
+
+            if device._get_organization__config_settings().estimated_location_enabled:
+                # the estimated location task should not run if old record is updated
+                # and location related fields are not updated
+                if update_fields and not any(
+                    i in update_fields for i in ["address", "coordinates"]
+                ):
+                    return
+                manage_estimated_locations.delay(
+                    device_pk=device_pk, ip_address=new_ip_address
+                )
+
+            # delete WHOIS record for initial IP if no devices are linked to it
+            if (
+                not Device.objects.filter(_is_deactivated=False)
+                .filter(last_ip=initial_ip_address)
+                .exists()
+            ):
+                delete_whois_record(ip_address=initial_ip_address)
 
 
 @shared_task
diff --git a/openwisp_controller/config/whois/test_whois.py b/openwisp_controller/config/whois/test_whois.py
diff --git a/openwisp_controller/config/whois/tests_utils.py b/openwisp_controller/config/whois/tests_utils.py
diff --git a/openwisp_controller/geo/estimated_location/test_estimated_location.py b/openwisp_controller/geo/estimated_location/test_estimated_location.py

Original file line number	Diff line number	Diff line change
`@@ -68,6 +68,7 @@ def get_setting(option, default):`
`68`	`68`	`"API_TASK_RETRY_OPTIONS",`
`69`	`69`	`dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True),`
`70`	`70`	`)`
	`71`	`+WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 14)`
`71`	`72`	`WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None)`
`72`	`73`	`WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None)`
`73`	`74`	`WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False)`