[feature] Updating stale WHOIS records #1058

Added required checks for ensuring WHOIS records
older than a certain configurable threshold will
be updated along with the coordinates.

Closes  #1058

Signed-off-by: DragnEmperor <[email protected]>

Author: DragnEmperor

docs/user/estimated-location.rst

@@ -16,14 +16,14 @@ Estimated Location
 Overview
 --------
 
-The Estimated Location feature automatically creates or updates a device’s
-location based on latitude and longitude information retrieved from the
-WHOIS Lookup feature.
+This feature automatically creates or updates a device’s location based on
+latitude and longitude information retrieved from the WHOIS Lookup
+feature.
 
 Trigger Conditions
 ------------------
 
-Estimated Location is triggered when:
+This feature is triggered when:
 
 - A **fresh WHOIS lookup** is performed for a device.
 - Or when a WHOIS record already exists for the device’s IP **and**:
@@ -78,3 +78,10 @@ In REST API, the field will be visible in the :ref:`Device Location
 <location_geojson_estimated>` if the feature is **enabled**. The field can
 also be used for filtering in the location list (including geojson)
 endpoints and in the :ref:`Device List <device_list_estimated_filters>`.
+
+Managing Older Estimated Locations
+----------------------------------
+
+Whenever location related fields in WHOIS records are updated as per
+:ref:`Managing WHOIS Older Records <whois_older_records>`; the location
+will also be updated automatically.

docs/user/settings.rst

@@ -829,3 +829,16 @@ Allows enabling the optional :doc:`Estimated Location feature
 .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png
     :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png
     :alt: Estimated Location setting
+
+.. _openwisp_controller_whois_refresh_threshold_days:
+
+``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS``
+----------------------------------------------------
+
+============ =======
+**type**:    ``int``
+**default**: ``14``
+============ =======
+
+Specifies the number of days after which the WHOIS information for a
+device is considered stale and eligible for refresh.

docs/user/whois.rst

@@ -15,10 +15,10 @@ WHOIS Lookup
 Overview
 --------
 
-The WHOIS Lookup feature displays information about the public IP address
-used by devices to communicate with OpenWISP (via the ``last_ip`` field).
-It helps identify the geographic location and ISP associated with the IP
-address, which can be useful for troubleshooting network issues.
+This feature displays information about the public IP address used by
+devices to communicate with OpenWISP (via the ``last_ip`` field). It helps
+identify the geographic location and ISP associated with the IP address,
+which can be useful for troubleshooting network issues.
 
 The retrieved information pertains to the Autonomous System (ASN)
 associated with the device's public IP address and includes:
@@ -36,6 +36,7 @@ Trigger Conditions
 A WHOIS lookup is triggered automatically when:
 
 - A new device is registered.
+- An existing device's last IP address is changed.
 - A device fetches its checksum.
 
 However, the lookup will only run if **all** the following conditions are
@@ -114,3 +115,15 @@ retrieved details can be viewed in the following locations:
 - **Device REST API**: See WHOIS details in the :ref:`Device List
   <device_list_whois>` and :ref:`Device Detail <device_detail_whois>`
   responses.
+
+.. _whois_older_records:
+
+Managing Older WHOIS Records
+----------------------------
+
+If a record is older than :ref:`Threshold
+<openwisp_controller_whois_refresh_threshold_days>`, it will be refreshed
+automatically.
+
+The update mechanism will be triggered whenever a device is registered or
+its last IP changes or fetches its checksum.

openwisp_controller/config/base/device.py

@@ -519,4 +519,6 @@ def _check_last_ip(self, creating=False):
             return
         if creating or self.last_ip != self._initial_last_ip:
             self.whois_service.process_ip_data_and_location()
+        else:
+            self.whois_service.update_whois_info()
         self._initial_last_ip = self.last_ip

openwisp_controller/config/controller/views.py

@@ -153,6 +153,9 @@ def get(self, request, pk):
         # updates cache if ip addresses changed
         if updated:
             self.update_device_cache(device)
+        # check if WHOIS Info of device requires update
+        else:
+            device.whois_service.update_whois_info()
         checksum_requested.send(
             sender=device.__class__, instance=device, request=request
         )

openwisp_controller/config/management/commands/clear_last_ip.py

@@ -4,7 +4,10 @@
 
 
 class Command(BaseCommand):
-    help = "Clear the last IP address, if set, of active devices of all organizations."
+    help = (
+        "Clears the last IP address (if set) for every active device"
+        " across all organizations."
+    )
 
     def add_arguments(self, parser):
         parser.add_argument(
@@ -29,11 +32,11 @@ def handle(self, *args, **options):
                 "Are you sure you want to do this?\n\n"
                 "Type 'yes' to continue, or 'no' to cancel: "
             )
-            if input("".join(message)) != "yes":
+            if input("".join(message)).lower() != "yes":
                 raise CommandError("Operation cancelled by user.")
 
         devices = Device.objects.filter(_is_deactivated=False).only("last_ip")
-        # Filter devices that have no WHOIS information for their last IP
+        # Filter out devices that have WHOIS information for their last IP
         devices = devices.exclude(last_ip=None).exclude(
             last_ip__in=Subquery(
                 WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values(

openwisp_controller/config/migrations/0061_config_checksum_db.py

@@ -1,7 +1,6 @@
 # Generated by Django for issue #1113 optimization
 
 from django.db import migrations, models
-from swapper import load_model
 
 
 def populate_checksum_db(apps, schema_editor):
@@ -13,7 +12,7 @@ def populate_checksum_db(apps, schema_editor):
     hence we use Config.objects.bulk_update() instead of
     Config.update_status_if_checksum_changed().
     """
-    Config = load_model("config", "Config")
+    Config = apps.get_model("config", "Config")
     chunk_size = 100
     updated_configs = []
     qs = (

…ller/config/migrations/0061_whoisinfo.py …ller/config/migrations/0062_whoisinfo.pyopenwisp_controller/config/migrations/0061_whoisinfo.py renamed to openwisp_controller/config/migrations/0062_whoisinfo.py openwisp_controller/config/migrations/0061_whoisinfo.py renamed to openwisp_controller/config/migrations/0062_whoisinfo.py

@@ -10,7 +10,7 @@
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("config", "0060_cleanup_api_task_notification_types"),
+        ("config", "0061_config_checksum_db"),
     ]
 
     operations = [

…approximate_location_enabled_and_more.py …approximate_location_enabled_and_more.pyopenwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py renamed to openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py renamed to openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py

@@ -9,7 +9,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("config", "0061_whoisinfo"),
+        ("config", "0062_whoisinfo"),
     ]
 
     operations = [

openwisp_controller/config/settings.py

@@ -68,6 +68,7 @@ def get_setting(option, default):
     "API_TASK_RETRY_OPTIONS",
     dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True),
 )
+WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 14)
 WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None)
 WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None)
 WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False)