Netbox: handle different host url in 'next' section of the response (#850)

* netbox: handle next page url with different host

The next url might contain a different url if netbox is
behind a reverse proxy. This commit sets the protocol, the host and the
port of the next page request to the same parameters of the
previous request which was successful

Signed-off-by: LucaNicosia <[email protected]>

* controller: add debug log

Signed-off-by: LucaNicosia <[email protected]>

* controller test fixes

Signed-off-by: LucaNicosia <[email protected]>

---------

Signed-off-by: LucaNicosia <[email protected]>

Author: LucaNicosia

suzieq/poller/controller/controller.py

@@ -318,6 +318,7 @@ async def _inventory_sync(self):
                         'too much time'
                     )
 
+                logger.debug(f'Received inventory from {inv_src.name}')
                 if cur_inv:
                     cur_inv_count = len(cur_inv)
                     duplicated_devices = [x for x in cur_inv

suzieq/poller/controller/source/netbox.py

@@ -242,7 +242,60 @@ async def _get_devices(self, url: str) -> Tuple[List, str]:
         async with self._session.get(url) as response:
             if int(response.status) == 200:
                 res = await response.json()
-                return res.get('results', []), res.get('next')
+                next_url = res.get('next')
+                if next_url:
+                    # The next url might contain a different url if netbox is
+                    # behind a reverse proxy.
+                    # The code below sets the protocol, the host and the port
+                    # of the next request to the same parameters of the
+                    # previous request which was successful
+
+                    logger.debug(f'Parsing next page url {next_url}')
+
+                    # parse urls
+                    url_data = urlparse(url)
+                    next_url_data = urlparse(next_url)
+
+                    # retrieve protocol, host and port from the urls
+                    host = url_data.hostname
+                    protocol = url_data.scheme or 'http'
+                    port = url_data.port or _DEFAULT_PORTS.get(protocol)
+
+                    next_host = next_url_data.hostname
+                    next_protocol = next_url_data.scheme or 'http'
+                    next_port = (next_url_data.port or
+                                 _DEFAULT_PORTS.get(next_protocol))
+
+                    # verify if the two elements are different. If so, log
+                    # what's different and set the value to content of the
+                    # previous url
+                    if host != next_host:
+                        logger.debug(
+                            'Detected a different host in response: original '
+                            f'host "{host}", received host "{next_host}". '
+                            f'Setting the request host to "{host}"'
+                        )
+                        next_host = host
+                    if protocol != next_protocol:
+                        logger.debug(
+                            'Detected a different protocol in response: '
+                            f'original protocol "{protocol}", received '
+                            f'protocol "{next_protocol}". Setting the request '
+                            f'protocol to "{protocol}"'
+                        )
+                        next_protocol = protocol
+                    if port != next_port:
+                        logger.debug(
+                            'Detected a different port in response: original '
+                            f'port "{port}", received port "{next_port}". '
+                            f'Setting the request port to "{port}"'
+                        )
+                        next_port = port
+
+                    # build the next_url
+                    next_url = (f'{next_protocol}://{next_host}:{next_port}'
+                                f'{next_url_data.path}?{next_url_data.query}')
+                return res.get('results', []), next_url
             else:
                 raise InventorySourceError(
                     f'{self.name}: error in inventory get '

tests/unit/poller/controller/test_controller.py

@@ -256,6 +256,9 @@ def mock_src_init(self, inv, validate=False):
         self.set_device()
         self._load(inv)
 
+    def mock_get_name(self):
+        return self._name
+
     def mock_set_inventory(self, inv):
         self._inventory = inv.copy()
         if not self._inv_is_set:
@@ -284,12 +287,14 @@ def mock_chunk(self, inv, n):
     native_mock = patch.multiple(SqNativeFile, __init__=mock_src_init,
                                  set_inventory=mock_set_inventory,
                                  get_inventory=mock_get_inventory,
+                                 name=mock_get_name,
                                  _load=mock_native_load,
                                  set_device=MagicMock())
 
     netbox_mock = patch.multiple(Netbox, __init__=mock_src_init,
                                  set_inventory=mock_set_inventory,
                                  get_inventory=mock_get_inventory,
+                                 name=mock_get_name,
                                  _load=MagicMock(), run=mock_netbox_run,
                                  set_device=MagicMock())