Merge pull request #174 from DrDroidLab/feature/new_metadata_extraction

feat: Delegate agent asset refresh to backend, update playbook task r…

Author: dimittal

connectors/apps.py

@@ -1,13 +1,10 @@
 import logging
-import uuid
 
 from django.apps import AppConfig
 from django.conf import settings
 
-from asset_manager.tasks import populate_connector_metadata
 from connectors.tasks import register_connectors
-from drdroid_debug_toolkit.core.protos.base_pb2 import Source
-from utils.credentilal_utils import credential_yaml_to_connector_proto, generate_credentials_dict
+from utils.credentilal_utils import credential_yaml_to_connector_proto
 from utils.static_mappings import integrations_connector_type_connector_keys_map
 
 logger = logging.getLogger(__name__)
@@ -25,7 +22,9 @@ def ready(self):
         drd_cloud_api_token = settings.DRD_CLOUD_API_TOKEN
         loaded_connections = settings.LOADED_CONNECTIONS if settings.LOADED_CONNECTIONS else {}
         if loaded_connections:
+            # Register connectors with DRD Cloud - asset refresh will be triggered by backend
             register_connectors(drd_cloud_host, drd_cloud_api_token, loaded_connections)
+            # Validate connector keys
             for c, metadata in loaded_connections.items():
                 connector_proto = credential_yaml_to_connector_proto(c, metadata)
                 connector_name = connector_proto.name.value
@@ -39,13 +38,4 @@ def ready(self):
                         break
                 if not all_keys_found:
                     raise ValueError(f'Missing required connector keys for {connector_name}')
-                connector_type: Source = connector_proto.type
-                credentials_dict = generate_credentials_dict(connector_type, connector_keys_proto)
-                if credentials_dict:
-                    request_id = uuid.uuid4().hex
-                    populate_connector_metadata.delay(request_id, connector_name, connector_type, credentials_dict)
-                elif settings.NATIVE_KUBERNETES_API_MODE:
-                    request_id = uuid.uuid4().hex
-                    populate_connector_metadata.delay(request_id, connector_name, connector_type, credentials_dict)
-                else:
-                    logger.warning(f'No credentials found for connector {connector_name}')
+            logger.info(f'Registered {len(loaded_connections)} connectors. Asset refresh will be triggered by backend.')

connectors/tasks.py

@@ -96,8 +96,14 @@ def register_connectors(drd_cloud_host, drd_cloud_api_token, loaded_connections)
         connector_proto = credential_yaml_to_connector_proto(c, metadata)
         connectors.append(proto_to_dict(Connector(name=connector_proto.name, type=connector_proto.type)))
 
-    response = requests.post(f'{drd_cloud_host}/connectors/proxy/register',
+    commit_hash = settings.VPC_AGENT_COMMIT_HASH
+    url = f'{drd_cloud_host}/connectors/proxy/register?commit_hash={commit_hash}'
+    logger.info(f'register_connectors:: Registering {len(connectors)} connectors with commit_hash: {commit_hash}')
+
+    response = requests.post(url,
                              headers={'Authorization': f'Bearer {drd_cloud_api_token}'},
                              json={'connectors': connectors})
     if response.status_code != 200:
         logger.error(f'Failed to register connectors with DRD Cloud: {response.json()}')
+    else:
+        logger.info(f'register_connectors:: Successfully registered connectors')

playbooks_engine/tasks.py

@@ -9,7 +9,7 @@
 
 from drdroid_debug_toolkit.core.integrations.source_facade import source_facade
 from drdroid_debug_toolkit.core.protos.base_pb2 import TimeRange, Source
-from drdroid_debug_toolkit.core.protos.playbooks.playbook_commons_pb2 import PlaybookTaskResult
+from drdroid_debug_toolkit.core.protos.playbooks.playbook_commons_pb2 import PlaybookTaskResult, TextResult
 from drdroid_debug_toolkit.core.protos.playbooks.playbook_pb2 import PlaybookTask
 from utils.proto_utils import dict_to_proto, proto_to_dict
 from utils.credentilal_utils import credential_yaml_to_connector_proto
@@ -70,10 +70,9 @@ def _execute_asset_refresh_task(playbook_task_execution_log):
             populate_connector_metadata(request_id, connector_name, connector_type, credentials_dict)
             message = f'Successfully executed full asset refresh for connector {connector_name}'
 
-        # Create success result
+        # Create success result using TextResult
         result = PlaybookTaskResult(
-            output=StringValue(value=message),
-            error=StringValue(value="")
+            text=TextResult(output=StringValue(value=message))
         )
 
     except Exception as e:

requirements.txt

@@ -149,5 +149,5 @@ wcwidth==0.2.13
 websocket-client==1.8.0
 yarl==1.12.1
 zstandard==0.23.0
-# Latest version of drdroid-debug-toolkit -> victoria logs & sentry test connection returns
+# Latest version of drdroid-debug-toolkit -> fixed asset refresh task in proxy agent class
 git+https://github.com/DrDroidLab/drdroid-debug-toolkit.git@master