Merge pull request #503 from DrDroidLab/feature/argo-and-jira-integration

Feature/argo and jira integration

Author: droid-mohit

connectors/assets/extractor/argocd_metadata_extractor.py

@@ -0,0 +1,40 @@
+import logging
+
+from executor.source_processors.argocd_api_processor import ArgoCDAPIProcessor
+from connectors.assets.extractor.metadata_extractor import SourceMetadataExtractor
+from protos.base_pb2 import Source, SourceModelType
+from utils.logging_utils import log_function_call
+
+logger = logging.getLogger(__name__)
+
+
+class ArgoCDSourceMetadataExtractor(SourceMetadataExtractor):
+
+    def __init__(self, argocd_server, argocd_token, account_id=None, connector_id=None):
+        self.argocd_processor = ArgoCDAPIProcessor(argocd_server, argocd_token)
+        super().__init__(account_id, connector_id, Source.ARGOCD)
+
+    @log_function_call
+    def extract_applications(self, save_to_db=False):
+        model_data = {}
+        model_type = SourceModelType.ARGOCD_APPS
+        try:
+            applications = self.argocd_processor.get_deployment_info()
+            if not applications:
+                return model_data
+
+            for application in applications.get('items', []):  # Iterate over application items
+                try:
+                    app_name = application.get("metadata", {}).get("name", "")  # Access the name safely
+                    path = application.get("spec", {}).get("source", {}).get("path", "")  # Access the path safely
+                    if app_name:
+                        model_data[app_name] = {"name": app_name, "path": path}
+                        if save_to_db:
+                            self.create_or_update_model_metadata(model_type, app_name,
+                                                                 model_data[app_name])  # Move inside loop
+                except KeyError as e:
+                    logger.error(f"Missing key {e} in application: {application}")  # Handle missing keys
+        except Exception as e:
+            logger.error(f'Error extracting ArgoCD applications: {e}')
+
+        return model_data

connectors/assets/extractor/jira_metadata_extractor.py

@@ -0,0 +1,48 @@
+import logging
+
+from executor.source_processors.jira_api_processor import JiraApiProcessor
+from connectors.assets.extractor.metadata_extractor import SourceMetadataExtractor
+from protos.base_pb2 import Source, SourceModelType
+from utils.logging_utils import log_function_call
+
+logger = logging.getLogger(__name__)
+
+
+class JiraSourceMetadataExtractor(SourceMetadataExtractor):
+
+    def __init__(self, jira_cloud_api_key, jira_domain, jira_email, account_id=None, connector_id=None):
+        self.jira_processor = JiraApiProcessor(jira_cloud_api_key, jira_domain, jira_email)
+        super().__init__(account_id, connector_id, Source.JIRA_CLOUD)
+
+    @log_function_call
+    def extract_projects(self, save_to_db=False):
+        model_data = {}
+        model_type = SourceModelType.JIRA_PROJECT
+        try:
+            projects = self.jira_processor.list_all_projects()
+            if not projects:
+                return model_data
+            for project in projects:
+                model_data[project['key']] = project
+                if save_to_db:
+                    self.create_or_update_model_metadata(model_type, project['key'], project)
+        except Exception as e:
+            logger.error(f'Error extracting Jira projects: {e}')
+        return model_data
+
+    @log_function_call
+    def extract_users(self, save_to_db=False):
+        model_data = {}
+        model_type = SourceModelType.JIRA_USER
+        try:
+            users = self.jira_processor.list_all_users()
+            if not users:
+                return model_data
+            for user in users:
+                if 'accountType' in user and user['accountType'] == 'atlassian':
+                    model_data[user['accountId']] = user
+                    if save_to_db:
+                        self.create_or_update_model_metadata(model_type, user['accountId'], user)
+        except Exception as e:
+            logger.error(f'Error extracting Jira users: {e}')
+        return model_data

connectors/assets/extractor/metadata_extractor_facade.py

@@ -10,6 +10,8 @@
 from connectors.assets.extractor.metadata_extractor import SourceMetadataExtractor
 from connectors.assets.extractor.mimir_metadata_extractor import MimirSourceMetadataExtractor
 from connectors.assets.extractor.newrelic_metadata_extractor import NewrelicSourceMetadataExtractor
+from connectors.assets.extractor.jira_metadata_extractor import JiraSourceMetadataExtractor
+from connectors.assets.extractor.argocd_metadata_extractor import ArgoCDSourceMetadataExtractor
 from protos.base_pb2 import Source
 
 
@@ -40,3 +42,5 @@ def get_connector_metadata_extractor_class(self, connector_type: Source):
 source_metadata_extractor_facade.register(Source.AZURE, AzureConnectorMetadataExtractor)
 source_metadata_extractor_facade.register(Source.GKE, GkeSourceMetadataExtractor)
 source_metadata_extractor_facade.register(Source.ELASTIC_SEARCH, ElasticSearchSourceMetadataExtractor)
+source_metadata_extractor_facade.register(Source.JIRA_CLOUD, JiraSourceMetadataExtractor)
+source_metadata_extractor_facade.register(Source.ARGOCD, ArgoCDSourceMetadataExtractor)

connectors/assets/manager/argocd_asset_manager.py

@@ -0,0 +1,64 @@
+from datetime import datetime, timezone
+
+from google.protobuf.wrappers_pb2 import UInt64Value, StringValue
+
+from connectors.assets.manager.asset_manager import ConnectorAssetManager as SourceAssetManager
+from protos.connectors.assets.argocd_asset_pb2 import ArgoCDAppsAssetOptions, ArgoCDAppsAssetModel, \
+    ArgoCDAssets, ArgoCDAssetModel
+from protos.connectors.connector_pb2 import Connector as ConnectorProto
+from protos.base_pb2 import Source, SourceModelType
+from protos.connectors.assets.asset_pb2 import ConnectorModelTypeOptions, AccountConnectorAssetsModelFilters, \
+    AccountConnectorAssets
+
+
+class ArgoCDAssetManager(SourceAssetManager):
+
+    def __init__(self):
+        # print("Inside ArgoCDAssetManager")
+        self.source = Source.ARGOCD
+        self.asset_type_callable_map = {
+            SourceModelType.ARGOCD_APPS: {
+                'options': self.get_apps_options,
+                'values': self.get_apps_values,
+            }
+        }
+
+    @staticmethod
+    def get_apps_options(apps_assets) -> ConnectorModelTypeOptions:
+        all_apps = []
+        for asset in apps_assets:
+            all_apps.append(asset.metadata.get('name', asset.model_uid))
+        apps_options = ArgoCDAppsAssetOptions(apps=all_apps)
+        return ConnectorModelTypeOptions(model_type=SourceModelType.ARGOCD_APPS,
+                                         argocd_apps_model_options=apps_options)
+
+    @staticmethod
+    def get_apps_values(connector: ConnectorProto, filters: AccountConnectorAssetsModelFilters,
+                        apps_assets):
+        which_one_of = filters.WhichOneof('filters')
+        if which_one_of and which_one_of != 'argocd_apps_model_filters':
+            raise ValueError(f"Invalid filter: {which_one_of}")
+
+        options: ArgoCDAppsAssetOptions = filters.argocd_apps_model_filters
+        filter_apps = options.apps
+        if filter_apps:
+            apps_assets = apps_assets.filter(metadata__name__in=filter_apps)
+
+        argocd_asset_protos = []
+
+        for asset in apps_assets:
+            key = asset.model_uid
+            metadata = asset.metadata
+            name = metadata.get("name", "")
+            path = metadata.get("path", "")
+
+            argocd_asset_protos.append(ArgoCDAssetModel(
+                id=UInt64Value(value=asset.id),
+                connector_type=asset.connector_type,
+                type=asset.model_type,
+                last_updated=int(asset.updated_at.replace(tzinfo=timezone.utc).timestamp()) if (
+                    asset.updated_at) else None,
+                argocd_apps=ArgoCDAppsAssetModel(name=StringValue(value=name),
+                                                 path=StringValue(value=path))))
+
+        return AccountConnectorAssets(argocd=ArgoCDAssets(assets=argocd_asset_protos))

connectors/assets/manager/asset_manager_facade.py

@@ -14,6 +14,8 @@
 from connectors.assets.manager.postgres_assets_manager import PostgresAssetManager
 from connectors.assets.manager.bash_assets_manager import BashAssetManager
 from connectors.assets.manager.slack_assets_manager import SlackAssetManager
+from connectors.assets.manager.jira_asset_manager import JiraAssetManager
+from connectors.assets.manager.argocd_asset_manager import ArgoCDAssetManager
 from protos.connectors.assets.asset_pb2 import \
     AccountConnectorAssetsModelFilters as AccountConnectorAssetsModelFiltersProto
 from protos.base_pb2 import Source, SourceModelType
@@ -65,3 +67,5 @@ def get_asset_model_values(self, connector: ConnectorProto, model_type: SourceMo
 asset_manager_facade.register(Source.AZURE, AzureAssetManager())
 asset_manager_facade.register(Source.GKE, GkeAssetManager())
 asset_manager_facade.register(Source.ELASTIC_SEARCH, ElasticSearchAssetManager())
+asset_manager_facade.register(Source.JIRA_CLOUD, JiraAssetManager())
+asset_manager_facade.register(Source.ARGOCD, ArgoCDAssetManager())

connectors/assets/manager/jira_asset_manager.py

@@ -0,0 +1,105 @@
+from datetime import timezone
+
+from google.protobuf.wrappers_pb2 import UInt64Value, StringValue
+
+from connectors.assets.manager.asset_manager import ConnectorAssetManager as SourceAssetManager
+from protos.connectors.assets.jira_asset_pb2 import JiraProjectAssetOptions, JiraAssetModel, JiraProjectAssetModel, \
+    JiraAssets, JiraUserAssetModel, JiraUserAssetOptions
+from protos.connectors.connector_pb2 import Connector as ConnectorProto
+from protos.base_pb2 import Source, SourceModelType
+from protos.connectors.assets.asset_pb2 import ConnectorModelTypeOptions, AccountConnectorAssetsModelFilters, \
+    AccountConnectorAssets
+
+
+class JiraAssetManager(SourceAssetManager):
+
+    def __init__(self):
+        self.source = Source.JIRA_CLOUD
+        self.asset_type_callable_map = {
+            SourceModelType.JIRA_PROJECT: {
+                'options': self.get_project_options,
+                'values': self.get_project_values,
+            },
+            SourceModelType.JIRA_USER: {
+                'options': self.get_user_options,
+                'values': self.get_user_values,
+            }
+        }
+
+    @staticmethod
+    def get_project_options(project_assets) -> ConnectorModelTypeOptions:
+        all_projects = []
+        for asset in project_assets:
+            all_projects.append(asset.metadata.get('name', asset.model_uid))
+        project_options = JiraProjectAssetOptions(p=all_projects)
+        return ConnectorModelTypeOptions(model_type=SourceModelType.JIRA_PROJECT,
+                                         jira_project_model_options=project_options)
+
+    @staticmethod
+    def get_project_values(connector: ConnectorProto, filters: AccountConnectorAssetsModelFilters,
+                           project_assets):
+        which_one_of = filters.WhichOneof('filters')
+        if which_one_of and which_one_of != 'jira_project_model_filters':
+            raise ValueError(f"Invalid filter: {which_one_of}")
+        options: JiraProjectAssetOptions = filters.jira_project_model_filters
+        filter_project_names = options.names
+        if filter_project_names:
+            project_assets = project_assets.filter(metadata__name__in=filter_project_names)
+
+        jira_asset_protos = []
+        for asset in project_assets:
+            key = asset.model_uid
+            metadata = asset.metadata
+            project_id = metadata.get('id', None)
+            name = metadata.get('name', '')
+            self_url = metadata.get('self', '')
+
+            jira_asset_protos.append(JiraAssetModel(
+                id=UInt64Value(value=asset.id),
+                connector_type=asset.connector_type,
+                type=asset.model_type,
+                last_updated=int(asset.updated_at.replace(tzinfo=timezone.utc).timestamp()) if (
+                    asset.updated_at) else None,
+                jira_project=JiraProjectAssetModel(id=StringValue(value=project_id), key=StringValue(value=key),
+                                                   name=StringValue(value=name), self_url=StringValue(value=self_url))))
+
+        return AccountConnectorAssets(jira_cloud=JiraAssets(assets=jira_asset_protos))
+
+    @staticmethod
+    def get_user_options(user_assets) -> ConnectorModelTypeOptions:
+        all_users = []
+        for asset in user_assets:
+            all_users.append(asset.metadata.get('displayName', asset.model_uid))
+        user_options = JiraUserAssetOptions(display_names=all_users)
+        return ConnectorModelTypeOptions(model_type=SourceModelType.JIRA_USER,
+                                         jira_user_model_options=user_options)
+
+    @staticmethod
+    def get_user_values(connector: ConnectorProto, filters: AccountConnectorAssetsModelFilters,
+                        user_assets):
+        which_one_of = filters.WhichOneof('filters')
+        if which_one_of and which_one_of != 'jira_user_model_filters':
+            raise ValueError(f"Invalid filter: {which_one_of}")
+        options: JiraUserAssetOptions = filters.jira_user_model_filters
+        filter_users = options.display_names
+        if filter_users:
+            user_assets = user_assets.filter(metadata__displayName__in=filter_users)
+
+        jira_asset_protos = []
+        for asset in user_assets:
+            account_id = asset.model_uid
+            metadata = asset.metadata
+            display_name = metadata.get('displayName', '')
+            self_url = metadata.get('self', '')
+
+            jira_asset_protos.append(JiraAssetModel(
+                id=UInt64Value(value=asset.id),
+                connector_type=asset.connector_type,
+                type=asset.model_type,
+                last_updated=int(asset.updated_at.replace(tzinfo=timezone.utc).timestamp()) if (
+                    asset.updated_at) else None,
+                jira_user=JiraUserAssetModel(account_id=StringValue(value=account_id),
+                                             display_name=StringValue(value=display_name),
+                                             self_url=StringValue(value=self_url))))
+
+        return AccountConnectorAssets(jira_cloud=JiraAssets(assets=jira_asset_protos))