diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector.zip b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector.zip deleted file mode 100644 index f1ace624272..00000000000 Binary files a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector.zip and /dev/null differ diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/.funcignore b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/.funcignore deleted file mode 100644 index 8b34d0503fe..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/.funcignore +++ /dev/null @@ -1,3 +0,0 @@ -.venv -.vscode -local.settings.json \ No newline at end of file diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/.gitignore b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/.gitignore deleted file mode 100644 index 4281320f8a3..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/.gitignore +++ /dev/null @@ -1,60 +0,0 @@ -# Azure Functions -local.settings.json -__pycache__/ -*.pyc -*.pyo -*.pyd -.Python -bin/ -lib/ -lib64/ -include/ -Scripts/ -pyvenv.cfg - -# Virtual environments -.venv/ -venv/ -env/ -ENV/ - -# VS Code - -*.code-workspace - -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# OS -.DS_Store -Thumbs.db - -# Logs -*.log -logs/ - -# Azure -.azure/ -.python_packages - -#vscode -.vscode \ No newline at end of file diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/README.md b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/README.md deleted file mode 100644 index 10ab37d4ef1..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Lumen Defender Threat Feed Connector - Azure Durable Functions - -## Overview - -This Azure Functions data connector uploads threat intelligence indicators generated by Black Lotus Labs to Microsoft Sentinel. - -**Current Version: 1.1 (Delta Sync)** - -## What It Does - -The connector automatically: - -- **V1.1 Delta Sync (Current):** - - Queries the Lumen Delta API every 15 minutes for changed indicators - - Polls for query completion - - Downloads only delta changes (combined ipv4 + domain indicators) - - Stores threat intel data to blob storage for processing - - Uploads chunks (100 indicators per request) to Microsoft Sentinel via the [STIX Object Upload API](https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api) - - Cleans up temp files in blob storage after upload is complete - -## Requirements - -- Azure Functions runtime v4 (`FUNCTIONS_EXTENSION_VERSION=~4`) -- Python 3.11 (`linuxFxVersion: Python|3.11`) -- Durable Functions extension (`azure-functions-durable`) - -## Schedule - -- **V1.1:** Runs every 15 minutes via timer trigger (`0 */15 * * * *`) -- Each run processes delta changes from the last 15-minute window - -## Architecture - -1. **Timer Function:** Initiates delta query, downloads results to blob storage -2. **Orchestrator:** Coordinates parallel upload activities with rate limit handling -3. **Activity Functions:** Upload indicators from blobs to Sentinel in chunks of 100 - -## Support - -For technical issues or questions about this connector: - -- **Lumen API access**: Contact Lumen support -- **Azure configuration**: Contact your system administrator -- **Connector functionality**: Review Azure Functions logs in the Azure portal - ---- - -*This connector requires active Lumen Reputation API access and Microsoft Sentinel workspace permissions.* - -**Version History:** -- **V1.1** (Oct 2025) - Delta sync with 15-minute intervals -- **V1.0** (Earlier) - Daily full sync diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_cleanup_blob/__init__.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_cleanup_blob/__init__.py deleted file mode 100644 index 283b9244939..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_cleanup_blob/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -import os -from azure.storage.blob import BlobServiceClient -from azure.core.exceptions import ResourceExistsError -from azure.core.exceptions import ResourceNotFoundError - -# Suppress verbose Azure SDK HTTP logs -logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.ERROR) -logging.getLogger('azure.storage').setLevel(logging.ERROR) - -def _get_blob_container(): - """Get blob container client, creating container if needed.""" - conn = os.environ.get('LUMEN_BLOB_CONNECTION_STRING') - if not conn: - raise ValueError('LUMEN_BLOB_CONNECTION_STRING environment variable not set') - - container_name = os.environ.get('LUMEN_BLOB_CONTAINER', 'lumenthreatfeed') - - service_client = BlobServiceClient.from_connection_string(conn) - container_client = service_client.get_container_client(container_name) - - # Create container if it doesn't exist - try: - if not container_client.exists(): - container_client.create_container() - logging.debug(f"Created blob container: {container_name}") - except ResourceExistsError: - pass # Container already exists (race) - except Exception as e: - logging.warning(f"Error ensuring container {container_name}: {e}") - - return container_client - -def main(cleanup_info): - """Activity function to cleanup processed blob files.""" - try: - blob_name = cleanup_info['blob_name'] - run_id = cleanup_info['run_id'] - - logging.debug(f"Cleaning up blob: {blob_name}") - - container_client = _get_blob_container() - blob_client = container_client.get_blob_client(blob_name) - - # Delete the blob - try: - blob_client.delete_blob() - except ResourceNotFoundError: - # Already deleted by a concurrent worker or a previous attempt – treat as success - logging.debug(f"Blob not found during cleanup (already deleted): {blob_name}") - logging.debug(f"✓ Deleted blob: {blob_name}") - - return {'status': 'success', 'blob_name': blob_name} - - except Exception as e: - logging.error(f"Cleanup error for {blob_name}: {e}") - return {'status': 'error', 'blob_name': blob_name, 'error': str(e)} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_cleanup_blob/function.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_cleanup_blob/function.json deleted file mode 100644 index 4a91e410222..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_cleanup_blob/function.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "name": "cleanup_info", - "type": "activityTrigger", - "direction": "in" - } - ] -} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_get_manifest_page/__init__.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_get_manifest_page/__init__.py deleted file mode 100644 index efaabc96f24..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_get_manifest_page/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -import json -import logging -import os -from azure.storage.blob import BlobServiceClient - - -def _get_blob_container(): - conn = os.environ.get('LUMEN_BLOB_CONNECTION_STRING') - if not conn: - raise ValueError('LUMEN_BLOB_CONNECTION_STRING environment variable not set') - - container_name = os.environ.get('LUMEN_BLOB_CONTAINER', 'lumenthreatfeed') - service_client = BlobServiceClient.from_connection_string(conn) - container_client = service_client.get_container_client(container_name) - return container_client - - -def main(work): - """Return a page slice from the manifest blob. - - work = { - 'manifest_blob_name': str, - 'offset': int, - 'limit': int - } - """ - manifest_blob_name = work['manifest_blob_name'] - offset = int(work.get('offset', 0)) - limit = int(work.get('limit', 1000)) - - container_client = _get_blob_container() - blob_client = container_client.get_blob_client(manifest_blob_name) - content = blob_client.download_blob(max_concurrency=1).readall() - items = json.loads(content) - # Return a slice of items - page = items[offset: offset + limit] - return { - 'items': page, - 'next_offset': offset + len(page), - 'has_more': (offset + len(page)) < len(items), - 'total': len(items) - } diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_get_manifest_page/function.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_get_manifest_page/function.json deleted file mode 100644 index b17f1f06d39..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_get_manifest_page/function.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "name": "work", - "type": "activityTrigger", - "direction": "in" - } - ] -} \ No newline at end of file diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_upload_from_blob/__init__.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_upload_from_blob/__init__.py deleted file mode 100644 index 750dbc93fa2..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_upload_from_blob/__init__.py +++ /dev/null @@ -1,159 +0,0 @@ -import logging -import json -import os -import sys -import tempfile -from typing import Optional -from azure.storage.blob import BlobServiceClient -from azure.core.exceptions import ResourceExistsError - -# Add parent directory to path for importing main module -sys.path.append(os.path.dirname(os.path.dirname(__file__))) -from main import LumenSentinelUpdater, LumenSetup, MSALSetup - -# Suppress verbose Azure SDK HTTP logs -logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.ERROR) -logging.getLogger('azure.storage').setLevel(logging.ERROR) - - -def _get_blob_container(): - """Get blob container client, creating container if needed.""" - conn = os.environ.get('LUMEN_BLOB_CONNECTION_STRING') - if not conn: - raise ValueError('LUMEN_BLOB_CONNECTION_STRING environment variable not set') - - container_name = os.environ.get('LUMEN_BLOB_CONTAINER', 'lumenthreatfeed') - - service_client = BlobServiceClient.from_connection_string(conn) - container_client = service_client.get_container_client(container_name) - - # Create container if it doesn't exist (avoid 409 by probing first) - try: - if not container_client.exists(): - container_client.create_container() - logging.debug(f"Created blob container: {container_name}") - else: - logging.debug(f"Blob container already present: {container_name}") - except ResourceExistsError: - logging.debug(f"Blob container {container_name} already exists (race)") - except Exception as e: - logging.warning(f"Error ensuring container {container_name}: {e}") - - return container_client - - -def main(work_unit): - """Activity uploads indicators for a single chunk-blob to Sentinel. - - Downloads the JSONL blob and uploads objects in batches of 100. If rate-limited and - external_backoff flag is set, returns a throttled signal with suggested retry_after seconds. - """ - # Extract parameters - blob_name = work_unit['blob_name'] - indicator_type = work_unit['indicator_type'] - indicators_per_request = int(work_unit.get('indicators_per_request', 100)) - run_id = work_unit['run_id'] - work_unit_id = work_unit['work_unit_id'] - # Default to internal backoff to avoid orchestrator timer floods - external_backoff = bool(work_unit.get('external_backoff', False)) - - logging.info(f"Processing work unit: {work_unit_id}") - - uploaded_total = 0 - error_total = 0 - throttle_total = 0 - - window_desc = "chunk" - - try: - # Initialize updater from environment (avoid secrets in inputs) - api_key = os.environ.get('LUMEN_API_KEY') - base_url = os.environ.get('LUMEN_BASE_URL') - tenant_id = os.environ.get('TENANT_ID') - client_id = os.environ.get('CLIENT_ID') - client_secret = os.environ.get('CLIENT_SECRET') - workspace_id = os.environ.get('WORKSPACE_ID') - - missing = [k for k, v in { - 'LUMEN_API_KEY': api_key, - 'LUMEN_BASE_URL': base_url, - 'TENANT_ID': tenant_id, - 'CLIENT_ID': client_id, - 'CLIENT_SECRET': client_secret, - 'WORKSPACE_ID': workspace_id, - }.items() if not v] - if missing: - raise ValueError(f"Missing required environment variables: {missing}") - - updater = LumenSentinelUpdater( - LumenSetup(api_key, base_url, 3), - MSALSetup(tenant_id, client_id, client_secret, workspace_id) - ) - - # Prepare blob download - container_client = _get_blob_container() - blob_client = container_client.get_blob_client(blob_name) - - # Stream download to a temp file to avoid memory spikes - with tempfile.NamedTemporaryFile(mode='wb', delete=False) as tmp: - tmp_path = tmp.name - downloader = blob_client.download_blob() - downloader.readinto(tmp) - - # Now iterate the file line by line and process all objects in this chunk blob - buffer = [] - CHUNK = indicators_per_request # logical chunk to send to API - - def flush_buffer(batch_info_suffix: str): - nonlocal uploaded_total, error_total, throttle_total, buffer - if not buffer: - return - result = updater.upload_indicators_to_sentinel(buffer, batch_info_suffix, external_backoff=external_backoff) - if result.get('throttled'): - # return early to let the orchestrator back off - return result - uploaded_total += result.get('uploaded_count', 0) - error_total += result.get('error_count', 0) - throttle_total += result.get('throttle_events', 0) - buffer = [] - - try: - with open(tmp_path, 'r', encoding='utf-8') as f: - for line in f: - if not line.strip(): - continue - try: - obj = json.loads(line) - except Exception: - # count malformed object as processed and record error - error_total += 1 - continue - buffer.append(obj) - if len(buffer) >= CHUNK: - result = flush_buffer(f"({indicator_type} {window_desc})") - if isinstance(result, dict) and result.get('throttled'): - return result - - # Flush any remainder for this window (can be < 100) - result = flush_buffer(f"({indicator_type} {window_desc})") - if isinstance(result, dict) and result.get('throttled'): - return result - finally: - try: - os.remove(tmp_path) - except Exception: - pass - - logging.debug( - f"✓ Work unit {work_unit_id} completed: uploaded={uploaded_total}, errors={error_total}, throttles={throttle_total}" - ) - return {'uploaded_count': uploaded_total, 'error_count': error_total, 'throttle_events': throttle_total} - - except Exception as e: - logging.error(f"Activity error for {work_unit_id}: {e}", exc_info=True) - return { - 'uploaded_count': 0, - 'error_count': error_total, - 'throttle_events': throttle_total, - 'error': str(e) - } diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_upload_from_blob/function.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_upload_from_blob/function.json deleted file mode 100644 index 3bc740ab514..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/activity_upload_from_blob/function.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "name": "work_unit", - "type": "activityTrigger", - "direction": "in" - } - ] -} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/main.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/main.py deleted file mode 100644 index 6aec0a7d3ad..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/main.py +++ /dev/null @@ -1,662 +0,0 @@ -""" -Lumen Threat Intelligence Connector for Microsoft Sentinel -Version: 1.1 (Delta Sync) - -This connector integrates Lumen threat intelligence feeds with Microsoft Sentinel. - -V1.1 Changes (Delta Sync): -- Migrated from daily full sync to 15-minute delta sync -- New API flow: POST /reputation-query → Poll GET /reputation-query/{cache_id} → Download results -- Combined indicator types (ipv4 + domain) in single endpoint -- Polling: 1-second intervals, 5-minute timeout -- Enhanced statistics tracking (poll attempts, query times) - -Architecture: -1. Timer Function (every 15 min): Initiates delta query, downloads results to blob storage -2. Orchestrator: Coordinates parallel upload activities with rate limit handling -3. Activity Functions: Upload indicators from blobs to Sentinel in chunks of 100 - -Environment Variables: -- LUMEN_API_KEY: API key for Lumen authentication -- LUMEN_BASE_URL: Base URL for delta API endpoint -- LUMEN_CONFIDENCE_THRESHOLD: Minimum confidence score (default: 80) -- LUMEN_MAX_INDICATORS_PER_TYPE: Testing limit per type (0 = no limit) -- LUMEN_MAX_TOTAL_INDICATORS: Testing limit total (0 = no limit) -- TENANT_ID, CLIENT_ID, CLIENT_SECRET, WORKSPACE_ID: Azure/Sentinel credentials -- LUMEN_BLOB_CONNECTION_STRING: Azure Blob Storage connection -- LUMEN_BLOB_CONTAINER: Blob container name (default: lumenthreatfeed) -""" - -import json -import logging -import os -import requests -import time -import uuid -import ijson -import tempfile -from typing import Dict, List, Optional, Any -from datetime import datetime, timedelta -from msal import ConfidentialClientApplication -from azure.storage.blob import BlobServiceClient - -# Configuration for indicator types and environment-based filtering -INDICATOR_TYPES = { - 'ipv4': os.environ.get('LUMEN_ENABLE_IPV4', 'true').lower() == 'true', - 'domain': os.environ.get('LUMEN_ENABLE_DOMAIN', 'true').lower() == 'true' -} - -# Filter to only enabled types -INDICATOR_TYPES = {k: v for k, v in INDICATOR_TYPES.items() if v} - -# Testing limits (0 = no limit) -MAX_INDICATORS_PER_TYPE = int(os.environ.get('LUMEN_MAX_INDICATORS_PER_TYPE', '0')) -MAX_TOTAL_INDICATORS = int(os.environ.get('LUMEN_MAX_TOTAL_INDICATORS', '0')) - -# Upload chunk size (aligns with Sentinel API limit) -CHUNK_SIZE = 100 - -class LumenSetup: - """Configuration for Lumen API access.""" - - def __init__(self, api_key: str, base_url: str, max_retries: int = 3): - self.api_key = api_key - self.base_url = base_url.rstrip('/') - self.max_retries = max_retries - - # Debug logging to verify environment variables - logging.debug(f"LUMEN_API_KEY loaded: {'***' + self.api_key[-4:] if self.api_key and len(self.api_key) > 4 else 'NOT SET'}") - logging.debug(f"LUMEN_BASE_URL loaded: {self.base_url}") - - if not self.api_key: - raise ValueError("LUMEN_API_KEY environment variable is required") - if not self.base_url: - raise ValueError("LUMEN_BASE_URL environment variable is required") - -class MSALSetup: - """Configuration for Microsoft Authentication Library.""" - - def __init__(self, tenant_id: str, client_id: str, client_secret: str, workspace_id: str): - self.tenant_id = tenant_id - self.client_id = client_id - self.client_secret = client_secret - self.workspace_id = workspace_id - self.authority = f"https://login.microsoftonline.com/{tenant_id}" - self.scope = ["https://management.azure.com/.default"] - - # Validate required parameters - required_vars = { - 'TENANT_ID': tenant_id, - 'CLIENT_ID': client_id, - 'CLIENT_SECRET': client_secret, - 'WORKSPACE_ID': workspace_id - } - - for var_name, value in required_vars.items(): - if not value: - raise ValueError(f"{var_name} environment variable is required") - -class LumenSentinelUpdater: - """Enhanced Lumen threat intelligence connector with blob storage support.""" - - def __init__(self, lumen_setup: LumenSetup, msal_setup: MSALSetup): - self.lumen_setup = lumen_setup - self.msal_setup = msal_setup - self.access_token = None - self.token_expiry = None - self.session = requests.Session() - self.session.headers.update({ - 'User-Agent': 'LumenSentinelConnector/1.1', - 'Accept': 'application/json' - }) - - # Configure confidence threshold - self.confidence_threshold = int(os.environ.get('LUMEN_CONFIDENCE_THRESHOLD', '80')) - - # Statistics tracking - self.stats = { - 'run_id': None, - 'start_time': None, - 'indicators_processed': 0, - 'indicators_uploaded': 0, - 'errors': 0, - 'rate_limit_events': 0, - 'processing_time': 0, - 'delta_query_time': 0, - 'poll_attempts': 0, - 'cache_query_time': 0 - } - - def log_config(self): - """Log current configuration for debugging.""" - enabled_types = [k for k, v in INDICATOR_TYPES.items() if v] - - logging.debug("=== LUMEN CONNECTOR V1.1 (DELTA SYNC) ===") - logging.debug(f"Mode: Delta sync (15-minute intervals)") - logging.debug(f"Enabled indicator types: {enabled_types if enabled_types else 'All types (combined endpoint)'}") - logging.debug(f"Confidence threshold: {self.confidence_threshold}") - logging.debug(f"Max indicators per type: {MAX_INDICATORS_PER_TYPE if MAX_INDICATORS_PER_TYPE > 0 else 'No limit'}") - logging.debug(f"Max total indicators: {MAX_TOTAL_INDICATORS if MAX_TOTAL_INDICATORS > 0 else 'No limit'}") - logging.debug(f"Lumen API base URL: {self.lumen_setup.base_url}") - logging.debug(f"Workspace ID: {self.msal_setup.workspace_id}") - logging.debug("=========================================") - - def _get_access_token(self) -> str: - """Get or refresh Microsoft access token for Sentinel API.""" - now = datetime.utcnow() - - # Check if we have a valid token - if self.access_token and self.token_expiry and now < self.token_expiry: - return self.access_token - - logging.debug("Obtaining new access token...") - - try: - app = ConfidentialClientApplication( - client_id=self.msal_setup.client_id, - client_credential=self.msal_setup.client_secret, - authority=self.msal_setup.authority - ) - - result = app.acquire_token_for_client(scopes=self.msal_setup.scope) - - if "access_token" in result: - self.access_token = result["access_token"] - # Set expiry to 50 minutes from now (tokens typically last 60 minutes) - self.token_expiry = now + timedelta(minutes=50) - logging.debug("✓ Access token obtained successfully") - return self.access_token - else: - error_msg = result.get("error_description", result.get("error", "Unknown error")) - raise Exception(f"Failed to obtain access token: {error_msg}") - - except Exception as e: - logging.error(f"Token acquisition error: {e}") - raise - - def initiate_delta_query(self) -> str: - """POST to /reputation-query endpoint to initiate delta query. - - Returns: - cache_id (str): The cache ID to use for polling - - Raises: - Exception: If POST fails or response doesn't contain cache_id - """ - headers = { - 'Authorization': self.lumen_setup.api_key, - 'Content-Type': 'application/json' - } - - url = self.lumen_setup.base_url - - for attempt in range(self.lumen_setup.max_retries): - try: - logging.info(f"Initiating delta query (attempt {attempt + 1})") - - response = self.session.post(url, headers=headers, json={}, timeout=30) - response.raise_for_status() - - data = response.json() - cache_id = data.get('cache_id') - - if cache_id: - logging.debug(f"✓ Delta query initiated, cache_id: {cache_id}") - return cache_id - else: - raise ValueError("No 'cache_id' field in POST response") - - except requests.exceptions.RequestException as e: - if attempt == self.lumen_setup.max_retries - 1: - logging.error(f"✗ Failed to initiate delta query: {e}") - raise - else: - logging.warning(f"Retry {attempt + 1} for delta query initiation: {e}") - time.sleep(2 ** attempt) - - def poll_query_status(self, cache_id: str, timeout: int = 300, poll_interval: int = 1) -> Dict[str, Any]: - """Poll GET /reputation-query/{cache_id} until status is COMPLETED or timeout. - - Args: - cache_id: The cache ID from initiate_delta_query - timeout: Maximum time to wait in seconds (default 300 = 5 minutes) - poll_interval: Time between polls in seconds (default 1 second) - - Returns: - dict: Response containing 'status' and 'results' (presigned URL) - - Raises: - TimeoutError: If polling exceeds timeout - Exception: If polling fails - """ - headers = { - 'Authorization': self.lumen_setup.api_key, - 'Content-Type': 'application/json' - } - - url = f"{self.lumen_setup.base_url}/{cache_id}" - start_time = time.time() - poll_count = 0 - - logging.info(f"Polling for query completion (timeout: {timeout}s, interval: {poll_interval}s)") - - while True: - poll_count += 1 - elapsed = time.time() - start_time - - if elapsed > timeout: - error_msg = f"Polling timeout after {elapsed:.1f}s ({poll_count} attempts)" - logging.error(f"✗ {error_msg}") - raise TimeoutError(error_msg) - - try: - response = self.session.get(url, headers=headers, timeout=30) - response.raise_for_status() - - data = response.json() - status = data.get('status', '').upper() - - if response.status_code == 200 and status == 'COMPLETED': - results_url = data.get('results') - if not results_url: - raise ValueError("COMPLETED response missing 'results' field") - - self.stats['poll_attempts'] = poll_count - self.stats['cache_query_time'] = elapsed - - logging.info(f"✓ Query completed after {elapsed:.1f}s ({poll_count} polls)") - logging.debug(f"Results URL obtained: {results_url[:80]}...") - return data - - elif response.status_code == 202: - # Still processing - if poll_count % 10 == 0: # Log every 10 polls to reduce noise - logging.info(f"Poll #{poll_count}: Status {status} (elapsed: {elapsed:.1f}s)") - time.sleep(poll_interval) - continue - - else: - raise ValueError(f"Unexpected response: status_code={response.status_code}, status={status}") - - except requests.exceptions.RequestException as e: - logging.warning(f"Poll attempt {poll_count} failed: {e}") - time.sleep(poll_interval) - continue - - def get_delta_results_url(self) -> str: - """Orchestrate the full delta query flow: POST → Poll → Extract presigned URL. - - Returns: - str: Presigned URL for downloading delta results - - Raises: - Exception: If any step fails - """ - start_time = time.time() - - logging.info("=== STARTING DELTA QUERY FLOW ===") - - # Step 1: Initiate query - cache_id = self.initiate_delta_query() - - # Step 2: Poll until completed - response = self.poll_query_status(cache_id) - - # Step 3: Extract results URL - results_url = response.get('results') - - total_time = time.time() - start_time - self.stats['delta_query_time'] = total_time - - logging.info(f"=== DELTA QUERY COMPLETE ({total_time:.1f}s) ===") - - return results_url - - def get_lumen_presigned_urls(self, indicator_types: Dict[str, bool]) -> Dict[str, str]: - """Get presigned URLs from Lumen API for enabled indicator types.""" - headers = { - 'Authorization': self.lumen_setup.api_key, - 'Content-Type': 'application/json' - } - presigned_urls = {} - - for indicator_type, enabled in indicator_types.items(): - if not enabled: - continue - - url = f"{self.lumen_setup.base_url}/{indicator_type}" - - for attempt in range(self.lumen_setup.max_retries): - try: - logging.info(f"Getting presigned URL for {indicator_type} (attempt {attempt + 1})") - - response = self.session.get(url, headers=headers, timeout=30) - response.raise_for_status() - - data = response.json() - presigned_url = data.get('url') - - if presigned_url: - presigned_urls[indicator_type] = presigned_url - logging.info(f"✓ Got presigned URL for {indicator_type}") - break - else: - raise ValueError(f"No 'url' field in response for {indicator_type}") - - except requests.exceptions.RequestException as e: - if attempt == self.lumen_setup.max_retries - 1: - logging.error(f"✗ Failed to get presigned URL for {indicator_type}: {e}") - else: - logging.warning(f"Retry {attempt + 1} for {indicator_type}: {e}") - time.sleep(2 ** attempt) - - return presigned_urls - - def stream_and_filter_to_blob(self, container_client, indicator_type: str, presigned_url: str, run_id: str, max_remaining: Optional[int] = None) -> List[Dict[str, Any]]: - """ - Stream threat data from Lumen and write filtered indicators into chunked JSONL blobs of size CHUNK_SIZE. - Returns a list of chunk descriptors: [{ indicator_type, blob_name, filtered_count, ... }] - If max_remaining is provided (>0), stop accepting new indicators once that budget is exhausted. - """ - start_time = time.time() - blob_name = f"{run_id}-{indicator_type}-{uuid.uuid4().hex[:8]}.jsonl" - - logging.debug(f"Streaming {indicator_type} data to blob: {blob_name}") - - total_downloaded = 0 - filtered_total = 0 - # How many indicators we can still accept globally for this call; None means unlimited - remaining_budget = max_remaining if (isinstance(max_remaining, int) and max_remaining > 0) else None - chunks: List[Dict[str, Any]] = [] - chunk_index = 0 - chunk_count = 0 - tmp = None - tmp_path = None - - def start_new_chunk(): - nonlocal tmp, tmp_path, chunk_index, chunk_count, blob_name - if tmp is not None: - try: - tmp.close() - except Exception: - pass - chunk_index += 1 - chunk_count = 0 - # Name per chunk - blob_name = f"{run_id}-{indicator_type}-{uuid.uuid4().hex[:8]}-part-{chunk_index:05d}.jsonl" - tmp = tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8', delete=False) - tmp_path = tmp.name - - def finalize_current_chunk(): - nonlocal tmp, tmp_path, chunk_count, filtered_total - if tmp is None: - return None - try: - tmp.flush() - os.fsync(tmp.fileno()) - except Exception: - pass - finally: - try: - tmp.close() - except Exception: - pass - # Upload if wrote anything - if chunk_count > 0 and tmp_path and os.path.exists(tmp_path): - try: - blob_client_local = container_client.get_blob_client(blob_name) - with open(tmp_path, 'rb') as f: - blob_client_local.upload_blob(f, overwrite=True) - chunks.append({ - 'indicator_type': indicator_type, - 'blob_name': blob_name, - 'filtered_count': chunk_count, - 'confidence_threshold': self.confidence_threshold - }) - filtered_total += chunk_count - finally: - try: - os.remove(tmp_path) - except Exception: - pass - - # Begin first chunk - start_new_chunk() - - try: - try: - # Stream response - response = self.session.get(presigned_url, stream=True, timeout=(10, 300)) - logging.debug( - "Presigned GET status=%s, content-type=%s, length=%s", - response.status_code, - response.headers.get('Content-Type'), - response.headers.get('Content-Length') - ) - response.raise_for_status() - - # Check if response is empty - if not response.content and response.status_code == 200: - logging.warning(f"Empty response from presigned URL for {indicator_type}") - logging.info(f"Response headers: {dict(response.headers)}") - return chunks # Return empty chunks list - - # For small responses or debugging, try reading as text first - response_text = response.text - - if not response_text or len(response_text.strip()) == 0: - logging.warning(f"Empty or whitespace-only response for {indicator_type}") - return chunks - - logging.debug(f"Response size: {len(response_text)} bytes") - logging.debug(f"Response preview (first 500 chars): {response_text[:500]}") - - parsed_any = False - - def handle_obj(obj: Dict[str, Any]): - nonlocal total_downloaded, chunk_count, remaining_budget - total_downloaded += 1 - confidence = obj.get('confidence', 0) - if confidence >= self.confidence_threshold: - # Enforce global remaining budget if provided - if remaining_budget is not None and remaining_budget <= 0: - return True # signal to stop - tmp.write(json.dumps(obj)) - tmp.write('\n') - chunk_count += 1 - if remaining_budget is not None: - remaining_budget -= 1 - # Rotate per-chunk - if chunk_count >= CHUNK_SIZE: - finalize_current_chunk() - start_new_chunk() - # Per-type limit (for testing) - if MAX_INDICATORS_PER_TYPE > 0 and (filtered_total + chunk_count) >= MAX_INDICATORS_PER_TYPE: - return True # signal to stop - return False - - # Parse the JSON response - try: - data = json.loads(response_text) - logging.debug(f"Successfully parsed JSON. Type: {type(data)}") - - if isinstance(data, dict): - if 'stixobjects' in data: - stix_objects = data['stixobjects'] - logging.debug(f"Found 'stixobjects' array with {len(stix_objects)} items") - for obj in stix_objects: - parsed_any = True - if handle_obj(obj): - break - else: - logging.warning(f"Response is a dict but missing 'stixobjects' key. Keys: {list(data.keys())}") - elif isinstance(data, list): - logging.debug(f"Response is a list with {len(data)} items") - for obj in data: - parsed_any = True - if handle_obj(obj): - break - else: - raise ValueError(f"Unexpected response type: {type(data)}") - - if not parsed_any: - logging.warning(f"No indicators were parsed from response") - - except json.JSONDecodeError as jde: - logging.error(f"JSON decode error for {indicator_type}: {jde}") - logging.error(f"Response text (first 1000 chars): {response_text[:1000]}") - raise - - except Exception as e: - logging.error(f"Error streaming {indicator_type} to blob: {e}") - raise - finally: - # finalize last partial chunk - finalize_current_chunk() - - except Exception: - # Ensure any open temp file is removed - try: - if tmp is not None: - tmp.close() - except Exception: - pass - try: - if tmp_path and os.path.exists(tmp_path): - os.remove(tmp_path) - except Exception: - pass - raise - - processing_time = time.time() - start_time - filter_rate = (filtered_total / total_downloaded * 100) if total_downloaded > 0 else 0 - logging.debug(f" • Confidence ≥{self.confidence_threshold}: {filtered_total:,} objects ({filter_rate:.1f}%)") - logging.debug(f" • Processing time: {processing_time:.1f}s") - logging.debug(f" • Chunks created: {len(chunks)}") - logging.info(f"📊 SUMMARY: {filtered_total:,} of {total_downloaded:,} {indicator_type} indicators passed filtering in {len(chunks)} chunk(s)") - - return chunks - - def upload_indicators_to_sentinel(self, stix_objects: List[Dict], batch_info: str = "", external_backoff: bool = False) -> Dict[str, Any]: - """Upload STIX indicators to Microsoft Sentinel with enhanced error handling.""" - if not stix_objects: - return {'uploaded_count': 0, 'error_count': 0, 'throttle_events': 0} - - access_token = self._get_access_token() - - # Prepare upload URL and headers - upload_url = ( - f"https://api.ti.sentinel.azure.com/workspaces/{self.msal_setup.workspace_id}/" - "threat-intelligence-stix-objects:upload" - ) - - headers = { - 'Authorization': f'Bearer {access_token}', - 'Content-Type': 'application/json', - 'User-Agent': 'LumenSentinelConnector/1.1' - } - - params = {'api-version': '2024-02-01-preview'} - - # Process in chunks of 100 (Sentinel API limit) - chunk_size = CHUNK_SIZE - uploaded_count = 0 - error_count = 0 - throttle_events = 0 - - for i in range(0, len(stix_objects), chunk_size): - chunk = stix_objects[i:i + chunk_size] - chunk_num = (i // chunk_size) + 1 - total_chunks = (len(stix_objects) + chunk_size - 1) // chunk_size - - payload = { - 'sourcesystem': 'Lumen', - 'stixobjects': chunk - } - - # Log a sample indicator to verify format - if chunk_num == 1 and len(chunk) > 0: - logging.debug(f"Sample indicator being sent: {json.dumps(chunk[0], indent=2)[:500]}") - - max_retries = 3 - retry_delay = 0.5 - - for attempt in range(max_retries): - try: - logging.info( - f"Uploading chunk {chunk_num}/{total_chunks} {batch_info} " - f"({len(chunk)} indicators, attempt {attempt + 1})" - ) - - response = self.session.post( - upload_url, json=payload, headers=headers, params=params, timeout=60 - ) - - # Log the response details - logging.info(f"Sentinel API Response - Status: {response.status_code}") - logging.debug(f"Sentinel API Response - Headers: {dict(response.headers)}") - - # ALWAYS log the response body for 200s to see what Sentinel says - if response.status_code == 200: - logging.debug(f"Sentinel API Response Body: {response.text[:1000]}") - else: - logging.warning( - f"Sentinel API Error {response.status_code}: {response.text[:200]}..." - ) - - if response.status_code == 200: - uploaded_count += len(chunk) - logging.info( - f"✓ Chunk {chunk_num} uploaded successfully {batch_info} - {len(chunk)} indicators" - ) - break - elif response.status_code == 429: - # Rate limiting - throttle_events += 1 - retry_after = int(response.headers.get('Retry-After', 60)) - logging.warning(f"Rate limited on chunk {chunk_num} {batch_info}") - if external_backoff: - # Let the orchestrator manage backoff deterministically - return { - 'uploaded_count': uploaded_count, - 'error_count': error_count, - 'throttle_events': throttle_events, - 'throttled': True, - 'retry_after': retry_after - } - else: - logging.warning( - f"Sleeping {retry_after}s inside activity (internal backoff)" - ) - time.sleep(retry_after) - continue - else: - error_msg = f"HTTP {response.status_code}: {response.text[:200]}" - if attempt == max_retries - 1: - logging.error( - f"✗ Chunk {chunk_num} failed {batch_info}: {error_msg}" - ) - error_count += len(chunk) - else: - logging.warning( - f"Retry chunk {chunk_num} {batch_info}: {error_msg}" - ) - time.sleep(retry_delay) - retry_delay *= 2 - - except requests.exceptions.RequestException as e: - if attempt == max_retries - 1: - logging.error( - f"✗ Chunk {chunk_num} network error {batch_info}: {e}" - ) - error_count += len(chunk) - else: - logging.warning(f"Retry chunk {chunk_num} {batch_info}: {e}") - time.sleep(retry_delay) - retry_delay *= 2 - - return { - 'uploaded_count': uploaded_count, - 'error_count': error_count, - 'throttle_events': throttle_events - } diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/orchestrator_function/__init__.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/orchestrator_function/__init__.py deleted file mode 100644 index a03178dc47a..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/orchestrator_function/__init__.py +++ /dev/null @@ -1,200 +0,0 @@ -import logging -import os -from datetime import timedelta -import azure.functions as func -import azure.durable_functions as df - -def orchestrator_function(context): - """Enhanced orchestrator with dynamic work distribution and progress tracking.""" - try: - input_data = context.get_input() or {} - - if not input_data: - raise ValueError("No input data provided to orchestrator") - - # Basic controls - run_id = input_data.get('run_id', 'unknown') - indicators_per_request = int(input_data.get('indicators_per_request', 100)) - max_concurrent = int(input_data.get('max_concurrent_activities', os.environ.get('LUMEN_MAX_CONCURRENT_ACTIVITIES', '10'))) - continue_after_units = int(os.environ.get('LUMEN_CONTINUE_AFTER_UNITS', '50')) - - # Manifest paging and optional legacy direct-list support - manifest_blob_name = input_data.get('manifest_blob_name') - page_offset = int(input_data.get('page_offset', 0)) - page_size = int(input_data.get('page_size', int(os.environ.get('LUMEN_MANIFEST_PAGE_SIZE', '500')))) - unit_start_index = int(input_data.get('unit_start_index', 0)) - blob_sources = input_data.get('blob_sources', []) - - # Accumulate totals across ContinueAsNew cycles - summary_accum = input_data.get('summary_accum', { - 'uploaded': 0, - 'errors': 0, - 'throttles': 0, - 'work_units_completed': 0, - }) - - # Work units can be provided for resume; otherwise build from manifest or blob_sources - work_units = input_data.get('work_units') - if work_units is None: - work_units = [] - if manifest_blob_name: - page = yield context.call_activity('activity_get_manifest_page', { - 'manifest_blob_name': manifest_blob_name, - 'offset': page_offset, - 'limit': page_size, - }) - items = page.get('items', []) - items_to_process = items[unit_start_index:] if unit_start_index > 0 else items - for source in items_to_process: - work_units.append({ - 'blob_name': source['blob_name'], - 'indicator_type': source.get('indicator_type', 'unknown'), - 'indicators_per_request': indicators_per_request, - 'run_id': run_id, - 'work_unit_id': f"{run_id}-{source.get('indicator_type','x')}-{source['blob_name']}", - 'process_all': True, - # internal backoff by default inside activity - }) - has_more = bool(page.get('has_more', False)) - next_offset = int(page.get('next_offset', page_offset + len(items))) - else: - for source in blob_sources: - work_units.append({ - 'blob_name': source['blob_name'], - 'indicator_type': source.get('indicator_type', 'unknown'), - 'indicators_per_request': indicators_per_request, - 'run_id': run_id, - 'work_unit_id': f"{run_id}-{source.get('indicator_type','x')}-{source['blob_name']}", - 'process_all': True, - # internal backoff by default inside activity - }) - has_more = False - next_offset = 0 - - total_work_units = len(work_units) - - # Execute with limited parallelism (fan-out/fan-in per batch) - results = [] - total_throttle_events = 0 - batch_size = max_concurrent - index = 0 - processed_since_continue = 0 - - while index < len(work_units): - # Important: slice by remaining, not assumed batch_size - batch = work_units[index:index + batch_size] - batch_num = (index // batch_size) + 1 - total_batches = (len(work_units) + batch_size - 1) // batch_size - - tasks = [context.call_activity('activity_upload_from_blob', inp) for inp in batch] - results_batch = yield context.task_all(tasks) - - batch_throttle = 0 - throttled_inputs = [] - max_retry_after = 0 - cleaned_up = 0 - for inp, res in zip(batch, results_batch): - if isinstance(res, dict) and res.get('throttled'): - retry_after = int(res.get('retry_after', 60)) - max_retry_after = max(max_retry_after, retry_after) - batch_throttle += int(res.get('throttle_events', 0)) - throttled_inputs.append(inp) - continue - - yield context.call_activity('activity_cleanup_blob', { - 'blob_name': inp['blob_name'], - 'run_id': run_id, - }) - cleaned_up += 1 - results.append(res) - - if throttled_inputs: - backoff_seconds = min(max_retry_after if max_retry_after > 0 else 60, 300) - logging.info( - f"Throttled by Sentinel: backing off {backoff_seconds}s; " - f"requeueing {len(throttled_inputs)} work unit(s)" - ) - yield context.create_timer(context.current_utc_datetime + timedelta(seconds=backoff_seconds)) - work_units.extend(throttled_inputs) - - total_throttle_events += batch_throttle - total_uploaded_so_far = sum((r or {}).get('uploaded_count', 0) for r in results) - logging.debug(f"Progress: Blob batch {batch_num}/{total_batches} - {total_uploaded_so_far:,} uploaded") - - processed_since_continue += cleaned_up - - # No mid-page ContinueAsNew: finish all requeued items in this page - - # Advance by actual batch length so that any re-appended throttled work - # will still be picked up by the while condition. - index += len(batch) - - total_uploaded = sum((r or {}).get('uploaded_count', 0) for r in results) - total_errors = sum((r or {}).get('error_count', 0) for r in results) - - # Combine with accumulator to produce unified totals across continues - cumulative_uploaded = summary_accum.get('uploaded', 0) + total_uploaded - cumulative_errors = summary_accum.get('errors', 0) + total_errors - cumulative_throttles = summary_accum.get('throttles', 0) + total_throttle_events - cumulative_units = summary_accum.get('work_units_completed', 0) + processed_since_continue - - logging.info("=== ORCHESTRATION COMPLETE ===") - logging.info(f"Run ID: {run_id}") - logging.info(f"This segment — work units: {total_work_units}, uploaded: {total_uploaded}, errors: {total_errors}, throttles: {total_throttle_events}") - logging.info( - f"CUMULATIVE — work units completed: {cumulative_units}, " - f"uploaded: {cumulative_uploaded}, errors: {cumulative_errors}, throttles: {cumulative_throttles}" - ) - logging.debug(f"Blob sources processed (if legacy path): {len(blob_sources)}") - - if 'has_more' in locals() and has_more and manifest_blob_name: - logging.debug(f"Paging manifest: advancing to next page at offset {next_offset}") - # Fold this segment's totals into the accumulator before paging to next - segment_uploaded = sum((r or {}).get('uploaded_count', 0) for r in results) - segment_errors = sum((r or {}).get('error_count', 0) for r in results) - segment_throttles = total_throttle_events - segment_units = processed_since_continue - new_summary = { - 'uploaded': summary_accum.get('uploaded', 0) + segment_uploaded, - 'errors': summary_accum.get('errors', 0) + segment_errors, - 'throttles': summary_accum.get('throttles', 0) + segment_throttles, - 'work_units_completed': summary_accum.get('work_units_completed', 0) + segment_units, - } - new_input = { - 'run_id': run_id, - 'indicators_per_request': indicators_per_request, - 'max_concurrent_activities': max_concurrent, - 'manifest_blob_name': manifest_blob_name, - 'page_offset': next_offset, - 'page_size': page_size, - 'summary_accum': new_summary, - } - context.continue_as_new(new_input) - return - - # No final safety carry-over: rely on manifest paging and unit_start_index - - return { - 'success': True, - 'run_id': run_id, - 'total_work_units': total_work_units, - 'segment_uploaded': total_uploaded, - 'segment_errors': total_errors, - 'segment_throttle_events': total_throttle_events, - 'cumulative_uploaded': cumulative_uploaded, - 'cumulative_errors': cumulative_errors, - 'cumulative_throttle_events': cumulative_throttles, - 'cumulative_work_units_completed': cumulative_units, - 'blob_sources_processed': len(blob_sources), - } - - except Exception as e: - logging.error(f"Orchestrator error: {type(e).__name__}: {e}", exc_info=True) - return { - 'success': False, - 'error': str(e), - 'run_id': run_id if 'run_id' in locals() else 'unknown', - } - -# Create the orchestrator function binding -main = df.Orchestrator.create(orchestrator_function) diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/orchestrator_function/function.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/orchestrator_function/function.json deleted file mode 100644 index 83baac61e40..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/orchestrator_function/function.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "name": "context", - "type": "orchestrationTrigger", - "direction": "in" - } - ] -} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/requirements.txt b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/requirements.txt deleted file mode 100644 index 6e0266bc1ff..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Azure Functions Core -azure-functions>=1.11.0 -azure-functions-durable>=1.1.6 - -# JSON streaming -ijson>=3.2.0 - -# Authentication -msal>=1.20.0 - -# HTTP client with rate limiting and retry logic -requests>=2.28.0 -requests-ratelimiter>=0.4.0 -urllib3>=1.26.0 - -# JSON handling -python-dateutil>=2.8.0 - -# Azure Blob Storage -azure-storage-blob diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/timer_starter_function/__init__.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/timer_starter_function/__init__.py deleted file mode 100644 index 8ced0310090..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/timer_starter_function/__init__.py +++ /dev/null @@ -1,221 +0,0 @@ -import logging -import json -import azure.functions as func -import azure.durable_functions as df -import os -import time -import uuid -import sys -from datetime import datetime, timezone -from azure.storage.blob import BlobServiceClient -from azure.core.exceptions import ResourceExistsError - -# Add parent directory to path for importing main module -sys.path.append(os.path.dirname(os.path.dirname(__file__))) -from main import LumenSetup, MSALSetup, LumenSentinelUpdater, INDICATOR_TYPES, MAX_TOTAL_INDICATORS - -# Suppress verbose Azure SDK logging (avoid 409 ContainerAlreadyExists noise) -logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.ERROR) -logging.getLogger('azure.storage').setLevel(logging.ERROR) - -# Configuration constants -CONFIDENCE_THRESHOLD = int(os.environ.get('LUMEN_CONFIDENCE_THRESHOLD', '80')) -BLOB_CONTAINER = os.environ.get('LUMEN_BLOB_CONTAINER', 'lumenthreatfeed') - -def _get_blob_container(): - """Get blob container client, creating container if needed.""" - conn = os.environ.get('LUMEN_BLOB_CONNECTION_STRING') - if not conn: - raise ValueError('LUMEN_BLOB_CONNECTION_STRING environment variable not set') - - service_client = BlobServiceClient.from_connection_string(conn) - container_client = service_client.get_container_client(BLOB_CONTAINER) - - # Create container if it doesn't exist (avoid 409 by probing first) - try: - if not container_client.exists(): - container_client.create_container() - logging.debug(f"Created blob container: {BLOB_CONTAINER}") - else: - logging.debug(f"Blob container already present: {BLOB_CONTAINER}") - except ResourceExistsError: - # Race: another worker created it after our exists() check - logging.debug(f"Blob container {BLOB_CONTAINER} already exists (race)") - except Exception as e: - logging.warning(f"Error ensuring container {BLOB_CONTAINER}: {e}") - - return container_client - -def _cleanup_blob_container(container_client): - """Clean up stale files in blob storage container.""" - try: - logging.debug("🧹 Starting blob storage housekeeping...") - - # List all blobs in the container - blob_list = list(container_client.list_blobs()) - - if not blob_list: - logging.debug("✓ Blob container is already clean (no files found)") - return - - deleted_count = 0 - total_size = 0 - - # Delete all blobs - for blob in blob_list: - try: - # Get blob size for reporting - blob_size = blob.size if hasattr(blob, 'size') else 0 - total_size += blob_size - - # Delete the blob - container_client.delete_blob(blob.name) - deleted_count += 1 - logging.debug(f"Deleted blob: {blob.name} ({blob_size:,} bytes)") - - except Exception as e: - logging.warning(f"Failed to delete blob {blob.name}: {e}") - - # Convert bytes to MB for reporting - total_size_mb = total_size / (1024 * 1024) - - logging.debug(f"✓ Housekeeping complete: deleted {deleted_count:,} files " - f"({total_size_mb:.2f} MB freed)") - - except Exception as e: - logging.error(f"✗ Blob housekeeping failed: {e}") - # Don't fail the entire process if housekeeping fails - pass - -def _generate_run_id() -> str: - """Generate unique run ID for tracking.""" - timestamp = datetime.utcnow().strftime('%Y%m%d%H%M%S') - unique_id = uuid.uuid4().hex[:6] - return f"{timestamp}-{unique_id}" - -async def main(mytimer: func.TimerRequest, starter: str) -> None: - """Timer trigger for scheduled threat intelligence updates.""" - try: - logging.info("=== TIMER TRIGGER FIRED ===") - logging.info("Starting scheduled Lumen threat feed update...") - - # Housekeeping: Clean up any stale files from previous runs - logging.debug("🧹 Performing housekeeping...") - try: - container_client = _get_blob_container() - _cleanup_blob_container(container_client) - except Exception as e: - logging.warning(f"Housekeeping failed: {e}") - - client = df.DurableOrchestrationClient(starter) - run_id = _generate_run_id() - logging.info(f"Generated run ID: {run_id}") - - # Get environment configuration - config = { - 'LUMEN_API_KEY': os.environ.get('LUMEN_API_KEY'), - 'LUMEN_BASE_URL': os.environ.get('LUMEN_BASE_URL'), - 'TENANT_ID': os.environ.get('TENANT_ID'), - 'CLIENT_ID': os.environ.get('CLIENT_ID'), - 'CLIENT_SECRET': os.environ.get('CLIENT_SECRET'), - 'WORKSPACE_ID': os.environ.get('WORKSPACE_ID') - } - - # Validate required config - missing_vars = [k for k, v in config.items() if not v] - if missing_vars: - logging.error(f"Missing environment variables: {missing_vars}") - return - - # Initialize components - lumen_setup = LumenSetup(config['LUMEN_API_KEY'], config['LUMEN_BASE_URL']) - updater = LumenSentinelUpdater( - lumen_setup, - MSALSetup(config['TENANT_ID'], config['CLIENT_ID'], - config['CLIENT_SECRET'], config['WORKSPACE_ID']) - ) - - updater.log_config() - - # Get blob container - container_client = _get_blob_container() - - # Phase 1: Stream delta data to blob storage - logging.info("=== PHASE 1: DELTA SYNC - STREAMING TO BLOB STORAGE ===") - - # Get delta results URL via POST → Poll → Extract - try: - presigned_url = updater.get_delta_results_url() - logging.info(f"✓ Delta results URL obtained") - except TimeoutError as e: - logging.error(f"✗ Delta query timeout: {e}") - logging.info("Waiting for next scheduled cycle (15 minutes)") - return - except Exception as e: - logging.error(f"✗ Failed to get delta results: {e}") - logging.info("Waiting for next scheduled cycle (15 minutes)") - return - - # Stream delta data to blobs with optional global cap - blob_sources = [] # flattened list of chunk descriptors - remaining_total = MAX_TOTAL_INDICATORS if MAX_TOTAL_INDICATORS > 0 else None - - try: - # Delta endpoint returns combined indicator types - indicator_type = "delta" - - # Calculate max_remaining for this call - per_call_remaining = remaining_total if remaining_total is not None else None - chunks = updater.stream_and_filter_to_blob( - container_client, indicator_type, presigned_url, run_id, max_remaining=per_call_remaining - ) - blob_sources.extend(chunks) - total_filtered = sum(c.get('filtered_count', 0) for c in chunks) - logging.info(f"✓ Streamed delta indicators: {total_filtered:,} objects in {len(chunks)} chunk(s)") - - except Exception as e: - logging.error(f"✗ Failed to stream delta indicators: {e}") - return - - if not blob_sources: - logging.error("No data was successfully streamed to blobs") - return - - # Write a compact manifest containing only the fields needed for upload - manifest_name = f"{run_id}-manifest.json" - manifest_items = [ - { - 'blob_name': s['blob_name'], - 'indicator_type': s.get('indicator_type', 'unknown') - } - for s in blob_sources - ] - try: - manifest_client = container_client.get_blob_client(manifest_name) - manifest_client.upload_blob(json.dumps(manifest_items).encode('utf-8'), overwrite=True) - logging.debug(f"✓ Wrote manifest: {manifest_name} ({len(manifest_items)} items)") - except Exception as e: - logging.error(f"Failed to write manifest {manifest_name}: {e}") - return - - # Phase 2: Start orchestration for upload - logging.info("=== PHASE 2: STARTING ORCHESTRATION ===") - - orchestration_input = { - 'run_id': run_id, - # pass manifest pointer instead of full list - 'manifest_blob_name': manifest_name, - # keep payload minimal; activities read config from environment - 'indicators_per_request': 100, # Keep at 100 (Sentinel API limit) - 'max_concurrent_activities': int(os.environ.get('LUMEN_MAX_CONCURRENT_ACTIVITIES', '10')), - # paging controls - 'page_offset': 0, - 'page_size': int(os.environ.get('LUMEN_MANIFEST_PAGE_SIZE', '500')) - } - - instance_id = await client.start_new("orchestrator_function", None, orchestration_input) - - logging.info(f"✓ Timer triggered orchestration started: {instance_id}") - - except Exception as e: - logging.error(f"Timer trigger error: {e}", exc_info=True) diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/timer_starter_function/function.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/timer_starter_function/function.json deleted file mode 100644 index bd6c96c78af..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/timer_starter_function/function.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "name": "mytimer", - "type": "timerTrigger", - "direction": "in", - "schedule": "0 */15 * * * *", - "runOnStartup": false, - "useMonitor": true - }, - { - "name": "starter", - "type": "orchestrationClient", - "direction": "in" - } - ] -} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/azuredeploy_Connector_LumenThreatFeed_AzureFunction.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/azuredeploy_Connector_LumenThreatFeed_AzureFunction.json deleted file mode 100644 index 21f869f6dd6..00000000000 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/azuredeploy_Connector_LumenThreatFeed_AzureFunction.json +++ /dev/null @@ -1,218 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "FunctionName": { - "type": "string", - "defaultValue": "LumenFeed", - "minLength": 1, - "maxLength": 24, - "metadata": { - "description": "Name prefix (11 character limit) for the Function App (will be lowercased and uniqued)." - } - }, - "WORKSPACE_ID": { - "type": "string", - "metadata": { - "description": "Microsoft Sentinel Workspace ID" - } - }, - "LUMEN_API_KEY": { - "type": "securestring", - "metadata": { - "description": "Lumen API Key" - } - }, - "LUMEN_BASE_URL": { - "type": "string", - "defaultValue": "https://microsoft-sentinel-api.us1.mss.lumen.com/v2/reputation-query", - "metadata": { - "description": "Lumen API base URL" - } - }, - "TENANT_ID": { - "type": "string", - "metadata": { - "description": "Azure Tenant ID" - } - }, - "CLIENT_ID": { - "type": "string", - "metadata": { - "description": "Azure App Registration Client ID" - } - }, - "CLIENT_SECRET": { - "type": "securestring", - "metadata": { - "description": "Azure App Registration Client Secret" - } - }, - "AppInsightsWorkspaceResourceID": { - "type": "string", - "metadata": { - "description": "(Optional) Log Analytics Workspace Resource ID for Application Insights linkage. If empty, classic (standalone) App Insights is created." - }, - "defaultValue": "" - }, - "BlobContainerName": { - "type": "string", - "defaultValue": "lumenthreatfeed", - "metadata": { - "description": "Blob container name to store staged indicator JSONL files." - } - } - }, - "variables": { - "uniqueFuncName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", - "storageAccountName": "[toLower(variables('uniqueFuncName'))]", - "appInsightsName": "[variables('uniqueFuncName')]", - "hostingPlanName": "[variables('uniqueFuncName')]", - "storageApiVersion": "2023-01-01", - "insightsApiVersion": "2020-02-02" - }, - "resources": [ - { - "type": "Microsoft.Insights/components", - "apiVersion": "[variables('insightsApiVersion')]", - "name": "[variables('appInsightsName')]", - "location": "[resourceGroup().location]", - "kind": "web", - "properties": { - "Application_Type": "web", - "ApplicationId": "[variables('appInsightsName')]", - "WorkspaceResourceId": "[if(equals(parameters('AppInsightsWorkspaceResourceID'), ''), json('null'), parameters('AppInsightsWorkspaceResourceID'))]" - } - }, - { - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "[variables('storageApiVersion')]", - "name": "[variables('storageAccountName')]", - "location": "[resourceGroup().location]", - "sku": { - "name": "Standard_LRS" - }, - "kind": "StorageV2", - "properties": { - "supportsHttpsTrafficOnly": true, - "encryption": { - "services": { - "file": { "keyType": "Account", "enabled": true }, - "blob": { "keyType": "Account", "enabled": true } - }, - "keySource": "Microsoft.Storage" - } - } - }, - { - "type": "Microsoft.Storage/storageAccounts/blobServices", - "apiVersion": "[variables('storageApiVersion')]", - "name": "[concat(variables('storageAccountName'), '/default')]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]" - ], - "properties": {} - }, - { - "type": "Microsoft.Storage/storageAccounts/blobServices/containers", - "apiVersion": "[variables('storageApiVersion')]", - "name": "[concat(variables('storageAccountName'), '/default/', parameters('BlobContainerName'))]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storageAccountName'), 'default')]" - ], - "properties": { - "publicAccess": "None" - } - }, - { - "type": "Microsoft.Web/serverfarms", - "apiVersion": "2024-04-01", - "name": "[variables('hostingPlanName')]", - "location": "[resourceGroup().location]", - "sku": { - "name": "Y1", - "tier": "Dynamic" - }, - "kind": "functionapp", - "properties": { - "computeMode": "Dynamic", - "reserved": true - } - }, - { - "type": "Microsoft.Web/sites", - "apiVersion": "2024-04-01", - "name": "[variables('uniqueFuncName')]", - "location": "[resourceGroup().location]", - "kind": "functionapp,linux", - "identity": { - "type": "SystemAssigned" - }, - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]", - "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]", - "[resourceId('Microsoft.Insights/components', variables('appInsightsName'))]" - ], - "properties": { - "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]", - "httpsOnly": true, - "reserved": true, - "siteConfig": { - "minTlsVersion": "1.2", - "linuxFxVersion": "Python|3.11", - "functionAppScaleLimit": 200, - "appSettings": [ - { "name": "FUNCTIONS_EXTENSION_VERSION", "value": "~4" }, - { "name": "FUNCTIONS_WORKER_RUNTIME", "value": "python" }, - { "name": "AzureWebJobsStorage", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), variables('storageApiVersion')).keys[0].value, ';EndpointSuffix=', environment().suffixes.storage)]" }, - { "name": "WEBSITE_RUN_FROM_PACKAGE", "value": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Solutions/Lumen%20Defender%20Threat%20Feed/Data%20Connectors/LumenThreatFeed/LumenThreatFeedConnector.zip" }, - { "name": "LUMEN_API_KEY", "value": "[parameters('LUMEN_API_KEY')]" }, - { "name": "LUMEN_BASE_URL", "value": "[parameters('LUMEN_BASE_URL')]" }, - { "name": "WORKSPACE_ID", "value": "[parameters('WORKSPACE_ID')]" }, - { "name": "TENANT_ID", "value": "[parameters('TENANT_ID')]" }, - { "name": "CLIENT_ID", "value": "[parameters('CLIENT_ID')]" }, - { "name": "CLIENT_SECRET", "value": "[parameters('CLIENT_SECRET')]" }, - { "name": "LUMEN_BLOB_CONTAINER", "value": "[parameters('BlobContainerName')]" }, - { "name": "LUMEN_BLOB_CONNECTION_STRING", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), variables('storageApiVersion')).keys[0].value, ';EndpointSuffix=', environment().suffixes.storage)]" }, - { "name": "LUMEN_ENABLE_IPV4", "value": "true" }, - { "name": "LUMEN_ENABLE_DOMAIN", "value": "true" }, - { "name": "APPINSIGHTS_INSTRUMENTATIONKEY", "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')), '2020-02-02').InstrumentationKey]" }, - { "name": "APPLICATIONINSIGHTS_CONNECTION_STRING", "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsightsName')), '2020-02-02').ConnectionString]" }, - { "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), variables('storageApiVersion')).keys[0].value, ';EndpointSuffix=', environment().suffixes.storage)]" }, - { "name": "WEBSITE_CONTENTSHARE", "value": "[toLower(variables('uniqueFuncName'))]" } - ] - } - } - } - ], - "outputs": { - "FunctionAppName": { - "type": "string", - "value": "[variables('uniqueFuncName')]", - "metadata": { - "description": "The name of the deployed Function App" - } - }, - "StorageAccountName": { - "type": "string", - "value": "[variables('storageAccountName')]", - "metadata": { - "description": "The name of the storage account used for blob storage" - } - }, - "BlobContainerName": { - "type": "string", - "value": "[parameters('BlobContainerName')]", - "metadata": { - "description": "The name of the blob container for indicator staging" - } - }, - "FunctionAppUrl": { - "type": "string", - "value": "[concat('https://', variables('uniqueFuncName'), '.azurewebsites.net')]", - "metadata": { - "description": "The URL of the deployed Function App" - } - } - } -} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/README.md b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/README.md new file mode 100644 index 00000000000..98f79214af3 --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/README.md @@ -0,0 +1,83 @@ +# Lumen Threat Feed Connector V2 - Function Code + +## Overview + +This Azure Function implements the Lumen Threat Feed Connector V2 for Microsoft Sentinel. It uses a timer-triggered function that retrieves threat intelligence indicators from the Lumen Threat Feed API and uploads them to Microsoft Sentinel. + +**Key Features:** + +- Timer-triggered function using Azure Functions V2 programming model +- Paginated Lumen API v3 with direct page-by-page processing +- Confidence threshold and indicator type filtering +- Automatic retry with exponential backoff for rate limiting +- Batch uploads to Sentinel TI API (100 indicators per batch) +- 15-minute sync intervals + +## File Structure + +| File | Purpose | +|------|---------| +| `function_app.py` | Azure Functions V2 entry point with timer trigger decorator | +| `main.py` | Core classes: MSALSetup, SentinelUploader, LumenClientV2 | +| `requirements.txt` | Python dependencies | +| `host.json` | Azure Functions host configuration | +| `__init__.py` | Package initialization | + +## Environment Variables + +### Required + +| Variable | Description | +|----------|-------------| +| `LUMEN_API_KEY` | Lumen API key for authentication | +| `WORKSPACE_ID` | Microsoft Sentinel workspace ID | +| `TENANT_ID` | Azure AD tenant ID | +| `CLIENT_ID` | Azure app registration client ID | +| `CLIENT_SECRET` | Azure app registration client secret | + +### Optional + +| Variable | Description | Default | +|----------|-------------|---------| +| `LUMEN_BASE_URL` | Lumen API base URL | `https://microsoft-sentinel-api.us1.mss.lumen.com` | +| `LUMEN_CONFIDENCE_THRESHOLD` | Minimum confidence score (0-100) | `65` | +| `LUMEN_ENABLE_IPV4` | Enable IPv4 address indicators | `true` | +| `LUMEN_ENABLE_DOMAIN` | Enable domain name indicators | `true` | +| `LUMEN_POLL_INTERVAL` | Seconds between status polls | `5` | +| `LUMEN_POLL_TIMEOUT` | Max seconds to wait for query completion | `300` | + +## Retry Configuration + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `DEFAULT_MAX_RETRIES` | 5 | Maximum retry attempts | +| `DEFAULT_BASE_DELAY` | 1.0s | Initial delay between retries | +| `DEFAULT_MAX_DELAY` | 60.0s | Maximum delay cap | +| `CHUNK_SIZE` | 100 | Sentinel API batch limit | +| `POLL_INTERVAL` | 5s | Query status poll interval | +| `POLL_TIMEOUT` | 300s | Query completion timeout | +| `MAX_PAGES` | 1000 | Pagination safeguard limit | + +## Troubleshooting + +### Deployment Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **403 Forbidden during deployment** | Storage account permissions not propagated | Wait 2-3 minutes and redeploy. The ARM template includes a wait, but RBAC propagation can take longer. | +| **Storage account name already exists** | Storage account names are globally unique | Change the `FunctionName` parameter to use a unique prefix | +| **Invalid App Insights Resource ID** | Incorrect resource ID format | Ensure the full resource path is provided, starting with `/subscriptions/` | + +### Runtime Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **403 Forbidden during sync** | Lumen API key invalid or incorrect | Verify your Lumen API key in Function App Configuration > Application Settings | +| **No indicators appearing in Sentinel** | API key invalid or expired | Verify your Lumen API key in Function App Configuration > Application Settings | +| **No indicators appearing in Sentinel** | App registration permissions | Verify the app has **Microsoft Sentinel Contributor** role assigned in the proper Log Analytics Workspace | +| **Function not triggering** | Timer schedule misconfigured | Check the timer trigger schedule in `function_app.py`; verify function is enabled | +| **429 Too Many Requests errors** | Sentinel API rate limiting | This is handled automatically with exponential backoff. If persistent, check logs for details | +| **Query timeout errors** | Lumen API taking too long | Increase `LUMEN_POLL_TIMEOUT` environment variable; contact Lumen support if persistent | +| **Missing environment variables** | Configuration not set | Check Application Settings in Azure portal; all required variables must be set | + +Note: The timer trigger runs every 15 minutes by default. For testing, you can manually trigger the function or modify the schedule. diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/__init__.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/__init__.py new file mode 100644 index 00000000000..d63075e09dd --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/__init__.py @@ -0,0 +1,2 @@ +# Lumen Threat Feed Connector V2 +# This file makes the directory a Python package for relative imports. diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/function_app.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/function_app.py new file mode 100644 index 00000000000..300efdd77f2 --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/function_app.py @@ -0,0 +1,269 @@ +"""Lumen Threat Feed Connector V2 - Azure Functions V2 Programming Model. + +This module provides the Azure Functions timer trigger using the V2 programming +model (decorators). +""" + +import azure.functions as func +import logging +import os +import sys + +# Add current directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from main import ( + MSALSetup, + SentinelUploader, + LumenClientV2, + LumenAPIError, + chunks, + CHUNK_SIZE, + MAX_PAGES, + DEFAULT_CONFIDENCE_THRESHOLD, + POLL_INTERVAL, + POLL_TIMEOUT, +) + +# Create the Function App +app = func.FunctionApp() + +logger = logging.getLogger(__name__) + + +def _parse_bool(value: str) -> bool: + """Parse string 'true'/'false' to boolean.""" + return value.lower() == 'true' + + +def _get_confidence(indicator: dict) -> int: + """Safely extract confidence value as integer. + + API v3 may return confidence as string (e.g., "75") or integer. + Returns 0 if confidence is missing, invalid, or cannot be converted. + """ + confidence = indicator.get('confidence') + if confidence is None: + return 0 + try: + return int(confidence) + except (ValueError, TypeError): + return 0 + + +def _filter_by_type(indicator: dict, enable_ipv4: bool, enable_domain: bool) -> bool: + """Check if indicator type is enabled based on its pattern.""" + pattern = indicator.get('pattern', '') + if 'ipv4-addr' in pattern: + return enable_ipv4 + elif 'domain-name' in pattern: + return enable_domain + return True + + +def run_sync( + lumen_client: LumenClientV2, + uploader: SentinelUploader, + confidence_threshold: int, + enable_ipv4: bool, + enable_domain: bool, + poll_interval: int, + poll_timeout: int +) -> dict: + """Execute the synchronization workflow.""" + logger.info("Starting Lumen Threat Feed sync") + + # Step 1: Initiate query + logger.info("Step 1: Initiating query") + cache_id = lumen_client.initiate_query() + logger.info("Query initiated, cache_id: %s", cache_id[:20] + "...") + + # Step 2: Poll for completion + logger.info("Step 2: Polling for completion") + token_id, inline_results = lumen_client.poll_status(cache_id, poll_interval, poll_timeout) + + # Handle case where API returns empty results inline (no pagination needed) + if token_id is None: + if inline_results is not None: + if len(inline_results) == 0: + logger.info("Query completed with no results - nothing to process") + return { + 'total_fetched': 0, + 'total_filtered': 0, + 'total_uploaded': 0, + 'upload_errors': 0, + 'page_count': 0 + } + else: + # Unexpected: inline results with data - process them directly + logger.warning("Processing %d inline results (unexpected)", len(inline_results)) + # Filter and upload inline results + filtered = [ + ind for ind in inline_results + if _get_confidence(ind) >= confidence_threshold + ] + filtered = [ + ind for ind in filtered + if _filter_by_type(ind, enable_ipv4, enable_domain) + ] + total_uploaded = 0 + upload_errors = 0 + for batch in chunks(filtered, CHUNK_SIZE): + result = uploader.upload_indicators(batch) + total_uploaded += result.get('uploaded_count', 0) + upload_errors += result.get('error_count', 0) + return { + 'total_fetched': len(inline_results), + 'total_filtered': len(filtered), + 'total_uploaded': total_uploaded, + 'upload_errors': upload_errors, + 'page_count': 0 + } + else: + logger.info("Query completed with no token_id and no results") + return { + 'total_fetched': 0, + 'total_filtered': 0, + 'total_uploaded': 0, + 'upload_errors': 0, + 'page_count': 0 + } + + logger.info("Query completed, token_id: %s", token_id[:20] + "...") + + # Step 3: Process pages + logger.info("Step 3: Processing pages") + total_fetched = 0 + total_filtered = 0 + total_uploaded = 0 + upload_errors = 0 + page_count = 0 + current_token = token_id + + while current_token and page_count < MAX_PAGES: + page_count += 1 + logger.info("Processing page %d", page_count) + + indicators, next_token = lumen_client.retrieve_page(current_token) + total_fetched += len(indicators) + + if not indicators: + logger.info("Page %d: No indicators", page_count) + current_token = next_token + continue + + # Filter by confidence (API v3 may return confidence as string) + filtered = [ + ind for ind in indicators + if _get_confidence(ind) >= confidence_threshold + ] + + # Filter by type + filtered = [ + ind for ind in filtered + if _filter_by_type(ind, enable_ipv4, enable_domain) + ] + + total_filtered += len(filtered) + logger.info("Page %d: %d fetched, %d after filtering", + page_count, len(indicators), len(filtered)) + + # Upload in batches + for batch in chunks(filtered, CHUNK_SIZE): + result = uploader.upload_indicators(batch) + total_uploaded += result.get('uploaded_count', 0) + upload_errors += result.get('error_count', 0) + + current_token = next_token + + if page_count >= MAX_PAGES and current_token: + logger.warning("Reached max pages limit (%d)", MAX_PAGES) + + stats = { + 'total_fetched': total_fetched, + 'total_filtered': total_filtered, + 'total_uploaded': total_uploaded, + 'upload_errors': upload_errors, + 'page_count': page_count + } + + logger.info("Sync complete: %s", stats) + return stats + + +@app.timer_trigger(schedule="0 */15 * * * *", arg_name="timer", run_on_startup=False) +def timer_function(timer: func.TimerRequest) -> None: + """Timer trigger function for Lumen Threat Feed sync.""" + logger.info("Lumen Threat Feed V2 timer triggered") + + if timer.past_due: + logger.warning("Timer is past due") + + try: + # Load required environment variables + api_key = os.environ.get('LUMEN_API_KEY') + workspace_id = os.environ.get('WORKSPACE_ID') + tenant_id = os.environ.get('TENANT_ID') + client_id = os.environ.get('CLIENT_ID') + client_secret = os.environ.get('CLIENT_SECRET') + + # Validate required variables + missing = [] + if not api_key: + missing.append('LUMEN_API_KEY') + if not workspace_id: + missing.append('WORKSPACE_ID') + if not tenant_id: + missing.append('TENANT_ID') + if not client_id: + missing.append('CLIENT_ID') + if not client_secret: + missing.append('CLIENT_SECRET') + + if missing: + raise ValueError(f"Missing required environment variables: {', '.join(missing)}") + + # Load optional environment variables + base_url = os.environ.get( + 'LUMEN_BASE_URL', + 'https://microsoft-sentinel-api.us1.mss.lumen.com' + ) + confidence_threshold = int(os.environ.get( + 'LUMEN_CONFIDENCE_THRESHOLD', + str(DEFAULT_CONFIDENCE_THRESHOLD) + )) + enable_ipv4 = _parse_bool(os.environ.get('LUMEN_ENABLE_IPV4', 'true')) + enable_domain = _parse_bool(os.environ.get('LUMEN_ENABLE_DOMAIN', 'true')) + poll_interval = int(os.environ.get('LUMEN_POLL_INTERVAL', str(POLL_INTERVAL))) + poll_timeout = int(os.environ.get('LUMEN_POLL_TIMEOUT', str(POLL_TIMEOUT))) + + # Initialize clients + lumen_client = LumenClientV2(api_key, base_url) + msal_setup = MSALSetup(tenant_id, client_id, client_secret, workspace_id) + uploader = SentinelUploader(msal_setup) + + # Run sync + stats = run_sync( + lumen_client=lumen_client, + uploader=uploader, + confidence_threshold=confidence_threshold, + enable_ipv4=enable_ipv4, + enable_domain=enable_domain, + poll_interval=poll_interval, + poll_timeout=poll_timeout + ) + + logger.info("Lumen Threat Feed V2 sync completed successfully: %s", stats) + + except LumenAPIError as e: + logger.error("Lumen API error: %s", str(e)) + raise + except TimeoutError as e: + logger.error("Timeout error: %s", str(e)) + raise + except ValueError as e: + logger.error("Configuration error: %s", str(e)) + raise + except Exception as e: + logger.error("Unexpected error: %s", str(e), exc_info=True) + raise diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/host.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/host.json similarity index 88% rename from Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/host.json rename to Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/host.json index d24b4d63cb2..06d01bdaa95 100644 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector/host.json +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/host.json @@ -11,6 +11,5 @@ "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", "version": "[4.*, 5.0.0)" - }, - "functionTimeout": "00:10:00" + } } diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/main.py b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/main.py new file mode 100644 index 00000000000..04d387fe5b4 --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/main.py @@ -0,0 +1,853 @@ +""" +Lumen Threat Feed Connector V2 - Simplified paginated API approach. + +This module provides core classes for Microsoft Sentinel integration: +- MSALSetup: Microsoft Authentication Library configuration +- SentinelUploader: Handles Sentinel TI API uploads with retry logic +- LumenClientV2: Handles Lumen API communication with retry logic + +V2 Architecture: +- Single timer-triggered function (no Durable Functions) +- Paginated Lumen API (no blob staging) +- Direct page-by-page processing and upload to Sentinel +- Robust retry logic for enterprise resilience + +Environment Variables Required: +- TENANT_ID: Azure tenant ID +- CLIENT_ID: Azure app registration client ID +- CLIENT_SECRET: Azure app registration client secret +- WORKSPACE_ID: Sentinel workspace ID +""" + +import base64 +import json +import logging +import time +from datetime import datetime, timedelta +from functools import wraps +from typing import Callable, Iterator, List, Optional, TypeVar + +import requests +from msal import ConfidentialClientApplication + +# Constants +CHUNK_SIZE = 100 # Sentinel API batch limit +DEFAULT_CONFIDENCE_THRESHOLD = 60 +POLL_INTERVAL = 5 # Seconds between status polls +POLL_TIMEOUT = 300 # Max seconds to wait for query completion +MAX_PAGES = 1000 # Safeguard against infinite pagination + +# Retry configuration +DEFAULT_MAX_RETRIES = 5 +DEFAULT_BASE_DELAY = 1.0 # seconds +DEFAULT_MAX_DELAY = 60.0 # seconds + +logger = logging.getLogger(__name__) + +T = TypeVar('T') + + +def retry_with_backoff( + max_retries: int = DEFAULT_MAX_RETRIES, + base_delay: float = DEFAULT_BASE_DELAY, + max_delay: float = DEFAULT_MAX_DELAY, + retryable_exceptions: tuple = (requests.exceptions.RequestException,), + retryable_status_codes: tuple = (429, 500, 502, 503, 504), +) -> Callable: + """Decorator that adds exponential backoff retry logic to a function. + + Retries on specified exceptions and HTTP status codes with exponential + backoff. Useful for handling transient network failures and rate limiting. + + Args: + max_retries: Maximum number of retry attempts (default: 5) + base_delay: Initial delay in seconds (default: 1.0) + max_delay: Maximum delay cap in seconds (default: 60.0) + retryable_exceptions: Tuple of exception types to retry on + retryable_status_codes: HTTP status codes that trigger a retry + + Returns: + Decorated function with retry logic + """ + def decorator(func: Callable[..., T]) -> Callable[..., T]: + @wraps(func) + def wrapper(*args, **kwargs) -> T: + last_exception = None + + for attempt in range(max_retries + 1): + try: + result = func(*args, **kwargs) + + # Check if result is a Response object with retryable status + if isinstance(result, requests.Response): + if result.status_code in retryable_status_codes: + if attempt < max_retries: + delay = _calculate_delay( + attempt, base_delay, max_delay, result + ) + logger.warning( + "%s returned HTTP %d on attempt %d/%d, " + "retrying in %.1fs", + func.__name__, result.status_code, + attempt + 1, max_retries + 1, delay + ) + time.sleep(delay) + continue + else: + # Max retries exhausted, let caller handle + result.raise_for_status() + + return result + + except retryable_exceptions as e: + last_exception = e + if attempt < max_retries: + delay = _calculate_delay(attempt, base_delay, max_delay) + logger.warning( + "%s failed on attempt %d/%d with %s: %s, " + "retrying in %.1fs", + func.__name__, attempt + 1, max_retries + 1, + type(e).__name__, str(e), delay + ) + time.sleep(delay) + else: + logger.error( + "%s failed after %d attempts: %s", + func.__name__, max_retries + 1, str(e) + ) + raise + + # Should not reach here, but just in case + if last_exception: + raise last_exception + return result + + return wrapper + return decorator + + +def _calculate_delay( + attempt: int, + base_delay: float, + max_delay: float, + response: Optional[requests.Response] = None +) -> float: + """Calculate delay for retry attempt with optional Retry-After header. + + Args: + attempt: Current attempt number (0-indexed) + base_delay: Base delay in seconds + max_delay: Maximum delay cap + response: Optional response object to check for Retry-After header + + Returns: + float: Delay in seconds + """ + # Check for Retry-After header + if response is not None: + retry_after = response.headers.get('Retry-After') + if retry_after: + try: + return min(float(retry_after), max_delay) + except ValueError: + pass # Fall through to exponential backoff + + # Exponential backoff with jitter + delay = base_delay * (2 ** attempt) + return min(delay, max_delay) + + +def chunks(lst: List, n: int) -> Iterator[List]: + """Yield successive n-sized chunks from lst. + + Args: + lst: List to split into chunks + n: Maximum size of each chunk + + Yields: + List chunks of size n or smaller (for the final chunk) + """ + for i in range(0, len(lst), n): + yield lst[i:i + n] + + +class MSALSetup: + """Microsoft Authentication Library configuration for Sentinel API access. + + Handles MSAL ConfidentialClientApplication setup for client credentials flow. + + Attributes: + tenant_id: Azure AD tenant ID + client_id: Application (client) ID from Azure app registration + client_secret: Client secret for the app registration + workspace_id: Microsoft Sentinel workspace ID + authority: Azure AD authority URL + scope: OAuth2 scope for Sentinel API + app: MSAL ConfidentialClientApplication instance + """ + + def __init__(self, tenant_id: str, client_id: str, client_secret: str, workspace_id: str): + """Initialize MSAL configuration. + + Args: + tenant_id: Azure AD tenant ID + client_id: Application (client) ID from Azure app registration + client_secret: Client secret for the app registration + workspace_id: Microsoft Sentinel workspace ID + + Raises: + ValueError: If any required parameter is empty or None + """ + # Validate required parameters + required_params = { + 'TENANT_ID': tenant_id, + 'CLIENT_ID': client_id, + 'CLIENT_SECRET': client_secret, + 'WORKSPACE_ID': workspace_id + } + + for param_name, value in required_params.items(): + if not value: + raise ValueError(f"{param_name} is required but was not provided") + + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + self.workspace_id = workspace_id + self.authority = f"https://login.microsoftonline.com/{tenant_id}" + self.scope = ["https://management.azure.com/.default"] + + # Create MSAL ConfidentialClientApplication + self.app = ConfidentialClientApplication( + client_id=self.client_id, + client_credential=self.client_secret, + authority=self.authority + ) + + logger.info("MSALSetup initialized for workspace %s", self.workspace_id) + + +class SentinelUploader: + """Handles Sentinel Threat Intelligence API uploads with retry logic. + + Uploads STIX indicators to Microsoft Sentinel using the TI API. + Implements exponential backoff for rate limiting (HTTP 429) and + transient failures. + + Attributes: + msal_setup: MSALSetup instance for authentication + access_token: Cached access token + token_expiry: Token expiration timestamp + session: Requests session for connection pooling + """ + + # Sentinel TI API endpoint template + UPLOAD_URL_TEMPLATE = ( + "https://api.ti.sentinel.azure.com/workspaces/{workspace_id}/" + "threat-intelligence-stix-objects:upload" + ) + API_VERSION = "2024-02-01-preview" + + def __init__(self, msal_setup: MSALSetup): + """Initialize SentinelUploader with MSAL configuration. + + Args: + msal_setup: MSALSetup instance containing authentication config + """ + self.msal_setup = msal_setup + self.access_token = None + self.token_expiry = None + self.session = requests.Session() + self.session.headers.update({ + 'User-Agent': 'LumenSentinelConnectorV2/2.0', + 'Content-Type': 'application/json' + }) + + logger.info("SentinelUploader initialized") + + def _get_access_token(self, force_refresh: bool = False) -> str: + """Acquire or return cached access token using MSAL client credentials flow. + + Implements retry logic for transient AAD failures. Tokens are cached + for 50 minutes (typical token lifetime is 60 minutes). + + Args: + force_refresh: If True, ignore cached token and acquire new one + + Returns: + str: Valid access token for Sentinel API + + Raises: + Exception: If token acquisition fails after all retries + """ + now = datetime.utcnow() + + # Return cached token if still valid (unless force refresh) + if not force_refresh and self.access_token and self.token_expiry and now < self.token_expiry: + logger.debug("Using cached access token (expires in %s)", + self.token_expiry - now) + return self.access_token + + logger.info("Acquiring new access token via MSAL client credentials flow") + + # Retry token acquisition with exponential backoff + max_retries = DEFAULT_MAX_RETRIES + base_delay = DEFAULT_BASE_DELAY + last_error = None + + for attempt in range(max_retries + 1): + try: + result = self.msal_setup.app.acquire_token_for_client( + scopes=self.msal_setup.scope + ) + + if "access_token" in result: + self.access_token = result["access_token"] + # Cache token for 50 minutes (buffer before 60-minute expiry) + self.token_expiry = now + timedelta(minutes=50) + logger.info("Access token acquired successfully") + return self.access_token + else: + error_msg = result.get("error_description", result.get("error", "Unknown error")) + last_error = f"Token acquisition failed: {error_msg}" + + # Check if error is retryable (transient) + error_code = result.get("error", "") + retryable_errors = [ + "temporarily_unavailable", + "service_unavailable", + "connection_error" + ] + + if any(err in error_code.lower() for err in retryable_errors): + if attempt < max_retries: + delay = min(base_delay * (2 ** attempt), DEFAULT_MAX_DELAY) + logger.warning( + "Token acquisition failed on attempt %d/%d: %s, " + "retrying in %.1fs", + attempt + 1, max_retries + 1, error_msg, delay + ) + time.sleep(delay) + continue + + # Non-retryable error or max retries exceeded + logger.error("Failed to acquire access token: %s", error_msg) + raise Exception(f"Failed to acquire access token: {error_msg}") + + except Exception as e: + last_error = str(e) + if attempt < max_retries: + delay = min(base_delay * (2 ** attempt), DEFAULT_MAX_DELAY) + logger.warning( + "Token acquisition error on attempt %d/%d: %s, " + "retrying in %.1fs", + attempt + 1, max_retries + 1, str(e), delay + ) + time.sleep(delay) + else: + logger.error("Token acquisition failed after %d attempts: %s", + max_retries + 1, str(e)) + raise + + raise Exception(f"Failed to acquire access token after {max_retries + 1} attempts: {last_error}") + + def upload_indicators(self, stix_objects: list) -> dict: + """Upload STIX indicators to Microsoft Sentinel. + + Uploads indicators in batches of CHUNK_SIZE (100) to comply with + Sentinel API limits. Implements exponential backoff for rate limiting + and automatic token refresh on 401 errors. + + Args: + stix_objects: List of STIX indicator objects to upload + + Returns: + dict: Upload results containing: + - uploaded_count: Number of successfully uploaded indicators + - error_count: Number of failed indicators + - rate_limit_events: Number of 429 responses encountered + - success: Overall success status (True if no errors) + """ + if not stix_objects: + logger.info("No indicators to upload") + return { + 'uploaded_count': 0, + 'error_count': 0, + 'rate_limit_events': 0, + 'success': True + } + + # Get access token + access_token = self._get_access_token() + + # Prepare upload URL + upload_url = self.UPLOAD_URL_TEMPLATE.format( + workspace_id=self.msal_setup.workspace_id + ) + + params = {'api-version': self.API_VERSION} + + # Statistics + uploaded_count = 0 + error_count = 0 + rate_limit_events = 0 + + # Process in chunks + indicator_chunks = list(chunks(stix_objects, CHUNK_SIZE)) + total_chunks = len(indicator_chunks) + + logger.info("Uploading %d indicators in %d batches", + len(stix_objects), total_chunks) + + for chunk_num, chunk in enumerate(indicator_chunks, 1): + payload = { + 'sourcesystem': 'Lumen', + 'stixobjects': chunk + } + + # Retry configuration + max_retries = 6 + base_delay = 1.0 + token_refreshed = False + + for attempt in range(max_retries): + try: + headers = { + 'Authorization': f'Bearer {access_token}', + 'Content-Type': 'application/json' + } + + logger.info("Uploading batch %d/%d (%d indicators, attempt %d)", + chunk_num, total_chunks, len(chunk), attempt + 1) + + response = self.session.post( + upload_url, + json=payload, + headers=headers, + params=params, + timeout=60 + ) + + if response.status_code == 200: + uploaded_count += len(chunk) + logger.info("Batch %d/%d uploaded successfully (%d indicators)", + chunk_num, total_chunks, len(chunk)) + break + + elif response.status_code == 401 and not token_refreshed: + # Token expired, refresh and retry + logger.warning("Received 401, refreshing access token") + access_token = self._get_access_token(force_refresh=True) + token_refreshed = True + continue + + elif response.status_code == 429: + # Rate limiting - apply exponential backoff + rate_limit_events += 1 + + if attempt < max_retries - 1: + delay = _calculate_delay(attempt, base_delay, DEFAULT_MAX_DELAY, response) + logger.warning( + "Rate limited on batch %d/%d (attempt %d), " + "retrying in %.1fs", + chunk_num, total_chunks, attempt + 1, delay + ) + time.sleep(delay) + continue + else: + logger.error( + "Batch %d/%d failed after %d rate limit retries", + chunk_num, total_chunks, max_retries + ) + error_count += len(chunk) + break + + elif response.status_code in (500, 502, 503, 504): + # Server error - retry with backoff + if attempt < max_retries - 1: + delay = min(base_delay * (2 ** attempt), DEFAULT_MAX_DELAY) + logger.warning( + "Server error %d on batch %d/%d (attempt %d), " + "retrying in %.1fs", + response.status_code, chunk_num, total_chunks, + attempt + 1, delay + ) + time.sleep(delay) + continue + else: + error_text = response.text[:500] if response.text else "No response body" + logger.error( + "Batch %d/%d failed with HTTP %d after %d retries: %s", + chunk_num, total_chunks, response.status_code, + max_retries, error_text + ) + error_count += len(chunk) + break + else: + # Other HTTP errors - don't retry + error_text = response.text[:500] if response.text else "No response body" + logger.error( + "Batch %d/%d failed with HTTP %d: %s", + chunk_num, total_chunks, response.status_code, error_text + ) + error_count += len(chunk) + break + + except requests.exceptions.RequestException as e: + logger.error( + "Batch %d/%d network error on attempt %d: %s", + chunk_num, total_chunks, attempt + 1, str(e) + ) + if attempt == max_retries - 1: + error_count += len(chunk) + else: + delay = min(base_delay * (2 ** attempt), DEFAULT_MAX_DELAY) + time.sleep(delay) + + # Log summary + success = error_count == 0 + logger.info( + "Upload complete: %d uploaded, %d errors, %d rate limit events", + uploaded_count, error_count, rate_limit_events + ) + + return { + 'uploaded_count': uploaded_count, + 'error_count': error_count, + 'rate_limit_events': rate_limit_events, + 'success': success + } + + +class LumenAPIError(Exception): + """Custom exception for Lumen API errors.""" + pass + + +class LumenClientV2: + """V2 Lumen API client using paginated endpoints with retry logic. + + Handles communication with the Lumen Threat Feed API v2 for retrieving + threat intelligence data using a paginated query flow: + 1. Initiate query -> get cache_id + 2. Poll status -> get token_id when completed + 3. Retrieve pages -> iterate through paginated results + + All API calls include retry logic with exponential backoff for resilience + against transient network failures. + + Attributes: + api_key: Lumen API key for authentication + base_url: Base URL for Lumen API endpoints + session: Requests session with pre-configured headers + max_retries: Maximum retry attempts for API calls + """ + + def __init__(self, api_key: str, base_url: str, max_retries: int = DEFAULT_MAX_RETRIES): + """Initialize LumenClientV2 with API credentials. + + Args: + api_key: Lumen API key for authentication + base_url: Base URL for Lumen API (trailing slash will be stripped) + max_retries: Maximum retry attempts for transient failures + """ + self.api_key = api_key + self.base_url = base_url.rstrip('/') + self.max_retries = max_retries + self.session = requests.Session() + self.session.headers.update({ + 'Authorization': api_key, + 'Content-Type': 'application/json', + 'User-Agent': 'LumenSentinelConnectorV2/2.0' + }) + + logger.info("LumenClientV2 initialized for %s", self.base_url) + + def _request_with_retry( + self, + method: str, + url: str, + **kwargs + ) -> requests.Response: + """Make HTTP request with retry logic. + + Args: + method: HTTP method (GET, POST, etc.) + url: Request URL + **kwargs: Additional arguments passed to requests + + Returns: + requests.Response: Successful response + + Raises: + requests.exceptions.HTTPError: If request fails after all retries + """ + last_exception = None + + for attempt in range(self.max_retries + 1): + try: + if method.upper() == 'GET': + response = self.session.get(url, **kwargs) + elif method.upper() == 'POST': + response = self.session.post(url, **kwargs) + else: + raise ValueError(f"Unsupported HTTP method: {method}") + + # Check for retryable status codes + if response.status_code in (429, 500, 502, 503, 504): + if attempt < self.max_retries: + delay = _calculate_delay( + attempt, DEFAULT_BASE_DELAY, DEFAULT_MAX_DELAY, response + ) + logger.warning( + "Request to %s returned HTTP %d on attempt %d/%d, " + "retrying in %.1fs", + url, response.status_code, + attempt + 1, self.max_retries + 1, delay + ) + time.sleep(delay) + continue + + # Raise for other error status codes + if response.status_code >= 400: + # Log response body for debugging before raising + error_body = response.text[:1000] if response.text else "No response body" + logger.warning( + "HTTP %d error from %s - Response body: %s", + response.status_code, url, error_body + ) + response.raise_for_status() + return response + + except requests.exceptions.HTTPError as e: + # Don't retry client errors (4xx) except 429 (rate limit) + # These are handled in the status_code checks above, so if we get here + # it's a non-retryable HTTP error + logger.error( + "Request to %s failed with HTTP error: %s", + url, str(e) + ) + raise + + except requests.exceptions.RequestException as e: + # Retry network/connection errors (not HTTP errors) + last_exception = e + if attempt < self.max_retries: + delay = min(DEFAULT_BASE_DELAY * (2 ** attempt), DEFAULT_MAX_DELAY) + logger.warning( + "Request to %s failed on attempt %d/%d: %s, " + "retrying in %.1fs", + url, attempt + 1, self.max_retries + 1, str(e), delay + ) + time.sleep(delay) + else: + logger.error( + "Request to %s failed after %d attempts: %s", + url, self.max_retries + 1, str(e) + ) + raise + + # Should not reach here + if last_exception: + raise last_exception + + def initiate_query(self) -> str: + """Initiate a reputation query to start data retrieval. + + POST /v3/reputation-query -> cache_id + + Includes retry logic for transient failures. + + Returns: + str: The cache_id to use for polling query status + + Raises: + requests.exceptions.HTTPError: If the API request fails after retries + KeyError: If response is missing cache_id + """ + url = f"{self.base_url}/v3/reputation-query" + + logger.info("Initiating reputation query at %s", url) + + response = self._request_with_retry('POST', url, timeout=60) + data = response.json() + cache_id = data['cache_id'] + + logger.info("Query initiated successfully, cache_id: %s", cache_id) + + return cache_id + + def poll_status(self, cache_id: str, poll_interval: int = POLL_INTERVAL, + poll_timeout: int = POLL_TIMEOUT) -> tuple[str | None, list | None]: + """Poll query status until completion or timeout. + + GET /v3/reputation-query/query-status/{cache_id} -> token_id or inline results + + Polls the query status endpoint in a loop, waiting for the query + to complete. Each individual poll request has retry logic. + + When the query completes with results, returns a token_id for pagination. + When the query completes with no results, the API returns inline empty results: + {"status": "COMPLETED", "results": {"sourcesystem": "Lumen", "stixobjects": []}} + + Args: + cache_id: The cache_id returned from initiate_query() + poll_interval: Seconds to wait between status checks (default: POLL_INTERVAL) + poll_timeout: Maximum seconds to wait for completion (default: POLL_TIMEOUT) + + Returns: + tuple: A tuple containing: + - str | None: The token_id for retrieving paginated results, or None if no results + - list | None: Inline results (empty list) when no results found, or None if token_id is returned + + Raises: + LumenAPIError: If query fails or expires + TimeoutError: If poll_timeout is exceeded + requests.exceptions.HTTPError: If the API request fails after retries + """ + url = f"{self.base_url}/v3/reputation-query/query-status/{cache_id}" + start_time = time.time() + + logger.info("Polling query status for cache_id: %s", cache_id) + + while True: + elapsed = time.time() - start_time + + if elapsed > poll_timeout: + logger.error("Query polling timed out after %d seconds", poll_timeout) + raise TimeoutError(f"Query polling timed out after {poll_timeout} seconds") + + logger.debug("Polling status (elapsed: %.1fs)", elapsed) + + try: + response = self._request_with_retry('GET', url, timeout=60) + data = response.json() + status = data.get('status') + + logger.debug("Query status: %s", status) + + if status == 'COMPLETED': + # API v3 always returns token_id for pagination, even for empty results + # (empty results are uploaded as an empty chunk file with total_chunks=1) + token_id = data.get('token_id') + if token_id: + logger.info("Query completed successfully, token_id: %s", token_id) + return (token_id, None) + + # Fallback: check for inline results (legacy API behavior) + # API v3 doesn't return inline results, but kept for backward compatibility + if 'results' in data: + stix_objects = data.get('results', {}).get('stixobjects', []) + if not stix_objects: + logger.info("Query completed with no results (empty stixobjects)") + return (None, []) + else: + logger.warning( + "Query completed with %d inline results (unexpected), " + "returning inline data", + len(stix_objects) + ) + return (None, stix_objects) + + # No token_id and no results - treat as empty + logger.info("Query completed with no token_id and no results") + return (None, []) + + elif status in ('PENDING', 'RUNNING', 'QUEUED'): + # API v3 uses QUEUED instead of PENDING, but we handle both for compatibility + logger.debug("Query %s, waiting %d seconds", status.lower(), poll_interval) + time.sleep(poll_interval) + continue + + elif status == 'FAILED': + error_msg = data.get('error', 'Unknown error') + logger.error("Query failed: %s", error_msg) + raise LumenAPIError(f"Query failed: {error_msg}") + + elif status == 'EXPIRED': + # Note: API v3 no longer uses EXPIRED status, kept for backward compatibility + logger.warning("Query expired for cache_id: %s", cache_id) + raise LumenAPIError("Query expired") + + else: + # Unknown status - log and continue polling + logger.warning("Unknown query status: %s, continuing to poll", status) + time.sleep(poll_interval) + + except (LumenAPIError, TimeoutError): + # Re-raise these specific exceptions + raise + except Exception as e: + # For other exceptions during polling, log and continue + # (the retry logic in _request_with_retry handles transient failures) + logger.warning("Error during poll: %s, continuing to poll", str(e)) + time.sleep(poll_interval) + + def retrieve_page(self, token_id: str) -> tuple[list, str | None]: + """Retrieve a page of results using the token_id. + + GET /v3/reputation-query/retrieve-results/{token_id} -> (indicators, next_token) + + Includes retry logic for transient failures. Handles the case where the + API returns a 400 error indicating no results found for the token. + + Args: + token_id: The token_id for retrieving results (from poll_status or previous page) + + Returns: + tuple: A tuple containing: + - list: STIX objects from the current page (empty list if no results) + - str | None: Next token for pagination, or None if no more pages + + Raises: + requests.exceptions.HTTPError: If the API request fails after retries + (except for 400 "no results" which returns empty list) + """ + url = f"{self.base_url}/v3/reputation-query/retrieve-results/{token_id}" + + logger.debug("Retrieving results page with token_id: %s", token_id) + + try: + response = self._request_with_retry('GET', url, timeout=60) + data = response.json() + except requests.exceptions.HTTPError as e: + # Handle 400 "No results found" as empty results, not an error + if e.response is not None and e.response.status_code == 400: + try: + error_data = e.response.json() + error_message = error_data.get('message', '') + if 'No results found' in error_message: + logger.info( + "No results found for token_id (API returned 400): %s", + token_id + ) + return ([], None) + except (ValueError, KeyError): + pass # Could not parse response, re-raise original error + raise + + stix_objects = data.get('results', {}).get('stixobjects', []) + next_token = data.get('next_token') + + # Validate next_token - check if it points to a valid chunk + # The API may return a next_token even on the last page that's invalid + if next_token: + try: + token_data = json.loads(base64.b64decode(next_token).decode()) + next_index = token_data.get('index', 0) + total_chunks = token_data.get('total_chunks', 1) + if next_index >= total_chunks: + logger.debug( + "next_token points beyond available chunks (index=%d, total=%d), " + "treating as last page", + next_index, total_chunks + ) + next_token = None + except (json.JSONDecodeError, ValueError, KeyError) as e: + logger.debug("Could not validate next_token format: %s", e) + # Keep the token as-is if we can't parse it + + logger.debug("Retrieved %d STIX objects, next_token: %s", + len(stix_objects), next_token if next_token else "None") + + return (stix_objects, next_token) diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/requirements.txt b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/requirements.txt new file mode 100644 index 00000000000..f802c802f21 --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2/requirements.txt @@ -0,0 +1,3 @@ +azure-functions>=1.17.0 +msal>=1.24.0 +requests>=2.31.0 diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector_ConnectorUI.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2_ConnectorUI.json similarity index 51% rename from Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector_ConnectorUI.json rename to Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2_ConnectorUI.json index 01b56b1ed03..f876b43991e 100644 --- a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeed/LumenThreatFeedConnector_ConnectorUI.json +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2_ConnectorUI.json @@ -1,9 +1,9 @@ { - "id": "LumenThreatFeedConnector", - "title": "Lumen Defender Threat Feed Data Connector", + "id": "LumenThreatFeedConnectorV2", + "title": "Lumen Defender Threat Feed Data Connector V2", "publisher": "Lumen Technologies, Inc.", - "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://bll-analytics.mss.lumen.com/analytics) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads daily threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.", - "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, and proper configuration of Azure AD authentication components.", + "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://www.lumen.com/en-us/security/black-lotus-labs.html) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.\n\n**NOTE:** This data connector uses the [Azure Functions Flex Consumption Plan](https://learn.microsoft.com/azure/azure-functions/flex-consumption-plan). More pricing details are [here](https://azure.microsoft.com/pricing/details/functions/#pricing).", + "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, and proper configuration of Azure Entra ID authentication components.", "graphQueries": [ { "metricName": "Total Threat Intelligence Indicators", @@ -87,7 +87,11 @@ "instructionSteps": [ { "title": "", - "description": ">**NOTE:** This connector uses Azure Functions with Durable Functions to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + "description": ">**NOTE:** This connector uses Azure Functions with the Flex Consumption Plan to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + }, + { + "title": "", + "description": ">**(Optional Step)** Securely store API keys and secrets in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Functions App." }, { "title": "Configuration", @@ -95,7 +99,7 @@ }, { "title": "", - "description": "**STEP 2 - Configure Azure Entra ID Application and gather information**\n\n1. Create an Entra application. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the Application ID, Tenant ID, and Client Secret\n4. Assign the **Microsoft Sentinel Contributor** role to the application on your Microsoft Sentinel Log Analytics Workspace\n5. Make note of your Workspace ID, as well as the App Insights Workspace Resource ID, which can be obtained from the overview page of the Log Analytics Workspace for your Microsoft Sentinel instance. Click on the “JSON View” link in the top right and the Resource ID will be displayed at the top with a copy button.", + "description": "**STEP 2 - Configure Azure Entra ID Application and gather information**\n\n1. Create a new Entra app registration from the **App registrations** tab in the Entra ID section of the Azure portal. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the **Application ID**, **Tenant ID**, and **Client Secret**\n3. Assign the **Microsoft Sentinel Contributor** role to the newly registered application in the **Access control (IAM)** menu of your Microsoft Sentinel Log Analytics Workspace\n4. Make note of your **Workspace ID**, which can be obtained from the **overview** page of the Log Analytics Workspace for your Microsoft Sentinel instance.", "instructions":[ { "parameters": { @@ -119,20 +123,20 @@ }, { "title": "", - "description": "**STEP 3 - Enable the Threat Intelligence Upload Indicators API (Preview) data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." + "description": "**STEP 3 - Enable the **Threat Intelligence Upload Indicators API (Preview)** data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." }, { "title": "", - "description": "**STEP 4 - Deploy the Azure Function**\n\n**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the Tenant ID, Workspace ID, App Insights Workspace Resource ID, Azure Entra application details (Client ID, Client Secret), and Lumen API key readily available.\n\n1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%20Defender%20Threat%20Feed%2FData%2520Connectors%2FLumenThreatFeed%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction.json)\n\n2. Fill in the appropriate values for each parameter:\n\n- Subscription: Confirm the correct subscription is selected or use the dropdown to change your selection\n- Resource Group: Select the resource group to be used by the Function App and related resources\n- Function Name: Enter a globally unique name with an 11-character limit. Adhere to your organization’s naming convention and ensure the name is globally unique since it is used (along with the uniqueString() function) to identify the ARM template being deployed.\n- Workspace ID: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance and provided for convenience on the connector information page.\n- Lumen API Key: Obtain an API key through Lumen support\n- Lumen Base URL: Filled in automatically and should generally not be changed. This URL contains API endpoints used by the connector\n- Tenant ID: Obtained from the Entra App Registration overview page for the registered application (listed as Directory ID) and can also be obtained from the Tenant Information page in Azure\n- Client ID: Obtained from the Entra App Registration overview page for the registered application (listed as Application ID)\n- Client Secret: Obtained when the secret is created during the app registration process. It can only be viewed when first created and is hidden permanently afterwards. Rerun the app registration process to obtain a new Client Secret if necessary.\n- App Insights Workspace Resource ID: Obtained from the overview page of the Log Analytics Workspace for your Microsoft Sentinel instance. Click on the \"JSON View\" link in the top right and the Resource ID will be displayed at the top with a copy button.\n- Blob Container Name: Use the default name unless otherwise required. Azure Blob Storage is used for temporary storage and processing of threat indicators." + "description": "**STEP 4 - Deploy the Azure Function**\n\n>**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the following information readily available:\n> - Tenant ID and Workspace ID\n> - Azure Entra application details (Client ID, Client Secret)\n> - Lumen API key\n>1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction_v2.json)\n\n2. Fill in the appropriate values for each parameter:\n\n**Basic Settings:**\n- **Subscription**: Confirm the correct subscription is selected or use the dropdown to change your selection\n- **Resource Group**: Select the resource group to be used by the Function App and related resources\n- **Function Name**: Enter a globally unique name for the Function App (11-character limit recommended)\n- **App Insights Workspace Resource ID**: The Resource ID of the Log Analytics Workspace for Application Insights (click **JSON View** on the Log Analytics workspace to copy)\n\n**Lumen API Settings:**\n- **Lumen API Key**: Obtain an API key through Lumen support\n- **Lumen Base URL**: Filled in automatically and should generally not be changed\n- **Confidence Threshold** (Optional): Minimum confidence score (60-100) for indicators (default: 60)\n- **Enable IPv4** (Optional): Enable IPv4 address indicators (default: true)\n- **Enable Domain** (Optional): Enable domain name indicators (default: true)\n\n**Azure Entra ID Settings:**\n- **Workspace ID**: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance\n- **Tenant ID**: Obtained from the Entra App Registration overview page (listed as Directory ID)\n- **Client ID**: Obtained from the Entra App Registration overview page (listed as Application ID)\n- **Client Secret**: Obtained when the secret is created during the app registration process" }, { "title": "", - "description": "**STEP 5 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. Monitor the Function App logs in the Azure Portal to verify successful execution\n3. After the app performs its first run, review the indicators ingested by either viewing the “Lumen Defender Threat Feed Overview” workbook or viewing the “Threat Intelligence” section in Microsoft Sentinel. In Microsoft Sentinel “Threat Intelligence”, filter for source “Lumen” to display only Lumen generated indicators." + "description": "**STEP 5 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. After the app performs its first run, review the indicators ingested by either viewing the \"Lumen Defender Threat Feed Overview\" workbook or viewing the \"Threat Intelligence\" section in Microsoft Sentinel. In Microsoft Sentinel \"Threat Intelligence\", filter for source \"Lumen\" to display only Lumen generated indicators." } ], "metadata": { - "id": "8d7d3b0e-4f6a-4f2a-bc2d-6d9f5a3c7e21", - "version": "1.0.0", + "id": "8d7d3b0e-4f6a-4f2a-bc2d-6d9f5a3c7e22", + "version": "2.0.0", "kind": "dataConnector", "source": { "kind": "solution", @@ -147,4 +151,3 @@ } } } - diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2_PrivateNetworking_ConnectorUI.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2_PrivateNetworking_ConnectorUI.json new file mode 100644 index 00000000000..06eee480f67 --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorV2_PrivateNetworking_ConnectorUI.json @@ -0,0 +1,166 @@ +{ + + "id": "LumenThreatFeedConnectorV2PrivateNetworking", + "title": "Lumen Defender Threat Feed Data Connector V2 (using Azure Functions Flex Consumption Plan with Private Networking)", + "publisher": "Lumen Technologies, Inc.", + "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://www.lumen.com/en-us/security/black-lotus-labs.html) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.\n\n**NOTE:** This data connector uses the [Azure Functions Flex Consumption Plan](https://learn.microsoft.com/azure/azure-functions/flex-consumption-plan) with VNet integration for secure, private network access to storage resources. More pricing details are [here](https://azure.microsoft.com/pricing/details/functions/#pricing).", + "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, proper configuration of Azure Entra ID authentication components, and Virtual Network configuration for private access.", + "graphQueries": [ + { + "metricName": "Total Threat Intelligence Indicators", + "legend": "ThreatIntelIndicators", + "baseQuery": "ThreatIntelIndicators | where SourceSystem == 'Lumen'" + } + ], + "sampleQueries": [ + { + "description": "All Lumen Threat Intelligence Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' | sort by TimeGenerated desc" + }, + { + "description": "High Confidence Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and Confidence >= 80 | sort by TimeGenerated desc" + }, + { + "description": "Malicious IP Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and ObservableValue != '' and ObservableKey == 'ipv4-addr:value' | sort by TimeGenerated desc" + }, + { + "description": "Malicious Domain Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and ObservableValue != '' and ObservableKey == 'domain-name:value' | sort by TimeGenerated desc" + }, + { + "description": "Recent Indicators (Last 7 Days)", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and TimeGenerated > ago(7d) | summarize count() by bin(TimeGenerated, 1d)" + } + ], + "dataTypes": [ + { + "name": "ThreatIntelIndicators(Lumen)", + "lastDataReceivedQuery": "ThreatIntelIndicators | where SourceSystem == 'Lumen' | summarize Time = max(TimeGenerated) | where isnotempty(Time)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "ThreatIntelIndicators | where SourceSystem == 'Lumen' | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "Read and write permissions on the Log Analytics workspace are required.", + "providerDisplayName": "Log Analytics Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": false + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "Azure Entra App Registration", + "description": "An Azure Entra application registration with the Microsoft Sentinel Contributor role assigned is required for STIX Objects API access. [See the documentation to learn more about Azure Entra applications](https://docs.microsoft.com/azure/active-directory/develop/quickstart-register-app)." + }, + { + "name": "Microsoft Sentinel Contributor Role", + "description": "Microsoft Sentinel Contributor role is required for the Azure Entra application to upload threat intelligence indicators." + }, + { + "name": "Lumen Defender Threat Feed API Key", + "description": "A Lumen Defender Threat Feed API Key is required for accessing threat intelligence data. [Contact Lumen for API access](mailto:DefenderThreatFeedSales@Lumen.com?subject=API%20Access%20Request)." + }, + { + "name": "Virtual Network permissions (for private access)", + "description": "For private storage account access, **Network Contributor** permissions are required on the Virtual Network and subnets. The Function App subnet must be delegated to **Microsoft.App/environments** for Flex Consumption VNet integration." + } + ] + }, + "instructionSteps": [ + { + "title": "", + "description": ">**NOTE:** This connector uses Azure Functions with the Flex Consumption Plan to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. The Flex Consumption Plan enables VNet integration for secure, private network access to storage resources. This might result in additional data ingestion and compute costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + }, + { + "title": "", + "description": ">**(Optional Step)** Securely store API keys and secrets in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Functions App." + }, + { + "title": "Configuration", + "description": "**STEP 1 - Network Prerequisites for Private Access**\n\n>**IMPORTANT:** When deploying with private storage account access, **you need a Virtual Network with two properly configured subnets.** You can either use an existing VNet or deploy one using the template below.\n\n**Option A: Deploy a New Virtual Network (Recommended for new deployments)**\n\nUse this template to create a properly configured VNet with two subnets:\n- **Function App Subnet**: Delegated to Microsoft.App/environments for Flex Consumption VNet integration\n- **Private Endpoint Subnet**: For storage account private endpoints\n\n[![Deploy VNet to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_VNet_for_PrivateEndpoint.json)\n\nAfter deployment, note the following output values for use in STEP 5:\n- **VNet Name** (default: lumen-threatfeed-vnet)\n- **VNet Resource Group**\n- **Function App Subnet Name** (default: functionapp-subnet)\n- **Private Endpoint Subnet Name** (default: privateendpoint-subnet)\n\n**Option B: Use an Existing Virtual Network**\n\nIf using an existing VNet, ensure the following requirements are met:\n> - **Virtual Network**: Must be in the same region where you plan to deploy the Function App\n> - **Function App Subnet**: Must be delegated to **Microsoft.App/environments** (required for Flex Consumption Plan)\n> - **Private Endpoint Subnet**: Must NOT be delegated to any service\n> - **Subnet Size**: Minimum /24 recommended for each subnet\n> - **Subnet Delegation**: Configure using one of the following methods:\n> - **Azure Portal**: Virtual networks → Select VNet → Subnets → Select subnet → Delegate to **Microsoft.App/environments**\n> - **Azure CLI**: `az network vnet subnet update --resource-group --vnet-name --name --delegations Microsoft.App/environments`\n\n>**Note:** The connector deployment will automatically create private endpoints for storage services (blob, queue, table, file) and configure Private DNS zones." + }, + { + "title": "", + "description": "**STEP 2 - Obtain Lumen Defender Threat Feed API Key**\n\n1. [Contact Lumen](mailto:DefenderThreatFeedSales@Lumen.com?subject=API%20Access%20Request) to obtain API access to our Threat Feed API service\n2. Obtain your API key for authentication." + }, + { + "title": "", + "description": "**STEP 3 - Configure Azure Entra ID Application and gather information**\n\n1. Create a new Entra app registration from the **App registrations** tab in the Entra ID section of the Azure portal. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the **Application ID**, **Tenant ID**, and **Client Secret**\n3. Assign the **Microsoft Sentinel Contributor** role to the newly registered application in the **Access control (IAM)** menu of your Microsoft Sentinel Log Analytics Workspace\n4. Make note of your **Workspace ID**, which can be obtained from the **overview** page of the Log Analytics Workspace for your Microsoft Sentinel instance.", + "instructions":[ + { + "parameters": { + "fillWith": [ + "TenantId" + ], + "label": "Tenant ID" + }, + "type": "CopyableLabel" + }, + { + "parameters": { + "fillWith": [ + "WorkspaceId" + ], + "label": "Workspace ID" + }, + "type": "CopyableLabel" + } + ] + }, + { + "title": "", + "description": "**STEP 4 - Enable the **Threat Intelligence Upload Indicators API (Preview)** data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." + }, + { + "title": "", + "description": "**STEP 5 - Deploy the Azure Function with Private Networking**\n\n>**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the following information readily available:\n> - Tenant ID and Workspace ID\n> - Azure Entra application details (Client ID, Client Secret)\n> - Lumen API key\n> - Virtual Network name and Resource Group\n> - Function App Subnet name (delegated to Microsoft.App/environments)\n> - Private Endpoint Subnet name (non-delegated)\n\n1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction_v2_privateendpoint.json)\n\n2. Fill in the appropriate values for each parameter:\n\n**Basic Settings:**\n- **Subscription**: Confirm the correct subscription is selected or use the dropdown to change your selection\n- **Resource Group**: Select the resource group to be used by the Function App and related resources\n- **Function Name**: Enter a globally unique name for the Function App (11-character limit recommended)\n- **App Insights Workspace Resource ID**: The Resource ID of the Log Analytics Workspace for Application Insights (click **JSON View** on the Log Analytics workspace to copy)\n\n**Lumen API Settings:**\n- **Lumen API Key**: Obtain an API key through Lumen support\n- **Lumen Base URL**: Filled in automatically and should generally not be changed\n- **Confidence Threshold** (Optional): Minimum confidence score (60-100) for indicators (default: 60)\n- **Enable IPv4** (Optional): Enable IPv4 address indicators (default: true)\n- **Enable Domain** (Optional): Enable domain name indicators (default: true)\n\n**Azure Entra ID Settings:**\n- **Workspace ID**: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance\n- **Tenant ID**: Obtained from the Entra App Registration overview page (listed as Directory ID)\n- **Client ID**: Obtained from the Entra App Registration overview page (listed as Application ID)\n- **Client Secret**: Obtained when the secret is created during the app registration process\n\n**Private Networking Settings:**\n- **VNet Resource Group Name**: The resource group containing the Virtual Network (if using the VNet template from STEP 1, this is where you deployed it)\n- **VNet Name**: The name of the Virtual Network (default from VNet template: lumen-threatfeed-vnet)\n- **Function App Subnet Name**: The subnet delegated to Microsoft.App/environments (default from VNet template: functionapp-subnet)\n- **Private Endpoint Subnet Name**: The subnet for private endpoints (default from VNet template: privateendpoint-subnet)\n- **Create Private DNS Zones**: Set to true to create new Private DNS Zones, or false to use existing ones\n\n>**Note:** Ensure the Function App subnet is delegated to Microsoft.App/environments before deployment. The deployment will create private endpoints for storage account services and configure Private DNS zones automatically." + }, + { + "title": "", + "description": "**STEP 6 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. Verify that the Function App is properly integrated with the Virtual Network by checking the Networking settings in the Azure Portal\n3. Confirm that private endpoints were created for the storage account services (blob, file, queue, table)\n4. After the app performs its first run, review the indicators ingested by either viewing the \"Lumen Defender Threat Feed Overview\" workbook or viewing the \"Threat Intelligence\" section in Microsoft Sentinel. In Microsoft Sentinel \"Threat Intelligence\", filter for source \"Lumen\" to display only Lumen generated indicators." + }, + { + "title": "", + "description": "**Troubleshooting Private Networking Issues**\n\nIf the Function App is not receiving data after deployment:\n\n1. **Check VNet Integration**: Navigate to Function App → Networking → VNet integration and verify the Function App subnet is connected\n2. **Verify Private Endpoints**: Navigate to the storage account → Networking → Private endpoint connections and verify all endpoints are in \"Approved\" state\n3. **Check DNS Resolution**: Ensure private DNS zones are properly linked to the VNet for storage account resolution\n4. **Review Function Logs**: Check Application Insights or Function App logs for connection errors\n5. **Subnet Delegation**: Confirm the Function App subnet is delegated to **Microsoft.App/environments** (required for Flex Consumption Plan)" + } + ], + "metadata": { + "id": "8d7d3b0e-4f6a-4f2a-bc2d-6d9f5a3c7e23", + "version": "2.0.0", + "kind": "dataConnector", + "source": { + "kind": "solution", + "name": "Lumen Defender Threat Feed for Microsoft Sentinel" + }, + "author": { + "name": "Lumen Technologies, Inc." + }, + "support": { + "tier": "developer", + "name": "Lumen Technologies, Inc." + } + } +} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorv2.zip b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorv2.zip new file mode 100644 index 00000000000..f8c00264be5 Binary files /dev/null and b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorv2.zip differ diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_Connector_LumenThreatFeed_AzureFunction_v2.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_Connector_LumenThreatFeed_AzureFunction_v2.json new file mode 100644 index 00000000000..f9f0214eabf --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_Connector_LumenThreatFeed_AzureFunction_v2.json @@ -0,0 +1,413 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "type": "string", + "defaultValue": "lumentfv2", + "minLength": 1, + "maxLength": 20, + "metadata": { + "description": "Name prefix for the Function App (will be lowercased and appended with a unique suffix)." + } + }, + "LumenAPIKey": { + "type": "securestring", + "metadata": { + "description": "Lumen API Key for authenticating with the Threat Feed service." + } + }, + "LumenBaseURL": { + "type": "string", + "defaultValue": "https://microsoft-sentinel-api.us1.mss.lumen.com", + "minLength": 1, + "metadata": { + "description": "Lumen Threat Feed API base URL." + } + }, + "WorkspaceID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Microsoft Sentinel Workspace ID." + } + }, + "TenantID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Azure Active Directory Tenant ID." + } + }, + "ClientID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Azure App Registration Client ID." + } + }, + "ClientSecret": { + "type": "securestring", + "minLength": 1, + "metadata": { + "description": "Azure App Registration Client Secret." + } + }, + "LumenConfidenceThreshold": { + "type": "string", + "defaultValue": "60", + "metadata": { + "description": "Minimum confidence threshold for threat indicators (60-100)." + } + }, + "LumenEnableIPv4": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Enable ingestion of IPv4 threat indicators." + } + }, + "LumenEnableDomain": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Enable ingestion of domain threat indicators." + } + }, + "AppInsightsWorkspaceResourceID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Log Analytics Workspace Resource ID for Application Insights. Format: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}" + } + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), take(uniqueString(resourceGroup().id), 3))]", + "StorageAccount": "[concat(variables('FunctionName'), take(uniqueString(resourceGroup().id), 3))]", + "deploymentStorageContainerName": "lumenthreatfeed-v2", + "StorageSuffix": "[environment().suffixes.storage]", + "storageBlobDataOwnerRoleId": "b7e6dc6d-f1e8-4753-8033-0f276bb0955b", + "storageQueueDataContributorRoleId": "974c5e8b-45b9-4653-ba55-5f855dd0fb88", + "storageTableDataContributorRoleId": "0a9a7e1f-b9d0-4cc4-a60d-0319b160aaa3", + "Polling": "*/15 * * * *" + }, + "resources": [ + { + "comments": "Application Insights for monitoring and diagnostics", + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[resourceGroup().location]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('AppInsightsWorkspaceResourceID')]" + } + }, + { + "comments": "Storage Account with enterprise security settings", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-05-01", + "name": "[variables('StorageAccount')]", + "location": "[resourceGroup().location]", + "kind": "StorageV2", + "sku": { + "name": "Standard_LRS" + }, + "properties": { + "supportsHttpsTrafficOnly": true, + "defaultToOAuthAuthentication": true, + "allowBlobPublicAccess": false, + "allowSharedKeyAccess": true, + "minimumTlsVersion": "TLS1_2", + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + } + } + }, + { + "comments": "Blob Services - explicit definition for reliability", + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}', variables('StorageAccount'), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + { + "comments": "Queue Services - required for Azure Functions runtime (timer triggers, etc.)", + "type": "Microsoft.Storage/storageAccounts/queueServices", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}', variables('StorageAccount'), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": {} + }, + { + "comments": "Table Services - required for Azure Functions runtime (lease management, etc.)", + "type": "Microsoft.Storage/storageAccounts/tableServices", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}', variables('StorageAccount'), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": {} + }, + { + "comments": "Deployment container for Function App code", + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}/{2}', variables('StorageAccount'), 'default', variables('deploymentStorageContainerName'))]", + "properties": { + "publicAccess": "None" + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('StorageAccount'), 'default')]" + ] + }, + { + "comments": "Flex Consumption App Service Plan - serverless with managed scaling", + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-12-01", + "name": "[variables('FunctionName')]", + "location": "[resourceGroup().location]", + "kind": "functionapp", + "sku": { + "tier": "FlexConsumption", + "name": "FC1" + }, + "properties": { + "reserved": true + } + }, + { + "comments": "Function App with System-Assigned Managed Identity", + "type": "Microsoft.Web/sites", + "apiVersion": "2023-12-01", + "name": "[variables('FunctionName')]", + "location": "[resourceGroup().location]", + "kind": "functionapp,linux", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "functionAppConfig": { + "deployment": { + "storage": { + "type": "blobContainer", + "value": "[format('https://{0}.blob.{1}/{2}', variables('StorageAccount'), variables('StorageSuffix'), variables('deploymentStorageContainerName'))]", + "authentication": { + "type": "SystemAssignedIdentity" + } + } + }, + "scaleAndConcurrency": { + "maximumInstanceCount": 100, + "instanceMemoryMB": 2048 + }, + "runtime": { + "name": "python", + "version": "3.11" + } + }, + "siteConfig": { + "minTlsVersion": "1.2", + "ftpsState": "Disabled", + "appSettings": [ + { + "name": "FUNCTIONS_EXTENSION_VERSION", + "value": "~4" + }, + { + "name": "APPINSIGHTS_INSTRUMENTATIONKEY", + "value": "[reference(resourceId('Microsoft.Insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]" + }, + { + "name": "APPLICATIONINSIGHTS_CONNECTION_STRING", + "value": "[reference(resourceId('Microsoft.Insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]" + }, + { + "name": "AzureWebJobsStorage__accountName", + "value": "[variables('StorageAccount')]" + }, + { + "name": "LUMEN_API_KEY", + "value": "[parameters('LumenAPIKey')]" + }, + { + "name": "LUMEN_BASE_URL", + "value": "[parameters('LumenBaseURL')]" + }, + { + "name": "WORKSPACE_ID", + "value": "[parameters('WorkspaceID')]" + }, + { + "name": "TENANT_ID", + "value": "[parameters('TenantID')]" + }, + { + "name": "CLIENT_ID", + "value": "[parameters('ClientID')]" + }, + { + "name": "CLIENT_SECRET", + "value": "[parameters('ClientSecret')]" + }, + { + "name": "LUMEN_CONFIDENCE_THRESHOLD", + "value": "[parameters('LumenConfidenceThreshold')]" + }, + { + "name": "LUMEN_ENABLE_IPV4", + "value": "[parameters('LumenEnableIPv4')]" + }, + { + "name": "LUMEN_ENABLE_DOMAIN", + "value": "[parameters('LumenEnableDomain')]" + }, + { + "name": "LUMEN_POLL_INTERVAL", + "value": "5" + }, + { + "name": "LUMEN_POLL_TIMEOUT", + "value": "300" + }, + { + "name": "Polling", + "value": "[variables('Polling')]" + } + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Storage/storageAccounts/blobServices/containers', variables('StorageAccount'), 'default', variables('deploymentStorageContainerName'))]", + "[resourceId('Microsoft.Storage/storageAccounts/queueServices', variables('StorageAccount'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts/tableServices', variables('StorageAccount'), 'default')]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ] + }, + { + "comments": "RBAC: Storage Blob Data Owner - required for deployment storage and runtime", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), variables('storageBlobDataOwnerRoleId'), resourceId('Microsoft.Web/sites', variables('FunctionName')))]", + "scope": "[concat('Microsoft.Storage/storageAccounts', '/', variables('StorageAccount'))]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('storageBlobDataOwnerRoleId'))]", + "principalId": "[reference(resourceId('Microsoft.Web/sites', variables('FunctionName')), '2023-12-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', variables('FunctionName'))]" + ] + }, + { + "comments": "RBAC: Storage Queue Data Contributor - required for timer triggers and internal messaging", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), variables('storageQueueDataContributorRoleId'), resourceId('Microsoft.Web/sites', variables('FunctionName')))]", + "scope": "[concat('Microsoft.Storage/storageAccounts', '/', variables('StorageAccount'))]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('storageQueueDataContributorRoleId'))]", + "principalId": "[reference(resourceId('Microsoft.Web/sites', variables('FunctionName')), '2023-12-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', variables('FunctionName'))]" + ] + }, + { + "comments": "RBAC: Storage Table Data Contributor - required for lease management and singleton locks", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), variables('storageTableDataContributorRoleId'), resourceId('Microsoft.Web/sites', variables('FunctionName')))]", + "scope": "[concat('Microsoft.Storage/storageAccounts', '/', variables('StorageAccount'))]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('storageTableDataContributorRoleId'))]", + "principalId": "[reference(resourceId('Microsoft.Web/sites', variables('FunctionName')), '2023-12-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', variables('FunctionName'))]" + ] + }, + { + "comments": "Wait for RBAC propagation before deploying code. Azure RBAC can take up to 5 minutes to propagate.", + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "name": "WaitForRBACPropagation", + "location": "[resourceGroup().location]", + "kind": "AzurePowerShell", + "properties": { + "azPowerShellVersion": "9.7", + "scriptContent": "Write-Output 'Waiting for RBAC propagation...'; Start-Sleep -Seconds 120; Write-Output 'RBAC propagation wait complete.'", + "cleanupPreference": "Always", + "retentionInterval": "PT1H", + "timeout": "PT10M" + }, + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), 'Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), variables('storageBlobDataOwnerRoleId'), resourceId('Microsoft.Web/sites', variables('FunctionName'))))]", + "[extensionResourceId(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), 'Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), variables('storageQueueDataContributorRoleId'), resourceId('Microsoft.Web/sites', variables('FunctionName'))))]", + "[extensionResourceId(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), 'Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), variables('storageTableDataContributorRoleId'), resourceId('Microsoft.Web/sites', variables('FunctionName'))))]" + ] + }, + { + "comments": "Deploy Function App code after RBAC is ready", + "type": "Microsoft.Web/sites/extensions", + "apiVersion": "2023-12-01", + "name": "[format('{0}/{1}', variables('FunctionName'), 'onedeploy')]", + "properties": { + "packageUri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Solutions/Lumen%20Defender%20Threat%20Feed/Data%20Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorv2.zip", + "remoteBuild": true + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deploymentScripts', 'WaitForRBACPropagation')]", + "[resourceId('Microsoft.Web/sites', variables('FunctionName'))]" + ] + } + ], + "outputs": { + "FunctionAppName": { + "type": "string", + "value": "[variables('FunctionName')]", + "metadata": { + "description": "The name of the deployed Function App" + } + }, + "StorageAccountName": { + "type": "string", + "value": "[variables('StorageAccount')]", + "metadata": { + "description": "The name of the storage account" + } + }, + "FunctionAppUrl": { + "type": "string", + "value": "[format('https://{0}.azurewebsites.net', variables('FunctionName'))]", + "metadata": { + "description": "The URL of the deployed Function App" + } + }, + "FunctionAppPrincipalId": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Web/sites', variables('FunctionName')), '2023-12-01', 'Full').identity.principalId]", + "metadata": { + "description": "The managed identity principal ID of the Function App" + } + } + } +} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_Connector_LumenThreatFeed_AzureFunction_v2_privateendpoint.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_Connector_LumenThreatFeed_AzureFunction_v2_privateendpoint.json new file mode 100644 index 00000000000..ab45fe019dc --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_Connector_LumenThreatFeed_AzureFunction_v2_privateendpoint.json @@ -0,0 +1,778 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "title": "Lumen Threat Feed Connector V2 - Private Endpoint Edition", + "description": "Deploys the Lumen Threat Feed Connector with VNet integration and private endpoints for enterprise environments requiring network isolation.", + "prerequisites": [ + "An existing Virtual Network with two subnets", + "Subnet 1: Delegated to Microsoft.App/environments (for Flex Consumption VNet integration)", + "Subnet 2: Non-delegated (for Private Endpoints)", + "Private DNS zones can be created by this template or use existing ones" + ] + }, + "parameters": { + "FunctionName": { + "type": "string", + "defaultValue": "lumentfv2", + "minLength": 1, + "maxLength": 11, + "metadata": { + "description": "Name prefix for the Function App (will be lowercased and appended with a unique suffix). Max 11 chars due to storage account naming limits." + } + }, + "LumenAPIKey": { + "type": "securestring", + "metadata": { + "description": "Lumen API Key for authenticating with the Threat Feed service." + } + }, + "LumenBaseURL": { + "type": "string", + "defaultValue": "https://microsoft-sentinel-api.us1.mss.lumen.com", + "minLength": 1, + "metadata": { + "description": "Lumen Threat Feed API base URL." + } + }, + "WorkspaceID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Microsoft Sentinel Workspace ID." + } + }, + "TenantID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Azure Active Directory Tenant ID." + } + }, + "ClientID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Azure App Registration Client ID." + } + }, + "ClientSecret": { + "type": "securestring", + "minLength": 1, + "metadata": { + "description": "Azure App Registration Client Secret." + } + }, + "LumenConfidenceThreshold": { + "type": "string", + "defaultValue": "60", + "metadata": { + "description": "Minimum confidence threshold for threat indicators (60-100)." + } + }, + "LumenEnableIPv4": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Enable ingestion of IPv4 threat indicators." + } + }, + "LumenEnableDomain": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Enable ingestion of domain threat indicators." + } + }, + "AppInsightsWorkspaceResourceID": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Log Analytics Workspace Resource ID for Application Insights. Format: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}" + } + }, + "VNetResourceGroupName": { + "type": "string", + "metadata": { + "description": "Name of the Resource Group containing the existing Virtual Network." + } + }, + "VNetName": { + "type": "string", + "metadata": { + "description": "Name of the existing Virtual Network." + } + }, + "FunctionAppSubnetName": { + "type": "string", + "metadata": { + "description": "Name of the subnet for Function App VNet integration. This subnet MUST be delegated to Microsoft.App/environments (required for Flex Consumption)." + } + }, + "PrivateEndpointSubnetName": { + "type": "string", + "metadata": { + "description": "Name of the subnet for Private Endpoints. This subnet must NOT be delegated." + } + }, + "CreatePrivateDnsZones": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Set to true to create new Private DNS Zones, or false to use existing ones in the VNet resource group." + } + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), take(uniqueString(resourceGroup().id), 3))]", + "StorageAccount": "[concat(variables('FunctionName'), take(uniqueString(resourceGroup().id), 3))]", + "deploymentStorageContainerName": "lumenthreatfeed-v2", + "StorageSuffix": "[environment().suffixes.storage]", + "storageBlobDataOwnerRoleId": "b7e6dc6d-f1e8-4753-8033-0f276bb0955b", + "Polling": "*/15 * * * *", + "vnetResourceId": "[resourceId(parameters('VNetResourceGroupName'), 'Microsoft.Network/virtualNetworks', parameters('VNetName'))]", + "functionAppSubnetResourceId": "[resourceId(parameters('VNetResourceGroupName'), 'Microsoft.Network/virtualNetworks/subnets', parameters('VNetName'), parameters('FunctionAppSubnetName'))]", + "privateEndpointSubnetResourceId": "[resourceId(parameters('VNetResourceGroupName'), 'Microsoft.Network/virtualNetworks/subnets', parameters('VNetName'), parameters('PrivateEndpointSubnetName'))]", + "privateEndpointBlobName": "[concat(variables('StorageAccount'), '-pe-blob')]", + "privateEndpointQueueName": "[concat(variables('StorageAccount'), '-pe-queue')]", + "privateEndpointTableName": "[concat(variables('StorageAccount'), '-pe-table')]", + "privateEndpointFileName": "[concat(variables('StorageAccount'), '-pe-file')]", + "privateDnsZoneBlob": "privatelink.blob.core.windows.net", + "privateDnsZoneQueue": "privatelink.queue.core.windows.net", + "privateDnsZoneTable": "privatelink.table.core.windows.net", + "privateDnsZoneFile": "privatelink.file.core.windows.net" + }, + "resources": [ + { + "comments": "Application Insights for monitoring and diagnostics", + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[resourceGroup().location]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('AppInsightsWorkspaceResourceID')]" + } + }, + { + "comments": "Storage Account with private network access only", + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-05-01", + "name": "[variables('StorageAccount')]", + "location": "[resourceGroup().location]", + "kind": "StorageV2", + "sku": { + "name": "Standard_LRS" + }, + "properties": { + "supportsHttpsTrafficOnly": true, + "defaultToOAuthAuthentication": true, + "allowBlobPublicAccess": false, + "allowSharedKeyAccess": true, + "minimumTlsVersion": "TLS1_2", + "publicNetworkAccess": "Disabled", + "networkAcls": { + "bypass": "None", + "virtualNetworkRules": [], + "ipRules": [], + "defaultAction": "Deny" + }, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + }, + "table": { + "keyType": "Account", + "enabled": true + }, + "queue": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage", + "requireInfrastructureEncryption": true + } + } + }, + { + "comments": "Blob Services", + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}', variables('StorageAccount'), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + { + "comments": "Queue Services", + "type": "Microsoft.Storage/storageAccounts/queueServices", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}', variables('StorageAccount'), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": {} + }, + { + "comments": "Table Services", + "type": "Microsoft.Storage/storageAccounts/tableServices", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}', variables('StorageAccount'), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": {} + }, + { + "comments": "File Services", + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}', variables('StorageAccount'), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": {} + }, + { + "comments": "Deployment container for Function App code", + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-05-01", + "name": "[format('{0}/{1}/{2}', variables('StorageAccount'), 'default', variables('deploymentStorageContainerName'))]", + "properties": { + "publicAccess": "None" + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('StorageAccount'), 'default')]" + ] + }, + { + "comments": "Private DNS Zone for Blob Storage", + "condition": "[parameters('CreatePrivateDnsZones')]", + "type": "Microsoft.Network/privateDnsZones", + "apiVersion": "2020-06-01", + "name": "[variables('privateDnsZoneBlob')]", + "location": "global", + "properties": {} + }, + { + "comments": "Private DNS Zone for Queue Storage", + "condition": "[parameters('CreatePrivateDnsZones')]", + "type": "Microsoft.Network/privateDnsZones", + "apiVersion": "2020-06-01", + "name": "[variables('privateDnsZoneQueue')]", + "location": "global", + "properties": {} + }, + { + "comments": "Private DNS Zone for Table Storage", + "condition": "[parameters('CreatePrivateDnsZones')]", + "type": "Microsoft.Network/privateDnsZones", + "apiVersion": "2020-06-01", + "name": "[variables('privateDnsZoneTable')]", + "location": "global", + "properties": {} + }, + { + "comments": "Private DNS Zone for File Storage", + "condition": "[parameters('CreatePrivateDnsZones')]", + "type": "Microsoft.Network/privateDnsZones", + "apiVersion": "2020-06-01", + "name": "[variables('privateDnsZoneFile')]", + "location": "global", + "properties": {} + }, + { + "comments": "Link Blob DNS Zone to VNet", + "condition": "[parameters('CreatePrivateDnsZones')]", + "type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks", + "apiVersion": "2020-06-01", + "name": "[concat(variables('privateDnsZoneBlob'), '/blob-vnet-link')]", + "location": "global", + "dependsOn": [ + "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneBlob'))]" + ], + "properties": { + "virtualNetwork": { + "id": "[variables('vnetResourceId')]" + }, + "registrationEnabled": false + } + }, + { + "comments": "Link Queue DNS Zone to VNet", + "condition": "[parameters('CreatePrivateDnsZones')]", + "type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks", + "apiVersion": "2020-06-01", + "name": "[concat(variables('privateDnsZoneQueue'), '/queue-vnet-link')]", + "location": "global", + "dependsOn": [ + "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneQueue'))]" + ], + "properties": { + "virtualNetwork": { + "id": "[variables('vnetResourceId')]" + }, + "registrationEnabled": false + } + }, + { + "comments": "Link Table DNS Zone to VNet", + "condition": "[parameters('CreatePrivateDnsZones')]", + "type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks", + "apiVersion": "2020-06-01", + "name": "[concat(variables('privateDnsZoneTable'), '/table-vnet-link')]", + "location": "global", + "dependsOn": [ + "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneTable'))]" + ], + "properties": { + "virtualNetwork": { + "id": "[variables('vnetResourceId')]" + }, + "registrationEnabled": false + } + }, + { + "comments": "Link File DNS Zone to VNet", + "condition": "[parameters('CreatePrivateDnsZones')]", + "type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks", + "apiVersion": "2020-06-01", + "name": "[concat(variables('privateDnsZoneFile'), '/file-vnet-link')]", + "location": "global", + "dependsOn": [ + "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneFile'))]" + ], + "properties": { + "virtualNetwork": { + "id": "[variables('vnetResourceId')]" + }, + "registrationEnabled": false + } + }, + { + "comments": "Private Endpoint for Blob Storage", + "type": "Microsoft.Network/privateEndpoints", + "apiVersion": "2023-05-01", + "name": "[variables('privateEndpointBlobName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": { + "subnet": { + "id": "[variables('privateEndpointSubnetResourceId')]" + }, + "privateLinkServiceConnections": [ + { + "name": "BlobConnection", + "properties": { + "privateLinkServiceId": "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]", + "groupIds": [ + "blob" + ] + } + } + ] + } + }, + { + "comments": "DNS Zone Group for Blob Private Endpoint", + "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups", + "apiVersion": "2023-05-01", + "name": "[concat(variables('privateEndpointBlobName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Network/privateEndpoints', variables('privateEndpointBlobName'))]", + "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneBlob'))]" + ], + "properties": { + "privateDnsZoneConfigs": [ + { + "name": "blob-config", + "properties": { + "privateDnsZoneId": "[if(parameters('CreatePrivateDnsZones'), resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneBlob')), resourceId(parameters('VNetResourceGroupName'), 'Microsoft.Network/privateDnsZones', variables('privateDnsZoneBlob')))]" + } + } + ] + } + }, + { + "comments": "Private Endpoint for Queue Storage", + "type": "Microsoft.Network/privateEndpoints", + "apiVersion": "2023-05-01", + "name": "[variables('privateEndpointQueueName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": { + "subnet": { + "id": "[variables('privateEndpointSubnetResourceId')]" + }, + "privateLinkServiceConnections": [ + { + "name": "QueueConnection", + "properties": { + "privateLinkServiceId": "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]", + "groupIds": [ + "queue" + ] + } + } + ] + } + }, + { + "comments": "DNS Zone Group for Queue Private Endpoint", + "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups", + "apiVersion": "2023-05-01", + "name": "[concat(variables('privateEndpointQueueName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Network/privateEndpoints', variables('privateEndpointQueueName'))]", + "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneQueue'))]" + ], + "properties": { + "privateDnsZoneConfigs": [ + { + "name": "queue-config", + "properties": { + "privateDnsZoneId": "[if(parameters('CreatePrivateDnsZones'), resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneQueue')), resourceId(parameters('VNetResourceGroupName'), 'Microsoft.Network/privateDnsZones', variables('privateDnsZoneQueue')))]" + } + } + ] + } + }, + { + "comments": "Private Endpoint for Table Storage", + "type": "Microsoft.Network/privateEndpoints", + "apiVersion": "2023-05-01", + "name": "[variables('privateEndpointTableName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": { + "subnet": { + "id": "[variables('privateEndpointSubnetResourceId')]" + }, + "privateLinkServiceConnections": [ + { + "name": "TableConnection", + "properties": { + "privateLinkServiceId": "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]", + "groupIds": [ + "table" + ] + } + } + ] + } + }, + { + "comments": "DNS Zone Group for Table Private Endpoint", + "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups", + "apiVersion": "2023-05-01", + "name": "[concat(variables('privateEndpointTableName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Network/privateEndpoints', variables('privateEndpointTableName'))]", + "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneTable'))]" + ], + "properties": { + "privateDnsZoneConfigs": [ + { + "name": "table-config", + "properties": { + "privateDnsZoneId": "[if(parameters('CreatePrivateDnsZones'), resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneTable')), resourceId(parameters('VNetResourceGroupName'), 'Microsoft.Network/privateDnsZones', variables('privateDnsZoneTable')))]" + } + } + ] + } + }, + { + "comments": "Private Endpoint for File Storage", + "type": "Microsoft.Network/privateEndpoints", + "apiVersion": "2023-05-01", + "name": "[variables('privateEndpointFileName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]" + ], + "properties": { + "subnet": { + "id": "[variables('privateEndpointSubnetResourceId')]" + }, + "privateLinkServiceConnections": [ + { + "name": "FileConnection", + "properties": { + "privateLinkServiceId": "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount'))]", + "groupIds": [ + "file" + ] + } + } + ] + } + }, + { + "comments": "DNS Zone Group for File Private Endpoint", + "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups", + "apiVersion": "2023-05-01", + "name": "[concat(variables('privateEndpointFileName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Network/privateEndpoints', variables('privateEndpointFileName'))]", + "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneFile'))]" + ], + "properties": { + "privateDnsZoneConfigs": [ + { + "name": "file-config", + "properties": { + "privateDnsZoneId": "[if(parameters('CreatePrivateDnsZones'), resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsZoneFile')), resourceId(parameters('VNetResourceGroupName'), 'Microsoft.Network/privateDnsZones', variables('privateDnsZoneFile')))]" + } + } + ] + } + }, + { + "comments": "Flex Consumption App Service Plan", + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-12-01", + "name": "[variables('FunctionName')]", + "location": "[resourceGroup().location]", + "kind": "functionapp", + "sku": { + "tier": "FlexConsumption", + "name": "FC1" + }, + "properties": { + "reserved": true + } + }, + { + "comments": "Function App with VNet Integration and System-Assigned Managed Identity", + "type": "Microsoft.Web/sites", + "apiVersion": "2023-12-01", + "name": "[variables('FunctionName')]", + "location": "[resourceGroup().location]", + "kind": "functionapp,linux", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "virtualNetworkSubnetId": "[variables('functionAppSubnetResourceId')]", + "vnetRouteAllEnabled": true, + "functionAppConfig": { + "deployment": { + "storage": { + "type": "blobContainer", + "value": "[format('https://{0}.blob.{1}/{2}', variables('StorageAccount'), variables('StorageSuffix'), variables('deploymentStorageContainerName'))]", + "authentication": { + "type": "SystemAssignedIdentity" + } + } + }, + "scaleAndConcurrency": { + "maximumInstanceCount": 100, + "instanceMemoryMB": 2048 + }, + "runtime": { + "name": "python", + "version": "3.11" + } + }, + "siteConfig": { + "minTlsVersion": "1.2", + "ftpsState": "Disabled", + "vnetRouteAllEnabled": true, + "appSettings": [ + { + "name": "FUNCTIONS_EXTENSION_VERSION", + "value": "~4" + }, + { + "name": "APPINSIGHTS_INSTRUMENTATIONKEY", + "value": "[reference(resourceId('Microsoft.Insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]" + }, + { + "name": "APPLICATIONINSIGHTS_CONNECTION_STRING", + "value": "[reference(resourceId('Microsoft.Insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]" + }, + { + "name": "AzureWebJobsStorage__accountName", + "value": "[variables('StorageAccount')]" + }, + { + "name": "WEBSITE_CONTENTOVERVNET", + "value": "1" + }, + { + "name": "LUMEN_API_KEY", + "value": "[parameters('LumenAPIKey')]" + }, + { + "name": "LUMEN_BASE_URL", + "value": "[parameters('LumenBaseURL')]" + }, + { + "name": "WORKSPACE_ID", + "value": "[parameters('WorkspaceID')]" + }, + { + "name": "TENANT_ID", + "value": "[parameters('TenantID')]" + }, + { + "name": "CLIENT_ID", + "value": "[parameters('ClientID')]" + }, + { + "name": "CLIENT_SECRET", + "value": "[parameters('ClientSecret')]" + }, + { + "name": "LUMEN_CONFIDENCE_THRESHOLD", + "value": "[parameters('LumenConfidenceThreshold')]" + }, + { + "name": "LUMEN_ENABLE_IPV4", + "value": "[parameters('LumenEnableIPv4')]" + }, + { + "name": "LUMEN_ENABLE_DOMAIN", + "value": "[parameters('LumenEnableDomain')]" + }, + { + "name": "LUMEN_POLL_INTERVAL", + "value": "5" + }, + { + "name": "LUMEN_POLL_TIMEOUT", + "value": "300" + }, + { + "name": "Polling", + "value": "[variables('Polling')]" + } + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Storage/storageAccounts/blobServices/containers', variables('StorageAccount'), 'default', variables('deploymentStorageContainerName'))]", + "[resourceId('Microsoft.Storage/storageAccounts/queueServices', variables('StorageAccount'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts/tableServices', variables('StorageAccount'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('StorageAccount'), 'default')]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]", + "[resourceId('Microsoft.Network/privateEndpoints/privateDnsZoneGroups', variables('privateEndpointBlobName'), 'default')]", + "[resourceId('Microsoft.Network/privateEndpoints/privateDnsZoneGroups', variables('privateEndpointQueueName'), 'default')]", + "[resourceId('Microsoft.Network/privateEndpoints/privateDnsZoneGroups', variables('privateEndpointTableName'), 'default')]", + "[resourceId('Microsoft.Network/privateEndpoints/privateDnsZoneGroups', variables('privateEndpointFileName'), 'default')]" + ] + }, + { + "comments": "RBAC: Storage Blob Data Owner - required for deployment storage and runtime", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), variables('storageBlobDataOwnerRoleId'), resourceId('Microsoft.Web/sites', variables('FunctionName')))]", + "scope": "[concat('Microsoft.Storage/storageAccounts', '/', variables('StorageAccount'))]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('storageBlobDataOwnerRoleId'))]", + "principalId": "[reference(resourceId('Microsoft.Web/sites', variables('FunctionName')), '2023-12-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', variables('FunctionName'))]" + ] + }, + { + "comments": "Wait for RBAC propagation before deploying code", + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "name": "WaitForRBACPropagation", + "location": "[resourceGroup().location]", + "kind": "AzurePowerShell", + "properties": { + "azPowerShellVersion": "9.7", + "scriptContent": "Write-Output 'Waiting for RBAC propagation...'; Start-Sleep -Seconds 60; Write-Output 'RBAC propagation wait complete.'", + "cleanupPreference": "Always", + "retentionInterval": "PT1H", + "timeout": "PT10M" + }, + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), 'Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccount')), variables('storageBlobDataOwnerRoleId'), resourceId('Microsoft.Web/sites', variables('FunctionName'))))]" + ] + }, + { + "comments": "Deploy Function App code after RBAC and private endpoints are ready", + "type": "Microsoft.Web/sites/extensions", + "apiVersion": "2023-12-01", + "name": "[format('{0}/{1}', variables('FunctionName'), 'onedeploy')]", + "properties": { + "packageUri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Solutions/Lumen%20Defender%20Threat%20Feed/Data%20Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorv2.zip", + "remoteBuild": true + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deploymentScripts', 'WaitForRBACPropagation')]", + "[resourceId('Microsoft.Web/sites', variables('FunctionName'))]" + ] + } + ], + "outputs": { + "FunctionAppName": { + "type": "string", + "value": "[variables('FunctionName')]", + "metadata": { + "description": "The name of the deployed Function App" + } + }, + "StorageAccountName": { + "type": "string", + "value": "[variables('StorageAccount')]", + "metadata": { + "description": "The name of the storage account" + } + }, + "FunctionAppUrl": { + "type": "string", + "value": "[format('https://{0}.azurewebsites.net', variables('FunctionName'))]", + "metadata": { + "description": "The URL of the deployed Function App" + } + }, + "FunctionAppPrincipalId": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Web/sites', variables('FunctionName')), '2023-12-01', 'Full').identity.principalId]", + "metadata": { + "description": "The managed identity principal ID of the Function App" + } + }, + "PrivateEndpoints": { + "type": "object", + "value": { + "blob": "[variables('privateEndpointBlobName')]", + "queue": "[variables('privateEndpointQueueName')]", + "table": "[variables('privateEndpointTableName')]", + "file": "[variables('privateEndpointFileName')]" + }, + "metadata": { + "description": "Names of the created private endpoints" + } + } + } +} diff --git a/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_VNet_for_PrivateEndpoint.json b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_VNet_for_PrivateEndpoint.json new file mode 100644 index 00000000000..f3069306aaa --- /dev/null +++ b/Solutions/Lumen Defender Threat Feed/Data Connectors/LumenThreatFeedv2/azuredeploy_VNet_for_PrivateEndpoint.json @@ -0,0 +1,118 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "description": "Creates a Virtual Network with subnets configured for the Lumen Threat Feed Connector Private Endpoint deployment. Deploy this BEFORE deploying the private endpoint connector template." + }, + "parameters": { + "VNetName": { + "type": "string", + "defaultValue": "lumen-threatfeed-vnet", + "metadata": { + "description": "Name of the Virtual Network to create" + } + }, + "VNetAddressPrefix": { + "type": "string", + "defaultValue": "10.0.0.0/16", + "metadata": { + "description": "Address space for the Virtual Network" + } + }, + "FunctionAppSubnetName": { + "type": "string", + "defaultValue": "functionapp-subnet", + "metadata": { + "description": "Name of the subnet for Function App VNet integration" + } + }, + "FunctionAppSubnetPrefix": { + "type": "string", + "defaultValue": "10.0.1.0/24", + "metadata": { + "description": "Address prefix for the Function App subnet" + } + }, + "PrivateEndpointSubnetName": { + "type": "string", + "defaultValue": "privateendpoint-subnet", + "metadata": { + "description": "Name of the subnet for Private Endpoints" + } + }, + "PrivateEndpointSubnetPrefix": { + "type": "string", + "defaultValue": "10.0.2.0/24", + "metadata": { + "description": "Address prefix for the Private Endpoint subnet" + } + } + }, + "resources": [ + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2023-05-01", + "name": "[parameters('VNetName')]", + "location": "[resourceGroup().location]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[parameters('VNetAddressPrefix')]" + ] + }, + "subnets": [ + { + "name": "[parameters('FunctionAppSubnetName')]", + "properties": { + "addressPrefix": "[parameters('FunctionAppSubnetPrefix')]", + "delegations": [ + { + "name": "Microsoft.App.environments", + "properties": { + "serviceName": "Microsoft.App/environments" + } + } + ], + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + }, + { + "name": "[parameters('PrivateEndpointSubnetName')]", + "properties": { + "addressPrefix": "[parameters('PrivateEndpointSubnetPrefix')]", + "privateEndpointNetworkPolicies": "Disabled", + "privateLinkServiceNetworkPolicies": "Enabled" + } + } + ] + } + } + ], + "outputs": { + "VNetName": { + "type": "string", + "value": "[parameters('VNetName')]" + }, + "VNetResourceGroup": { + "type": "string", + "value": "[resourceGroup().name]" + }, + "FunctionAppSubnetName": { + "type": "string", + "value": "[parameters('FunctionAppSubnetName')]" + }, + "PrivateEndpointSubnetName": { + "type": "string", + "value": "[parameters('PrivateEndpointSubnetName')]" + }, + "FunctionAppSubnetId": { + "type": "string", + "value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('VNetName'), parameters('FunctionAppSubnetName'))]" + }, + "PrivateEndpointSubnetId": { + "type": "string", + "value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('VNetName'), parameters('PrivateEndpointSubnetName'))]" + } + } +} diff --git a/Solutions/Lumen Defender Threat Feed/Data/Solution_LumenDefenderThreatFeed.json b/Solutions/Lumen Defender Threat Feed/Data/Solution_LumenDefenderThreatFeed.json index cb77d09ca38..4729dde8722 100644 --- a/Solutions/Lumen Defender Threat Feed/Data/Solution_LumenDefenderThreatFeed.json +++ b/Solutions/Lumen Defender Threat Feed/Data/Solution_LumenDefenderThreatFeed.json @@ -27,10 +27,11 @@ "Hunting Queries/Lumen_IPIndicator_CommonSecurityLog.yaml" ], "Data Connectors": [ - "Data Connectors/LumenThreatFeed/LumenThreatFeedConnector_ConnectorUI.json" + "Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorv2_ConnectorUI.json", + "Data Connectors/LumenThreatFeedv2/LumenThreatFeedConnectorv2_PrivateNetworking_ConnectorUI.json" ], "BasePath": "C:\\GitHub\\Azure-Sentinel\\Solutions\\Lumen Defender Threat Feed", - "Version": "3.1.0", + "Version": "3.2.0", "Metadata": "SolutionMetadata.json", "TemplateSpec": false } \ No newline at end of file diff --git a/Solutions/Lumen Defender Threat Feed/Package/3.2.0.zip b/Solutions/Lumen Defender Threat Feed/Package/3.2.0.zip new file mode 100644 index 00000000000..bdb534343a1 Binary files /dev/null and b/Solutions/Lumen Defender Threat Feed/Package/3.2.0.zip differ diff --git a/Solutions/Lumen Defender Threat Feed/Package/createUiDefinition.json b/Solutions/Lumen Defender Threat Feed/Package/createUiDefinition.json index 02a6365b6f9..92ddf569fd2 100644 --- a/Solutions/Lumen Defender Threat Feed/Package/createUiDefinition.json +++ b/Solutions/Lumen Defender Threat Feed/Package/createUiDefinition.json @@ -6,7 +6,7 @@ "config": { "isWizard": false, "basics": { - "description": "\n\n**Note:** Please refer to the following before installing the solution: \n\n• Review the solution [Release Notes](https://github.com/Azure/Azure-Sentinel/tree/master/Solutions/Lumen%20Defender%20Threat%20Feed/ReleaseNotes.md)\n\n • There may be [known issues](https://aka.ms/sentinelsolutionsknownissues) pertaining to this Solution, please refer to them before installing.\n\nThe Lumen Defender Threat Feed for Microsoft Sentinel solution delivers high-confidence threat intelligence indicators of compromise directly into your Sentinel workspace.\n\n**Data Connectors:** 1, **Workbooks:** 1, **Analytic Rules:** 8, **Hunting Queries:** 1\n\n[Learn more about Microsoft Sentinel](https://aka.ms/azuresentinel) | [Learn more about Solutions](https://aka.ms/azuresentinelsolutionsdoc)", + "description": "\n\n**Note:** Please refer to the following before installing the solution: \n\n• Review the solution [Release Notes](https://github.com/Azure/Azure-Sentinel/tree/master/Solutions/Lumen%20Defender%20Threat%20Feed/ReleaseNotes.md)\n\n • There may be [known issues](https://aka.ms/sentinelsolutionsknownissues) pertaining to this Solution, please refer to them before installing.\n\nThe Lumen Defender Threat Feed for Microsoft Sentinel solution delivers high-confidence threat intelligence indicators of compromise directly into your Sentinel workspace.\n\n**Data Connectors:** 2, **Workbooks:** 1, **Analytic Rules:** 8, **Hunting Queries:** 1\n\n[Learn more about Microsoft Sentinel](https://aka.ms/azuresentinel) | [Learn more about Solutions](https://aka.ms/azuresentinelsolutionsdoc)", "subscription": { "resourceProviders": [ "Microsoft.OperationsManagement/solutions", @@ -64,7 +64,14 @@ } }, { - "name": "dataconnectors-link1", + "name": "dataconnectors2-text", + "type": "Microsoft.Common.TextBlock", + "options": { + "text": "This Solution installs the data connector for Lumen Defender Threat Feed. You can get Lumen Defender Threat Feed custom log data in your Microsoft Sentinel workspace. After installing the solution, configure and enable this data connector by following guidance in Manage solution view." + } + }, + { + "name": "dataconnectors-link2", "type": "Microsoft.Common.TextBlock", "options": { "link": { diff --git a/Solutions/Lumen Defender Threat Feed/Package/mainTemplate.json b/Solutions/Lumen Defender Threat Feed/Package/mainTemplate.json index a3dcd4ee423..afc8cbbc3c8 100644 --- a/Solutions/Lumen Defender Threat Feed/Package/mainTemplate.json +++ b/Solutions/Lumen Defender Threat Feed/Package/mainTemplate.json @@ -41,8 +41,8 @@ "email": "matthew.collier@lumen.com", "_email": "[variables('email')]", "_solutionName": "Lumen Defender Threat Feed", - "_solutionVersion": "3.1.0", - "solutionId": "centurylink.azure-sentinel-solution-lumen-defender-threat-feed", + "_solutionVersion": "3.2.0", + "solutionId": "centurylink.lumen-defender-threat-feed", "_solutionId": "[variables('solutionId')]", "workbookVersion1": "1.0", "workbookContentId1": "LumenDefenderThreatFeedOverviewWorkbook", @@ -112,15 +112,24 @@ "_huntingQuerycontentId1": "4e329d3a-9fc0-4be7-9000-e092e7f68011", "huntingQueryTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-hq-',uniquestring('4e329d3a-9fc0-4be7-9000-e092e7f68011')))]" }, - "uiConfigId1": "LumenThreatFeedConnector", + "uiConfigId1": "LumenThreatFeedConnectorV2", "_uiConfigId1": "[variables('uiConfigId1')]", - "dataConnectorContentId1": "LumenThreatFeedConnector", + "dataConnectorContentId1": "LumenThreatFeedConnectorV2", "_dataConnectorContentId1": "[variables('dataConnectorContentId1')]", "dataConnectorId1": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId1'))]", "_dataConnectorId1": "[variables('dataConnectorId1')]", "dataConnectorTemplateSpecName1": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId1'))))]", - "dataConnectorVersion1": "1.0.0", + "dataConnectorVersion1": "2.0.0", "_dataConnectorcontentProductId1": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId1'),'-', variables('dataConnectorVersion1'))))]", + "uiConfigId2": "LumenThreatFeedConnectorV2PrivateNetworking", + "_uiConfigId2": "[variables('uiConfigId2')]", + "dataConnectorContentId2": "LumenThreatFeedConnectorV2PrivateNetworking", + "_dataConnectorContentId2": "[variables('dataConnectorContentId2')]", + "dataConnectorId2": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId2'))]", + "_dataConnectorId2": "[variables('dataConnectorId2')]", + "dataConnectorTemplateSpecName2": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId2'))))]", + "dataConnectorVersion2": "2.0.0", + "_dataConnectorcontentProductId2": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId2'),'-', variables('dataConnectorVersion2'))))]", "_solutioncontentProductId": "[concat(take(variables('_solutionId'),50),'-','sl','-', uniqueString(concat(variables('_solutionId'),'-','Solution','-',variables('_solutionId'),'-', variables('_solutionVersion'))))]" }, "resources": [ @@ -133,7 +142,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen-Threat-Feed-Overview Workbook with template version 3.1.0", + "description": "Lumen-Threat-Feed-Overview Workbook with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('workbookVersion1')]", @@ -221,7 +230,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_DomainEntity_DNS_AnalyticalRules Analytics Rule with template version 3.1.0", + "description": "Lumen_DomainEntity_DNS_AnalyticalRules Analytics Rule with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('analyticRuleObject1').analyticRuleVersion1]", @@ -275,13 +284,13 @@ ], "entityMappings": [ { + "entityType": "DNS", "fieldMappings": [ { - "columnName": "DNS_domainEntity", - "identifier": "DomainName" + "identifier": "DomainName", + "columnName": "DNS_domainEntity" } - ], - "entityType": "DNS" + ] } ] } @@ -337,7 +346,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_IPEntity_CommonSecurityLog_AnalyticalRules Analytics Rule with template version 3.1.0", + "description": "Lumen_IPEntity_CommonSecurityLog_AnalyticalRules Analytics Rule with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('analyticRuleObject2').analyticRuleVersion2]", @@ -397,13 +406,13 @@ ], "entityMappings": [ { + "entityType": "IP", "fieldMappings": [ { - "columnName": "CS_ipEntity", - "identifier": "Address" + "identifier": "Address", + "columnName": "CS_ipEntity" } - ], - "entityType": "IP" + ] } ] } @@ -459,7 +468,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_IPEntity_DeviceEvents_AnalyticalRules Analytics Rule with template version 3.1.0", + "description": "Lumen_IPEntity_DeviceEvents_AnalyticalRules Analytics Rule with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('analyticRuleObject3').analyticRuleVersion3]", @@ -513,13 +522,13 @@ ], "entityMappings": [ { + "entityType": "IP", "fieldMappings": [ { - "columnName": "DE_ipEntity", - "identifier": "Address" + "identifier": "Address", + "columnName": "DE_ipEntity" } - ], - "entityType": "IP" + ] } ] } @@ -575,7 +584,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_IPEntity_IdentityLogonEvents_AnalyticalRules Analytics Rule with template version 3.1.0", + "description": "Lumen_IPEntity_IdentityLogonEvents_AnalyticalRules Analytics Rule with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('analyticRuleObject4').analyticRuleVersion4]", @@ -632,13 +641,13 @@ ], "entityMappings": [ { + "entityType": "IP", "fieldMappings": [ { - "columnName": "ILE_ipEntity", - "identifier": "Address" + "identifier": "Address", + "columnName": "ILE_ipEntity" } - ], - "entityType": "IP" + ] } ] } @@ -694,7 +703,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_IPEntity_OfficeActivity_AnalyticalRules Analytics Rule with template version 3.1.0", + "description": "Lumen_IPEntity_OfficeActivity_AnalyticalRules Analytics Rule with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('analyticRuleObject5').analyticRuleVersion5]", @@ -748,13 +757,13 @@ ], "entityMappings": [ { + "entityType": "IP", "fieldMappings": [ { - "columnName": "OA_ipEntity", - "identifier": "Address" + "identifier": "Address", + "columnName": "OA_ipEntity" } - ], - "entityType": "IP" + ] } ] } @@ -810,7 +819,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_IPEntity_SecurityEvent_AnalyticalRules Analytics Rule with template version 3.1.0", + "description": "Lumen_IPEntity_SecurityEvent_AnalyticalRules Analytics Rule with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('analyticRuleObject6').analyticRuleVersion6]", @@ -870,13 +879,13 @@ ], "entityMappings": [ { + "entityType": "IP", "fieldMappings": [ { - "columnName": "SE_ipEntity", - "identifier": "Address" + "identifier": "Address", + "columnName": "SE_ipEntity" } - ], - "entityType": "IP" + ] } ] } @@ -932,7 +941,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_IPEntity_SigninLogs_AnalyticalRules Analytics Rule with template version 3.1.0", + "description": "Lumen_IPEntity_SigninLogs_AnalyticalRules Analytics Rule with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('analyticRuleObject7').analyticRuleVersion7]", @@ -986,13 +995,13 @@ ], "entityMappings": [ { + "entityType": "IP", "fieldMappings": [ { - "columnName": "SL_ipEntity", - "identifier": "Address" + "identifier": "Address", + "columnName": "SL_ipEntity" } - ], - "entityType": "IP" + ] } ] } @@ -1048,7 +1057,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_IPEntity_WindowsEvents_AnalyticalRules Analytics Rule with template version 3.1.0", + "description": "Lumen_IPEntity_WindowsEvents_AnalyticalRules Analytics Rule with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('analyticRuleObject8').analyticRuleVersion8]", @@ -1102,13 +1111,13 @@ ], "entityMappings": [ { + "entityType": "IP", "fieldMappings": [ { - "columnName": "WE_ipEntity", - "identifier": "Address" + "identifier": "Address", + "columnName": "WE_ipEntity" } - ], - "entityType": "IP" + ] } ] } @@ -1164,7 +1173,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen_IPIndicator_CommonSecurityLog_HuntingQueries Hunting Query with template version 3.1.0", + "description": "Lumen_IPIndicator_CommonSecurityLog_HuntingQueries Hunting Query with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('huntingQueryObject1').huntingQueryVersion1]", @@ -1249,7 +1258,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Lumen Defender Threat Feed data connector with template version 3.1.0", + "description": "Lumen Defender Threat Feed data connector with template version 3.2.0", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('dataConnectorVersion1')]", @@ -1265,10 +1274,10 @@ "properties": { "connectorUiConfig": { "id": "[variables('_uiConfigId1')]", - "title": "Lumen Defender Threat Feed Data Connector", + "title": "Lumen Defender Threat Feed Data Connector V2", "publisher": "Lumen Technologies, Inc.", - "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://bll-analytics.mss.lumen.com/analytics) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads daily threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.", - "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, and proper configuration of Azure AD authentication components.", + "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://www.lumen.com/en-us/security/black-lotus-labs.html) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.\n\n**NOTE:** This data connector uses the [Azure Functions Flex Consumption Plan](https://learn.microsoft.com/azure/azure-functions/flex-consumption-plan). More pricing details are [here](https://azure.microsoft.com/pricing/details/functions/#pricing).", + "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, and proper configuration of Azure Entra ID authentication components.", "graphQueries": [ { "metricName": "Total Threat Intelligence Indicators", @@ -1351,14 +1360,17 @@ }, "instructionSteps": [ { - "description": ">**NOTE:** This connector uses Azure Functions with Durable Functions to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + "description": ">**NOTE:** This connector uses Azure Functions with the Flex Consumption Plan to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + }, + { + "description": ">**(Optional Step)** Securely store API keys and secrets in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Functions App." }, { "description": "**STEP 1 - Obtain Lumen Defender Threat Feed API Key**\n\n1. [Contact Lumen](mailto:DefenderThreatFeedSales@Lumen.com?subject=API%20Access%20Request) to obtain API access to our Threat Feed API service\n2. Obtain your API key for authentication.", "title": "Configuration" }, { - "description": "**STEP 2 - Configure Azure Entra ID Application and gather information**\n\n1. Create an Entra application. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the Application ID, Tenant ID, and Client Secret\n4. Assign the **Microsoft Sentinel Contributor** role to the application on your Microsoft Sentinel Log Analytics Workspace\n5. Make note of your Workspace ID, as well as the App Insights Workspace Resource ID, which can be obtained from the overview page of the Log Analytics Workspace for your Microsoft Sentinel instance. Click on the “JSON View” link in the top right and the Resource ID will be displayed at the top with a copy button.", + "description": "**STEP 2 - Configure Azure Entra ID Application and gather information**\n\n1. Create a new Entra app registration from the **App registrations** tab in the Entra ID section of the Azure portal. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the **Application ID**, **Tenant ID**, and **Client Secret**\n3. Assign the **Microsoft Sentinel Contributor** role to the newly registered application in the **Access control (IAM)** menu of your Microsoft Sentinel Log Analytics Workspace\n4. Make note of your **Workspace ID**, which can be obtained from the **overview** page of the Log Analytics Workspace for your Microsoft Sentinel instance.", "instructions": [ { "parameters": { @@ -1381,18 +1393,18 @@ ] }, { - "description": "**STEP 3 - Enable the Threat Intelligence Upload Indicators API (Preview) data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." + "description": "**STEP 3 - Enable the **Threat Intelligence Upload Indicators API (Preview)** data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." }, { - "description": "**STEP 4 - Deploy the Azure Function**\n\n**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the Tenant ID, Workspace ID, App Insights Workspace Resource ID, Azure Entra application details (Client ID, Client Secret), and Lumen API key readily available.\n\n1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%20Defender%20Threat%20Feed%2FData%2520Connectors%2FLumenThreatFeed%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction.json)\n\n2. Fill in the appropriate values for each parameter:\n\n- Subscription: Confirm the correct subscription is selected or use the dropdown to change your selection\n- Resource Group: Select the resource group to be used by the Function App and related resources\n- Function Name: Enter a globally unique name with an 11-character limit. Adhere to your organization’s naming convention and ensure the name is globally unique since it is used (along with the uniqueString() function) to identify the ARM template being deployed.\n- Workspace ID: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance and provided for convenience on the connector information page.\n- Lumen API Key: Obtain an API key through Lumen support\n- Lumen Base URL: Filled in automatically and should generally not be changed. This URL contains API endpoints used by the connector\n- Tenant ID: Obtained from the Entra App Registration overview page for the registered application (listed as Directory ID) and can also be obtained from the Tenant Information page in Azure\n- Client ID: Obtained from the Entra App Registration overview page for the registered application (listed as Application ID)\n- Client Secret: Obtained when the secret is created during the app registration process. It can only be viewed when first created and is hidden permanently afterwards. Rerun the app registration process to obtain a new Client Secret if necessary.\n- App Insights Workspace Resource ID: Obtained from the overview page of the Log Analytics Workspace for your Microsoft Sentinel instance. Click on the \"JSON View\" link in the top right and the Resource ID will be displayed at the top with a copy button.\n- Blob Container Name: Use the default name unless otherwise required. Azure Blob Storage is used for temporary storage and processing of threat indicators." + "description": "**STEP 4 - Deploy the Azure Function**\n\n>**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the following information readily available:\n> - Tenant ID and Workspace ID\n> - Azure Entra application details (Client ID, Client Secret)\n> - Lumen API key\n>1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction_v2.json)\n\n2. Fill in the appropriate values for each parameter:\n\n**Basic Settings:**\n- **Subscription**: Confirm the correct subscription is selected or use the dropdown to change your selection\n- **Resource Group**: Select the resource group to be used by the Function App and related resources\n- **Function Name**: Enter a globally unique name for the Function App (11-character limit recommended)\n- **App Insights Workspace Resource ID**: The Resource ID of the Log Analytics Workspace for Application Insights (click **JSON View** on the Log Analytics workspace to copy)\n\n**Lumen API Settings:**\n- **Lumen API Key**: Obtain an API key through Lumen support\n- **Lumen Base URL**: Filled in automatically and should generally not be changed\n- **Confidence Threshold** (Optional): Minimum confidence score (60-100) for indicators (default: 60)\n- **Enable IPv4** (Optional): Enable IPv4 address indicators (default: true)\n- **Enable Domain** (Optional): Enable domain name indicators (default: true)\n\n**Azure Entra ID Settings:**\n- **Workspace ID**: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance\n- **Tenant ID**: Obtained from the Entra App Registration overview page (listed as Directory ID)\n- **Client ID**: Obtained from the Entra App Registration overview page (listed as Application ID)\n- **Client Secret**: Obtained when the secret is created during the app registration process" }, { - "description": "**STEP 5 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. Monitor the Function App logs in the Azure Portal to verify successful execution\n3. After the app performs its first run, review the indicators ingested by either viewing the “Lumen Defender Threat Feed Overview” workbook or viewing the “Threat Intelligence” section in Microsoft Sentinel. In Microsoft Sentinel “Threat Intelligence”, filter for source “Lumen” to display only Lumen generated indicators." + "description": "**STEP 5 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. After the app performs its first run, review the indicators ingested by either viewing the \"Lumen Defender Threat Feed Overview\" workbook or viewing the \"Threat Intelligence\" section in Microsoft Sentinel. In Microsoft Sentinel \"Threat Intelligence\", filter for source \"Lumen\" to display only Lumen generated indicators." } ], "metadata": { - "id": "8d7d3b0e-4f6a-4f2a-bc2d-6d9f5a3c7e21", - "version": "1.0.0", + "id": "8d7d3b0e-4f6a-4f2a-bc2d-6d9f5a3c7e22", + "version": "2.0.0", "kind": "dataConnector", "source": { "kind": "solution", @@ -1444,7 +1456,7 @@ "contentSchemaVersion": "3.0.0", "contentId": "[variables('_dataConnectorContentId1')]", "contentKind": "DataConnector", - "displayName": "Lumen Defender Threat Feed Data Connector", + "displayName": "Lumen Defender Threat Feed Data Connector V2", "contentProductId": "[variables('_dataConnectorcontentProductId1')]", "id": "[variables('_dataConnectorcontentProductId1')]", "version": "[variables('dataConnectorVersion1')]" @@ -1488,9 +1500,9 @@ "kind": "GenericUI", "properties": { "connectorUiConfig": { - "title": "Lumen Defender Threat Feed Data Connector", + "title": "Lumen Defender Threat Feed Data Connector V2", "publisher": "Lumen Technologies, Inc.", - "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://bll-analytics.mss.lumen.com/analytics) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads daily threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.", + "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://www.lumen.com/en-us/security/black-lotus-labs.html) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.\n\n**NOTE:** This data connector uses the [Azure Functions Flex Consumption Plan](https://learn.microsoft.com/azure/azure-functions/flex-consumption-plan). More pricing details are [here](https://azure.microsoft.com/pricing/details/functions/#pricing).", "graphQueries": [ { "metricName": "Total Threat Intelligence Indicators", @@ -1573,14 +1585,17 @@ }, "instructionSteps": [ { - "description": ">**NOTE:** This connector uses Azure Functions with Durable Functions to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + "description": ">**NOTE:** This connector uses Azure Functions with the Flex Consumption Plan to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + }, + { + "description": ">**(Optional Step)** Securely store API keys and secrets in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Functions App." }, { "description": "**STEP 1 - Obtain Lumen Defender Threat Feed API Key**\n\n1. [Contact Lumen](mailto:DefenderThreatFeedSales@Lumen.com?subject=API%20Access%20Request) to obtain API access to our Threat Feed API service\n2. Obtain your API key for authentication.", "title": "Configuration" }, { - "description": "**STEP 2 - Configure Azure Entra ID Application and gather information**\n\n1. Create an Entra application. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the Application ID, Tenant ID, and Client Secret\n4. Assign the **Microsoft Sentinel Contributor** role to the application on your Microsoft Sentinel Log Analytics Workspace\n5. Make note of your Workspace ID, as well as the App Insights Workspace Resource ID, which can be obtained from the overview page of the Log Analytics Workspace for your Microsoft Sentinel instance. Click on the “JSON View” link in the top right and the Resource ID will be displayed at the top with a copy button.", + "description": "**STEP 2 - Configure Azure Entra ID Application and gather information**\n\n1. Create a new Entra app registration from the **App registrations** tab in the Entra ID section of the Azure portal. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the **Application ID**, **Tenant ID**, and **Client Secret**\n3. Assign the **Microsoft Sentinel Contributor** role to the newly registered application in the **Access control (IAM)** menu of your Microsoft Sentinel Log Analytics Workspace\n4. Make note of your **Workspace ID**, which can be obtained from the **overview** page of the Log Analytics Workspace for your Microsoft Sentinel instance.", "instructions": [ { "parameters": { @@ -1603,17 +1618,420 @@ ] }, { - "description": "**STEP 3 - Enable the Threat Intelligence Upload Indicators API (Preview) data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." + "description": "**STEP 3 - Enable the **Threat Intelligence Upload Indicators API (Preview)** data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." }, { - "description": "**STEP 4 - Deploy the Azure Function**\n\n**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the Tenant ID, Workspace ID, App Insights Workspace Resource ID, Azure Entra application details (Client ID, Client Secret), and Lumen API key readily available.\n\n1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%20Defender%20Threat%20Feed%2FData%2520Connectors%2FLumenThreatFeed%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction.json)\n\n2. Fill in the appropriate values for each parameter:\n\n- Subscription: Confirm the correct subscription is selected or use the dropdown to change your selection\n- Resource Group: Select the resource group to be used by the Function App and related resources\n- Function Name: Enter a globally unique name with an 11-character limit. Adhere to your organization’s naming convention and ensure the name is globally unique since it is used (along with the uniqueString() function) to identify the ARM template being deployed.\n- Workspace ID: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance and provided for convenience on the connector information page.\n- Lumen API Key: Obtain an API key through Lumen support\n- Lumen Base URL: Filled in automatically and should generally not be changed. This URL contains API endpoints used by the connector\n- Tenant ID: Obtained from the Entra App Registration overview page for the registered application (listed as Directory ID) and can also be obtained from the Tenant Information page in Azure\n- Client ID: Obtained from the Entra App Registration overview page for the registered application (listed as Application ID)\n- Client Secret: Obtained when the secret is created during the app registration process. It can only be viewed when first created and is hidden permanently afterwards. Rerun the app registration process to obtain a new Client Secret if necessary.\n- App Insights Workspace Resource ID: Obtained from the overview page of the Log Analytics Workspace for your Microsoft Sentinel instance. Click on the \"JSON View\" link in the top right and the Resource ID will be displayed at the top with a copy button.\n- Blob Container Name: Use the default name unless otherwise required. Azure Blob Storage is used for temporary storage and processing of threat indicators." + "description": "**STEP 4 - Deploy the Azure Function**\n\n>**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the following information readily available:\n> - Tenant ID and Workspace ID\n> - Azure Entra application details (Client ID, Client Secret)\n> - Lumen API key\n>1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction_v2.json)\n\n2. Fill in the appropriate values for each parameter:\n\n**Basic Settings:**\n- **Subscription**: Confirm the correct subscription is selected or use the dropdown to change your selection\n- **Resource Group**: Select the resource group to be used by the Function App and related resources\n- **Function Name**: Enter a globally unique name for the Function App (11-character limit recommended)\n- **App Insights Workspace Resource ID**: The Resource ID of the Log Analytics Workspace for Application Insights (click **JSON View** on the Log Analytics workspace to copy)\n\n**Lumen API Settings:**\n- **Lumen API Key**: Obtain an API key through Lumen support\n- **Lumen Base URL**: Filled in automatically and should generally not be changed\n- **Confidence Threshold** (Optional): Minimum confidence score (60-100) for indicators (default: 60)\n- **Enable IPv4** (Optional): Enable IPv4 address indicators (default: true)\n- **Enable Domain** (Optional): Enable domain name indicators (default: true)\n\n**Azure Entra ID Settings:**\n- **Workspace ID**: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance\n- **Tenant ID**: Obtained from the Entra App Registration overview page (listed as Directory ID)\n- **Client ID**: Obtained from the Entra App Registration overview page (listed as Application ID)\n- **Client Secret**: Obtained when the secret is created during the app registration process" }, { - "description": "**STEP 5 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. Monitor the Function App logs in the Azure Portal to verify successful execution\n3. After the app performs its first run, review the indicators ingested by either viewing the “Lumen Defender Threat Feed Overview” workbook or viewing the “Threat Intelligence” section in Microsoft Sentinel. In Microsoft Sentinel “Threat Intelligence”, filter for source “Lumen” to display only Lumen generated indicators." + "description": "**STEP 5 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. After the app performs its first run, review the indicators ingested by either viewing the \"Lumen Defender Threat Feed Overview\" workbook or viewing the \"Threat Intelligence\" section in Microsoft Sentinel. In Microsoft Sentinel \"Threat Intelligence\", filter for source \"Lumen\" to display only Lumen generated indicators." } ], "id": "[variables('_uiConfigId1')]", - "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, and proper configuration of Azure AD authentication components." + "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, and proper configuration of Azure Entra ID authentication components." + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName2')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "Lumen Defender Threat Feed data connector with template version 3.2.0", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion2')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId2'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId2')]", + "title": "Lumen Defender Threat Feed Data Connector V2 (using Azure Functions Flex Consumption Plan with Private Networking)", + "publisher": "Lumen Technologies, Inc.", + "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://www.lumen.com/en-us/security/black-lotus-labs.html) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.\n\n**NOTE:** This data connector uses the [Azure Functions Flex Consumption Plan](https://learn.microsoft.com/azure/azure-functions/flex-consumption-plan) with VNet integration for secure, private network access to storage resources. More pricing details are [here](https://azure.microsoft.com/pricing/details/functions/#pricing).", + "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, proper configuration of Azure Entra ID authentication components, and Virtual Network configuration for private access.", + "graphQueries": [ + { + "metricName": "Total Threat Intelligence Indicators", + "legend": "ThreatIntelIndicators", + "baseQuery": "ThreatIntelIndicators | where SourceSystem == 'Lumen'" + } + ], + "sampleQueries": [ + { + "description": "All Lumen Threat Intelligence Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' | sort by TimeGenerated desc" + }, + { + "description": "High Confidence Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and Confidence >= 80 | sort by TimeGenerated desc" + }, + { + "description": "Malicious IP Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and ObservableValue != '' and ObservableKey == 'ipv4-addr:value' | sort by TimeGenerated desc" + }, + { + "description": "Malicious Domain Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and ObservableValue != '' and ObservableKey == 'domain-name:value' | sort by TimeGenerated desc" + }, + { + "description": "Recent Indicators (Last 7 Days)", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and TimeGenerated > ago(7d) | summarize count() by bin(TimeGenerated, 1d)" + } + ], + "dataTypes": [ + { + "name": "ThreatIntelIndicators(Lumen)", + "lastDataReceivedQuery": "ThreatIntelIndicators | where SourceSystem == 'Lumen' | summarize Time = max(TimeGenerated) | where isnotempty(Time)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "ThreatIntelIndicators | where SourceSystem == 'Lumen' | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": false + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "Read and write permissions on the Log Analytics workspace are required.", + "providerDisplayName": "Log Analytics Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": false + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "Azure Entra App Registration", + "description": "An Azure Entra application registration with the Microsoft Sentinel Contributor role assigned is required for STIX Objects API access. [See the documentation to learn more about Azure Entra applications](https://docs.microsoft.com/azure/active-directory/develop/quickstart-register-app)." + }, + { + "name": "Microsoft Sentinel Contributor Role", + "description": "Microsoft Sentinel Contributor role is required for the Azure Entra application to upload threat intelligence indicators." + }, + { + "name": "Lumen Defender Threat Feed API Key", + "description": "A Lumen Defender Threat Feed API Key is required for accessing threat intelligence data. [Contact Lumen for API access](mailto:DefenderThreatFeedSales@Lumen.com?subject=API%20Access%20Request)." + }, + { + "name": "Virtual Network permissions (for private access)", + "description": "For private storage account access, **Network Contributor** permissions are required on the Virtual Network and subnets. The Function App subnet must be delegated to **Microsoft.App/environments** for Flex Consumption VNet integration." + } + ] + }, + "instructionSteps": [ + { + "description": ">**NOTE:** This connector uses Azure Functions with the Flex Consumption Plan to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. The Flex Consumption Plan enables VNet integration for secure, private network access to storage resources. This might result in additional data ingestion and compute costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + }, + { + "description": ">**(Optional Step)** Securely store API keys and secrets in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Functions App." + }, + { + "description": "**STEP 1 - Network Prerequisites for Private Access**\n\n>**IMPORTANT:** When deploying with private storage account access, **you need a Virtual Network with two properly configured subnets.** You can either use an existing VNet or deploy one using the template below.\n\n**Option A: Deploy a New Virtual Network (Recommended for new deployments)**\n\nUse this template to create a properly configured VNet with two subnets:\n- **Function App Subnet**: Delegated to Microsoft.App/environments for Flex Consumption VNet integration\n- **Private Endpoint Subnet**: For storage account private endpoints\n\n[![Deploy VNet to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_VNet_for_PrivateEndpoint.json)\n\nAfter deployment, note the following output values for use in STEP 5:\n- **VNet Name** (default: lumen-threatfeed-vnet)\n- **VNet Resource Group**\n- **Function App Subnet Name** (default: functionapp-subnet)\n- **Private Endpoint Subnet Name** (default: privateendpoint-subnet)\n\n**Option B: Use an Existing Virtual Network**\n\nIf using an existing VNet, ensure the following requirements are met:\n> - **Virtual Network**: Must be in the same region where you plan to deploy the Function App\n> - **Function App Subnet**: Must be delegated to **Microsoft.App/environments** (required for Flex Consumption Plan)\n> - **Private Endpoint Subnet**: Must NOT be delegated to any service\n> - **Subnet Size**: Minimum /24 recommended for each subnet\n> - **Subnet Delegation**: Configure using one of the following methods:\n> - **Azure Portal**: Virtual networks → Select VNet → Subnets → Select subnet → Delegate to **Microsoft.App/environments**\n> - **Azure CLI**: `az network vnet subnet update --resource-group --vnet-name --name --delegations Microsoft.App/environments`\n\n>**Note:** The connector deployment will automatically create private endpoints for storage services (blob, queue, table, file) and configure Private DNS zones.", + "title": "Configuration" + }, + { + "description": "**STEP 2 - Obtain Lumen Defender Threat Feed API Key**\n\n1. [Contact Lumen](mailto:DefenderThreatFeedSales@Lumen.com?subject=API%20Access%20Request) to obtain API access to our Threat Feed API service\n2. Obtain your API key for authentication." + }, + { + "description": "**STEP 3 - Configure Azure Entra ID Application and gather information**\n\n1. Create a new Entra app registration from the **App registrations** tab in the Entra ID section of the Azure portal. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the **Application ID**, **Tenant ID**, and **Client Secret**\n3. Assign the **Microsoft Sentinel Contributor** role to the newly registered application in the **Access control (IAM)** menu of your Microsoft Sentinel Log Analytics Workspace\n4. Make note of your **Workspace ID**, which can be obtained from the **overview** page of the Log Analytics Workspace for your Microsoft Sentinel instance.", + "instructions": [ + { + "parameters": { + "fillWith": [ + "TenantId" + ], + "label": "Tenant ID" + }, + "type": "CopyableLabel" + }, + { + "parameters": { + "fillWith": [ + "WorkspaceId" + ], + "label": "Workspace ID" + }, + "type": "CopyableLabel" + } + ] + }, + { + "description": "**STEP 4 - Enable the **Threat Intelligence Upload Indicators API (Preview)** data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." + }, + { + "description": "**STEP 5 - Deploy the Azure Function with Private Networking**\n\n>**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the following information readily available:\n> - Tenant ID and Workspace ID\n> - Azure Entra application details (Client ID, Client Secret)\n> - Lumen API key\n> - Virtual Network name and Resource Group\n> - Function App Subnet name (delegated to Microsoft.App/environments)\n> - Private Endpoint Subnet name (non-delegated)\n\n1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction_v2_privateendpoint.json)\n\n2. Fill in the appropriate values for each parameter:\n\n**Basic Settings:**\n- **Subscription**: Confirm the correct subscription is selected or use the dropdown to change your selection\n- **Resource Group**: Select the resource group to be used by the Function App and related resources\n- **Function Name**: Enter a globally unique name for the Function App (11-character limit recommended)\n- **App Insights Workspace Resource ID**: The Resource ID of the Log Analytics Workspace for Application Insights (click **JSON View** on the Log Analytics workspace to copy)\n\n**Lumen API Settings:**\n- **Lumen API Key**: Obtain an API key through Lumen support\n- **Lumen Base URL**: Filled in automatically and should generally not be changed\n- **Confidence Threshold** (Optional): Minimum confidence score (60-100) for indicators (default: 60)\n- **Enable IPv4** (Optional): Enable IPv4 address indicators (default: true)\n- **Enable Domain** (Optional): Enable domain name indicators (default: true)\n\n**Azure Entra ID Settings:**\n- **Workspace ID**: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance\n- **Tenant ID**: Obtained from the Entra App Registration overview page (listed as Directory ID)\n- **Client ID**: Obtained from the Entra App Registration overview page (listed as Application ID)\n- **Client Secret**: Obtained when the secret is created during the app registration process\n\n**Private Networking Settings:**\n- **VNet Resource Group Name**: The resource group containing the Virtual Network (if using the VNet template from STEP 1, this is where you deployed it)\n- **VNet Name**: The name of the Virtual Network (default from VNet template: lumen-threatfeed-vnet)\n- **Function App Subnet Name**: The subnet delegated to Microsoft.App/environments (default from VNet template: functionapp-subnet)\n- **Private Endpoint Subnet Name**: The subnet for private endpoints (default from VNet template: privateendpoint-subnet)\n- **Create Private DNS Zones**: Set to true to create new Private DNS Zones, or false to use existing ones\n\n>**Note:** Ensure the Function App subnet is delegated to Microsoft.App/environments before deployment. The deployment will create private endpoints for storage account services and configure Private DNS zones automatically." + }, + { + "description": "**STEP 6 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. Verify that the Function App is properly integrated with the Virtual Network by checking the Networking settings in the Azure Portal\n3. Confirm that private endpoints were created for the storage account services (blob, file, queue, table)\n4. After the app performs its first run, review the indicators ingested by either viewing the \"Lumen Defender Threat Feed Overview\" workbook or viewing the \"Threat Intelligence\" section in Microsoft Sentinel. In Microsoft Sentinel \"Threat Intelligence\", filter for source \"Lumen\" to display only Lumen generated indicators." + }, + { + "description": "**Troubleshooting Private Networking Issues**\n\nIf the Function App is not receiving data after deployment:\n\n1. **Check VNet Integration**: Navigate to Function App → Networking → VNet integration and verify the Function App subnet is connected\n2. **Verify Private Endpoints**: Navigate to the storage account → Networking → Private endpoint connections and verify all endpoints are in \"Approved\" state\n3. **Check DNS Resolution**: Ensure private DNS zones are properly linked to the VNet for storage account resolution\n4. **Review Function Logs**: Check Application Insights or Function App logs for connection errors\n5. **Subnet Delegation**: Confirm the Function App subnet is delegated to **Microsoft.App/environments** (required for Flex Consumption Plan)" + } + ], + "metadata": { + "id": "8d7d3b0e-4f6a-4f2a-bc2d-6d9f5a3c7e23", + "version": "2.0.0", + "kind": "dataConnector", + "source": { + "kind": "solution", + "name": "Lumen Defender Threat Feed for Microsoft Sentinel" + }, + "author": { + "name": "Lumen Technologies, Inc." + }, + "support": { + "tier": "developer", + "name": "Lumen Technologies, Inc." + } + } + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId2'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId2'))]", + "contentId": "[variables('_dataConnectorContentId2')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion2')]", + "source": { + "kind": "Solution", + "name": "Lumen Defender Threat Feed", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Matthew Collier", + "email": "[variables('_email')]" + }, + "support": { + "name": "Lumen Technologies, Inc.", + "email": "DefenderThreatFeedSupport@lumen.com", + "tier": "Partner", + "link": "https://www.lumen.com/en-us/contact-us/support.html" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "3.0.0", + "contentId": "[variables('_dataConnectorContentId2')]", + "contentKind": "DataConnector", + "displayName": "Lumen Defender Threat Feed Data Connector V2 (using Azure Functions Flex Consumption Plan with Private Networking)", + "contentProductId": "[variables('_dataConnectorcontentProductId2')]", + "id": "[variables('_dataConnectorcontentProductId2')]", + "version": "[variables('dataConnectorVersion2')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId2'),'/'))))]", + "dependsOn": [ + "[variables('_dataConnectorId2')]" + ], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId2'))]", + "contentId": "[variables('_dataConnectorContentId2')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion2')]", + "source": { + "kind": "Solution", + "name": "Lumen Defender Threat Feed", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "Matthew Collier", + "email": "[variables('_email')]" + }, + "support": { + "name": "Lumen Technologies, Inc.", + "email": "DefenderThreatFeedSupport@lumen.com", + "tier": "Partner", + "link": "https://www.lumen.com/en-us/contact-us/support.html" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId2'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "Lumen Defender Threat Feed Data Connector V2 (using Azure Functions Flex Consumption Plan with Private Networking)", + "publisher": "Lumen Technologies, Inc.", + "descriptionMarkdown": "The [Lumen Defender Threat Feed](https://www.lumen.com/en-us/security/black-lotus-labs.html) connector provides the capability to ingest STIX-formatted threat intelligence indicators from Lumen's Black Lotus Labs research team into Microsoft Sentinel. The connector automatically downloads and uploads threat intelligence indicators including IPv4 addresses and domains to the ThreatIntelIndicators table via the STIX Objects Upload API.\n\n**NOTE:** This data connector uses the [Azure Functions Flex Consumption Plan](https://learn.microsoft.com/azure/azure-functions/flex-consumption-plan) with VNet integration for secure, private network access to storage resources. More pricing details are [here](https://azure.microsoft.com/pricing/details/functions/#pricing).", + "graphQueries": [ + { + "metricName": "Total Threat Intelligence Indicators", + "legend": "ThreatIntelIndicators", + "baseQuery": "ThreatIntelIndicators | where SourceSystem == 'Lumen'" + } + ], + "dataTypes": [ + { + "name": "ThreatIntelIndicators(Lumen)", + "lastDataReceivedQuery": "ThreatIntelIndicators | where SourceSystem == 'Lumen' | summarize Time = max(TimeGenerated) | where isnotempty(Time)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "ThreatIntelIndicators | where SourceSystem == 'Lumen' | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "sampleQueries": [ + { + "description": "All Lumen Threat Intelligence Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' | sort by TimeGenerated desc" + }, + { + "description": "High Confidence Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and Confidence >= 80 | sort by TimeGenerated desc" + }, + { + "description": "Malicious IP Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and ObservableValue != '' and ObservableKey == 'ipv4-addr:value' | sort by TimeGenerated desc" + }, + { + "description": "Malicious Domain Indicators", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and ObservableValue != '' and ObservableKey == 'domain-name:value' | sort by TimeGenerated desc" + }, + { + "description": "Recent Indicators (Last 7 Days)", + "query": "ThreatIntelIndicators | where SourceSystem == 'Lumen' and TimeGenerated > ago(7d) | summarize count() by bin(TimeGenerated, 1d)" + } + ], + "availability": { + "status": 1, + "isPreview": false + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "Read and write permissions on the Log Analytics workspace are required.", + "providerDisplayName": "Log Analytics Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": false + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "Azure Entra App Registration", + "description": "An Azure Entra application registration with the Microsoft Sentinel Contributor role assigned is required for STIX Objects API access. [See the documentation to learn more about Azure Entra applications](https://docs.microsoft.com/azure/active-directory/develop/quickstart-register-app)." + }, + { + "name": "Microsoft Sentinel Contributor Role", + "description": "Microsoft Sentinel Contributor role is required for the Azure Entra application to upload threat intelligence indicators." + }, + { + "name": "Lumen Defender Threat Feed API Key", + "description": "A Lumen Defender Threat Feed API Key is required for accessing threat intelligence data. [Contact Lumen for API access](mailto:DefenderThreatFeedSales@Lumen.com?subject=API%20Access%20Request)." + }, + { + "name": "Virtual Network permissions (for private access)", + "description": "For private storage account access, **Network Contributor** permissions are required on the Virtual Network and subnets. The Function App subnet must be delegated to **Microsoft.App/environments** for Flex Consumption VNet integration." + } + ] + }, + "instructionSteps": [ + { + "description": ">**NOTE:** This connector uses Azure Functions with the Flex Consumption Plan to connect to the Lumen Defender Threat Feed API and upload threat intelligence indicators to Microsoft Sentinel via the STIX Objects API. The Flex Consumption Plan enables VNet integration for secure, private network access to storage resources. This might result in additional data ingestion and compute costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." + }, + { + "description": ">**(Optional Step)** Securely store API keys and secrets in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Functions App." + }, + { + "description": "**STEP 1 - Network Prerequisites for Private Access**\n\n>**IMPORTANT:** When deploying with private storage account access, **you need a Virtual Network with two properly configured subnets.** You can either use an existing VNet or deploy one using the template below.\n\n**Option A: Deploy a New Virtual Network (Recommended for new deployments)**\n\nUse this template to create a properly configured VNet with two subnets:\n- **Function App Subnet**: Delegated to Microsoft.App/environments for Flex Consumption VNet integration\n- **Private Endpoint Subnet**: For storage account private endpoints\n\n[![Deploy VNet to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_VNet_for_PrivateEndpoint.json)\n\nAfter deployment, note the following output values for use in STEP 5:\n- **VNet Name** (default: lumen-threatfeed-vnet)\n- **VNet Resource Group**\n- **Function App Subnet Name** (default: functionapp-subnet)\n- **Private Endpoint Subnet Name** (default: privateendpoint-subnet)\n\n**Option B: Use an Existing Virtual Network**\n\nIf using an existing VNet, ensure the following requirements are met:\n> - **Virtual Network**: Must be in the same region where you plan to deploy the Function App\n> - **Function App Subnet**: Must be delegated to **Microsoft.App/environments** (required for Flex Consumption Plan)\n> - **Private Endpoint Subnet**: Must NOT be delegated to any service\n> - **Subnet Size**: Minimum /24 recommended for each subnet\n> - **Subnet Delegation**: Configure using one of the following methods:\n> - **Azure Portal**: Virtual networks → Select VNet → Subnets → Select subnet → Delegate to **Microsoft.App/environments**\n> - **Azure CLI**: `az network vnet subnet update --resource-group --vnet-name --name --delegations Microsoft.App/environments`\n\n>**Note:** The connector deployment will automatically create private endpoints for storage services (blob, queue, table, file) and configure Private DNS zones.", + "title": "Configuration" + }, + { + "description": "**STEP 2 - Obtain Lumen Defender Threat Feed API Key**\n\n1. [Contact Lumen](mailto:DefenderThreatFeedSales@Lumen.com?subject=API%20Access%20Request) to obtain API access to our Threat Feed API service\n2. Obtain your API key for authentication." + }, + { + "description": "**STEP 3 - Configure Azure Entra ID Application and gather information**\n\n1. Create a new Entra app registration from the **App registrations** tab in the Entra ID section of the Azure portal. [See the documentation for a guide to registering an application in Microsoft Entra ID.](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app)\n2. Create a client secret and note the **Application ID**, **Tenant ID**, and **Client Secret**\n3. Assign the **Microsoft Sentinel Contributor** role to the newly registered application in the **Access control (IAM)** menu of your Microsoft Sentinel Log Analytics Workspace\n4. Make note of your **Workspace ID**, which can be obtained from the **overview** page of the Log Analytics Workspace for your Microsoft Sentinel instance.", + "instructions": [ + { + "parameters": { + "fillWith": [ + "TenantId" + ], + "label": "Tenant ID" + }, + "type": "CopyableLabel" + }, + { + "parameters": { + "fillWith": [ + "WorkspaceId" + ], + "label": "Workspace ID" + }, + "type": "CopyableLabel" + } + ] + }, + { + "description": "**STEP 4 - Enable the **Threat Intelligence Upload Indicators API (Preview)** data connector in Microsoft Sentinel**\n\n1. Deploy the **Threat Intelligence (New) Solution**, which includes the **Threat Intelligence Upload Indicators API (Preview)**\n2. Browse to the Content Hub, find and select the **Threat Intelligence (NEW)** solution.\n3. Select the **Install/Update** button." + }, + { + "description": "**STEP 5 - Deploy the Azure Function with Private Networking**\n\n>**IMPORTANT:** Before deploying the Lumen Defender Threat Feed connector, have the following information readily available:\n> - Tenant ID and Workspace ID\n> - Azure Entra application details (Client ID, Client Secret)\n> - Lumen API key\n> - Virtual Network name and Resource Group\n> - Function App Subnet name (delegated to Microsoft.App/environments)\n> - Private Endpoint Subnet name (non-delegated)\n\n1. Click the Deploy to Azure button.\n\n[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FSolutions%2FLumen%2520Defender%2520Threat%2520Feed%2FData%2520Connectors%2FLumenThreatFeedv2%2Fazuredeploy_Connector_LumenThreatFeed_AzureFunction_v2_privateendpoint.json)\n\n2. Fill in the appropriate values for each parameter:\n\n**Basic Settings:**\n- **Subscription**: Confirm the correct subscription is selected or use the dropdown to change your selection\n- **Resource Group**: Select the resource group to be used by the Function App and related resources\n- **Function Name**: Enter a globally unique name for the Function App (11-character limit recommended)\n- **App Insights Workspace Resource ID**: The Resource ID of the Log Analytics Workspace for Application Insights (click **JSON View** on the Log Analytics workspace to copy)\n\n**Lumen API Settings:**\n- **Lumen API Key**: Obtain an API key through Lumen support\n- **Lumen Base URL**: Filled in automatically and should generally not be changed\n- **Confidence Threshold** (Optional): Minimum confidence score (60-100) for indicators (default: 60)\n- **Enable IPv4** (Optional): Enable IPv4 address indicators (default: true)\n- **Enable Domain** (Optional): Enable domain name indicators (default: true)\n\n**Azure Entra ID Settings:**\n- **Workspace ID**: Found in the \"Overview\" tab for the Log Analytics Workspace of the Microsoft Sentinel instance\n- **Tenant ID**: Obtained from the Entra App Registration overview page (listed as Directory ID)\n- **Client ID**: Obtained from the Entra App Registration overview page (listed as Application ID)\n- **Client Secret**: Obtained when the secret is created during the app registration process\n\n**Private Networking Settings:**\n- **VNet Resource Group Name**: The resource group containing the Virtual Network (if using the VNet template from STEP 1, this is where you deployed it)\n- **VNet Name**: The name of the Virtual Network (default from VNet template: lumen-threatfeed-vnet)\n- **Function App Subnet Name**: The subnet delegated to Microsoft.App/environments (default from VNet template: functionapp-subnet)\n- **Private Endpoint Subnet Name**: The subnet for private endpoints (default from VNet template: privateendpoint-subnet)\n- **Create Private DNS Zones**: Set to true to create new Private DNS Zones, or false to use existing ones\n\n>**Note:** Ensure the Function App subnet is delegated to Microsoft.App/environments before deployment. The deployment will create private endpoints for storage account services and configure Private DNS zones automatically." + }, + { + "description": "**STEP 6 - Verify Deployment**\n\n1. The connector polls for indicator updates every 15 minutes.\n2. Verify that the Function App is properly integrated with the Virtual Network by checking the Networking settings in the Azure Portal\n3. Confirm that private endpoints were created for the storage account services (blob, file, queue, table)\n4. After the app performs its first run, review the indicators ingested by either viewing the \"Lumen Defender Threat Feed Overview\" workbook or viewing the \"Threat Intelligence\" section in Microsoft Sentinel. In Microsoft Sentinel \"Threat Intelligence\", filter for source \"Lumen\" to display only Lumen generated indicators." + }, + { + "description": "**Troubleshooting Private Networking Issues**\n\nIf the Function App is not receiving data after deployment:\n\n1. **Check VNet Integration**: Navigate to Function App → Networking → VNet integration and verify the Function App subnet is connected\n2. **Verify Private Endpoints**: Navigate to the storage account → Networking → Private endpoint connections and verify all endpoints are in \"Approved\" state\n3. **Check DNS Resolution**: Ensure private DNS zones are properly linked to the VNet for storage account resolution\n4. **Review Function Logs**: Check Application Insights or Function App logs for connection errors\n5. **Subnet Delegation**: Confirm the Function App subnet is delegated to **Microsoft.App/environments** (required for Flex Consumption Plan)" + } + ], + "id": "[variables('_uiConfigId2')]", + "additionalRequirementBanner": "This connector requires API access to Lumen Defender Threat Feed API service, Microsoft Sentinel STIX Objects API permissions, proper configuration of Azure Entra ID authentication components, and Virtual Network configuration for private access." } } }, @@ -1622,12 +2040,12 @@ "apiVersion": "2023-04-01-preview", "location": "[parameters('workspace-location')]", "properties": { - "version": "3.1.0", + "version": "3.2.0", "kind": "Solution", "contentSchemaVersion": "3.0.0", "displayName": "Lumen Defender Threat Feed", "publisherDisplayName": "Lumen Technologies, Inc.", - "descriptionHtml": "

Note: Please refer to the following before installing the solution:

\n

• Review the solution Release Notes

\n

• There may be known issues pertaining to this Solution, please refer to them before installing.

\n

The Lumen Defender Threat Feed for Microsoft Sentinel solution delivers high-confidence threat intelligence indicators of compromise directly into your Sentinel workspace.

\n

Data Connectors: 1, Workbooks: 1, Analytic Rules: 8, Hunting Queries: 1

\n

Learn more about Microsoft Sentinel | Learn more about Solutions

\n", + "descriptionHtml": "

Note: Please refer to the following before installing the solution:

\n

• Review the solution Release Notes

\n

• There may be known issues pertaining to this Solution, please refer to them before installing.

\n

The Lumen Defender Threat Feed for Microsoft Sentinel solution delivers high-confidence threat intelligence indicators of compromise directly into your Sentinel workspace.

\n

Data Connectors: 2, Workbooks: 1, Analytic Rules: 8, Hunting Queries: 1

\n

Learn more about Microsoft Sentinel | Learn more about Solutions

\n", "contentKind": "Solution", "contentProductId": "[variables('_solutioncontentProductId')]", "id": "[variables('_solutioncontentProductId')]", @@ -1706,11 +2124,16 @@ "kind": "DataConnector", "contentId": "[variables('_dataConnectorContentId1')]", "version": "[variables('dataConnectorVersion1')]" + }, + { + "kind": "DataConnector", + "contentId": "[variables('_dataConnectorContentId2')]", + "version": "[variables('dataConnectorVersion2')]" } ] }, "firstPublishDate": "2025-09-12", - "lastPublishDate": "2025-09-12", + "lastPublishDate": "2026-02-04", "providers": [ "Lumen Technologies, Inc." ], diff --git a/Solutions/Lumen Defender Threat Feed/README.md b/Solutions/Lumen Defender Threat Feed/README.md index 651b452a1dc..3412b2ae478 100644 --- a/Solutions/Lumen Defender Threat Feed/README.md +++ b/Solutions/Lumen Defender Threat Feed/README.md @@ -30,7 +30,7 @@ Lumen Defender Threat Feed for Microsoft Sentinel offers powerful intelligence c ## Solution contents - Data Connector - - `Data Connectors/LumenThreatFeed` (ARM templates + Function App implementation) + - `Data Connectors/LumenThreatFeedv2` (ARM templates + Function App implementation) - Analytic Rules (examples) - `Lumen_DomainEntity_DNS.yaml` - `Lumen_IPEntity_CommonSecurityLog.yaml` diff --git a/Solutions/Lumen Defender Threat Feed/ReleaseNotes.md b/Solutions/Lumen Defender Threat Feed/ReleaseNotes.md index 9522a224338..f3e5429a95e 100644 --- a/Solutions/Lumen Defender Threat Feed/ReleaseNotes.md +++ b/Solutions/Lumen Defender Threat Feed/ReleaseNotes.md @@ -4,3 +4,4 @@ |---------|----------------------------|----------------| | 1.0.0 | 09-12-2025 | Initial Solution Release | | 1.1.0 | 10-23-2025 | Update data connector to utilize more frequent TI object updates and improvements to Workbook | +| 2.0.0 | 02-03-2026 | Deprecated and removed V1.1 Connector. Update V2 data connector for API v3 compatibility: added QUEUED status handling, improved confidence value type conversion for string/integer support, and reordered response handling for new pagination behavior | diff --git a/Solutions/Lumen Defender Threat Feed/SolutionMetadata.json b/Solutions/Lumen Defender Threat Feed/SolutionMetadata.json index 4fca05243eb..5ff027cf95a 100644 --- a/Solutions/Lumen Defender Threat Feed/SolutionMetadata.json +++ b/Solutions/Lumen Defender Threat Feed/SolutionMetadata.json @@ -1,8 +1,8 @@ { "publisherId": "centurylink", - "offerId": "azure-sentinel-solution-lumen-defender-threat-feed", + "offerId": "lumen-defender-threat-feed", "firstPublishDate": "2025-09-12", - "lastPublishDate": "2025-09-12", + "lastPublishDate": "2026-02-04", "providers": ["Lumen Technologies, Inc."], "categories": { "domains" : ["Security - Threat Intelligence"],