Fix IPinfo connectors: runtime pinning, dependency updates, and multi-workspace ASN support

Author: Prasanna92Sekar

Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/constants.py

@@ -24,8 +24,8 @@
 RESOURCE_GROUP_NAME = parts[4]
 WORKSPACE_NAME = parts[8]
 
-DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion"
-ASN_DCR_NAME = "ipinfo_rule_for_ASN_tables"
+DATA_COLLECTION_ENDPOINT_NAME = f"ipinfo-logs-ingestion-{WORKSPACE_NAME.lower()}"
+ASN_DCR_NAME = f"ipinfo_rule_for_ASN_tables_{WORKSPACE_NAME.lower()}"
 ASN_TABLE_NAME = "Ipinfo_ASN_CL"
 ASN_STREAM_DECLARATION = "Custom-Ipinfo_ASN_CL"
 

Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/main.py

@@ -22,7 +22,7 @@ def upload_data_to_ASN_table(dce_endpoint, dcr_immutableid, stream_name):
         client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True)
         mmdb_file_path = "/tmp/asn.mmdb"
         reader = maxminddb.open_database(mmdb_file_path)
-        chunk_size = 10000
+        chunk_size = 100
         data_chunk = []
         logging.info("Uploading Standard ASN Data.\n")
         for ip, ip_data in reader:
@@ -55,7 +55,6 @@ def upload_data_to_ASN_table(dce_endpoint, dcr_immutableid, stream_name):
     access_token = credential.get_token(AZURE_SCOPE).token
     if access_token:
         logging.info("\nAccess Token Retrieved\n")
-        logging.info(access_token)
     else:
         logging.error("\nFailed to retrieve access token\n")
 

Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/utils.py

@@ -109,10 +109,25 @@ def get_data_collection_rule(access_token, data_collection_rule_name):
     url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name)
     headers = {"Authorization": "Bearer " + access_token}
     response = requests.get(url, headers=headers)
+    
     if response.status_code == 200:
         data = response.json()
         immutableId = data["properties"]["immutableId"]
         streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0]
+        
+        try:
+            actual_workspace = data["properties"]["destinations"]["logAnalytics"][0]["workspaceResourceId"]
+            expected_workspace = f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}"
+            
+            if actual_workspace.lower() != expected_workspace.lower():
+                logging.warning(f"DCR '{data_collection_rule_name}' points to WRONG workspace!")
+                logging.warning(f"Expected: {expected_workspace}")
+                logging.warning(f"Found: {actual_workspace}")
+                return None, None  
+        except (KeyError, IndexError) as e:
+            logging.error(f"Failed to extract workspace from DCR: {e}")
+            return None, None
+        
         return immutableId, streamDeclarations
 
     logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}")
@@ -159,9 +174,12 @@ def check_and_create_data_collection_rules(
     access_token, data_collection_rule_name, stream_declaration, columns, endpoint
 ):
     dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name)
+    
     if dcr_immutableid is not None and stream_name is not None:
-        logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.")
+
+        logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists and is valid.")
         return dcr_immutableid, stream_name
-    logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...")
+    
+    logging.info(f"\nCreating Data collection Rule for {data_collection_rule_name}...")
     create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint)
-    return get_data_collection_rule(access_token, data_collection_rule_name)
+    return get_data_collection_rule(access_token, data_collection_rule_name)

Solutions/IPinfo/Data Connectors/ASN/azuredeploy_Connector_IPinfo_ASN_AzureFunction.json

@@ -9,26 +9,34 @@
             "type": "string"
         },
         "RESOURCE_ID": {
-            "type": "string",
-            "defaultValue": "Resouce ID",
+            "type": "string",  
             "metadata": {
-                "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'"
+                "description": "REQUIRED. Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'"
             }
         },
         "TENANT_ID": {
             "type": "string",
-            "defaultValue": "Tenant ID"
+             "metadata": {
+                "description": "REQUIRED. Azure AD Tenant ID (found in Azure Active Directory --> Properties)"
+            }
         },
         "CLIENT_ID": {
             "type": "string",
-            "defaultValue": "Client ID"
+            "metadata": {
+                "description": "REQUIRED. Service Principal Application (Client) ID"
+            }
         },
         "CLIENT_SECRET": {
-            "type": "securestring"
+            "type": "securestring",
+            "metadata": {
+                "description": "REQUIRED. Service Principal Client Secret"
+            }
         },
         "IPINFO_TOKEN": {
             "type": "string",
-            "defaultValue": "IPinfo Token"
+            "metadata": {
+                "description": "REQUIRED. IPinfo API Token (obtain from ipinfo.io)"
+            }    
         },
         "RETENTION_IN_DAYS": {
             "type": "string",
@@ -105,17 +113,14 @@
                 "name": "EP2",
                 "tier": "ElasticPremium",
                 "family": "EP"
-            },
-            "kind": "elastic",
+                },
+            "kind": "linux",  
             "properties": {
                 "name": "[variables('FunctionName')]",
                 "targetWorkerCount": 1,
                 "targetWorkerSizeId": 3,
                 "reserved": true,
-                "maximumElasticWorkerCount": 20,
-                "siteConfig": {
-                    "linuxFxVersion": "python|3.11"
-                }
+                "maximumElasticWorkerCount": 20 
             }
         },
 
@@ -160,16 +165,18 @@
                 "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]",
                 "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]"
             ],
-            "kind": "functionapp",
+            "kind": "functionapp,linux",
             "identity": {
                 "type": "SystemAssigned"
             },
             "properties": {
                 "name": "[variables('FunctionName')]",
                 "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]",
                 "httpsOnly": true,
-                "clientAffinityEnabled": true,
-                "alwaysOn": true
+                "reserved": true,
+                "siteConfig": {                       
+                "linuxFxVersion": "python|3.11"     
+            }
             },
             "resources": [
                 {

Solutions/IPinfo/Data Connectors/ASN/host.json

@@ -11,6 +11,6 @@
     },
     "extensionBundle": {
         "id": "Microsoft.Azure.Functions.ExtensionBundle",
-        "version": "[3.*, 4.0.0)"
+        "version": "[4.0.0, 5.0.0)"
     }
 }

Solutions/IPinfo/Data Connectors/ASN/requirements.txt

@@ -7,3 +7,4 @@ azure.identity
 azure.monitor.ingestion
 requests
 maxminddb
+cffi