Merge pull request #14 from axoflow/feat/host-attribution

feat: add deviceid for host attribution

Author: csatib02

README.md

@@ -28,12 +28,13 @@ You can find guides per connector:
 | `AXOROUTER_TLS_MAX_VERSION` | No | - | Maximum TLS version to use |
 | `AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL` | No | `false` | Whether to include system CA certificates |
 | `AXOROUTER_TLS_INSECURE_SKIP_VERIFY` | No | `false` | Whether to skip TLS certificate verification |
+| `AXOCLOUDCONNECTOR_DEVICE_ID` | Yes | - | A service id that will be used to identify the cloud connector in Axoflow |
 
 ### Azure Provider
 
 | Variable | Required | Default | Description |
 |----------|----------|---------|-------------|
-| `AZURE_EVENT_HUBS_CONNECTION_STRING` | Yes | - | Azure Event Hub connection string |
+| `AZURE_EVENT_HUBS_CONNECTION_STRING` | Yes | - | Azure Event Hubs connection string |
 
 ### AWS Provider
 

connectors/aws/README.md

@@ -11,35 +11,50 @@ There are many ways you can authenticate and use the CloudWatch service:
 ### Using AWS Profile with a config file
 
 ```bash
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 docker run --rm \
+        -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \
         -e AWS_PROFILE="${AWS_PROFILE}" \
         -e AWS_REGION="${AWS_REGION}" \
         -e AWS_SDK_LOAD_CONFIG=1 \
         -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \
         -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \
+        -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \
         -v "${HOME}/.aws:/cloudconnectors/.aws:ro" \
         ghcr.io/axoflow/axocloudconnectors:latest
 ```
 
 ### Direct AWS credentials
 
 ```bash
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 docker run --rm \
+        -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \
         -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
         -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
         -e AWS_REGION="${AWS_REGION}" \
         -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \
         -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \
+        -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \
         ghcr.io/axoflow/axocloudconnectors:latest
 ```
 
 ### Using EC2 instance profile
 
 ```bash
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 docker run --rm \
+        -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \
         -e AWS_REGION="${AWS_REGION}" \
         -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \
         -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \
+        -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \
         ghcr.io/axoflow/axocloudconnectors:latest
 ```
 
@@ -75,17 +90,22 @@ kubectl create secret generic aws-credentials \
   --namespace cloudconnectors \
   --dry-run=client -o yaml | kubectl apply -f -
 
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 helm upgrade --install --wait --namespace cloudconnectors cloudconnectors ./charts/cloudconnectors \
   --set image.repository="axocloudconnectors" \
   --set image.tag="dev" \
   --set 'env[0].name=AXOROUTER_ENDPOINT' \
   --set 'env[0].value=axorouter.axoflow-local.svc.cluster.local:4317' \
-  --set 'env[1].name=AWS_REGION' \
-  --set 'env[1].value=<YOUR-AWS-REGION>' \
-  --set 'env[2].name=AWS_ACCESS_KEY_ID' \
-  --set 'env[2].valueFrom.secretKeyRef.name=aws-credentials' \
-  --set 'env[2].valueFrom.secretKeyRef.key=access-key-id' \
-  --set 'env[3].name=AWS_SECRET_ACCESS_KEY' \
+  --set 'env[1].name=AXOCLOUDCONNECTOR_DEVICE_ID' \
+  --set "env[1].value=${AXOCLOUDCONNECTOR_DEVICE_ID}" \
+  --set 'env[2].name=AWS_REGION' \
+  --set 'env[2].value=<YOUR-AWS-REGION>' \
+  --set 'env[3].name=AWS_ACCESS_KEY_ID' \
   --set 'env[3].valueFrom.secretKeyRef.name=aws-credentials' \
-  --set 'env[3].valueFrom.secretKeyRef.key=secret-access-key'
+  --set 'env[3].valueFrom.secretKeyRef.key=access-key-id' \
+  --set 'env[4].name=AWS_SECRET_ACCESS_KEY' \
+  --set 'env[4].valueFrom.secretKeyRef.name=aws-credentials' \
+  --set 'env[4].valueFrom.secretKeyRef.key=secret-access-key'
 ```

connectors/aws/config.yaml

@@ -1,20 +1,70 @@
-receivers:
+exporters:
+  otlp/axorouter:
+    endpoint: ${env:AXOROUTER_ENDPOINT}
+    retry_on_failure:
+      enabled: true
+      max_elapsed_time: 0
+    sending_queue:
+      enabled: true
+      storage: file_storage
+    tls:
+      insecure: ${env:AXOROUTER_TLS_INSECURE:-false}
+      ca_file: ${env:AXOROUTER_TLS_CA_FILE}
+      ca_pem: ${env:AXOROUTER_TLS_CA_PEM}
+      cert_file: ${env:AXOROUTER_TLS_CERT_FILE}
+      cert_pem: ${env:AXOROUTER_TLS_CERT_PEM}
+      key_file: ${env:AXOROUTER_TLS_KEY_FILE}
+      key_pem: ${env:AXOROUTER_TLS_KEY_PEM}
+      min_version: ${env:AXOROUTER_TLS_MIN_VERSION:-1.2}
+      max_version: ${env:AXOROUTER_TLS_MAX_VERSION}
+      include_system_ca_certs_pool: ${env:AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL:-false}
+      insecure_skip_verify: ${env:AXOROUTER_TLS_INSECURE_SKIP_VERIFY:-false}
+
+processors:
+  resource/axoflow_device_id:
+    attributes:
+      - key: "com.axoflow.device_id"
+        action: insert
+        value: "${env:AXOCLOUDCONNECTOR_DEVICE_ID}"
+
+  resourcedetection/system:
+    detectors: ["system", "env"]
+    system:
+      hostname_sources: ["dns", "os", "cname", "lookup"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.id:
+          enabled: true
+
+  resource/axoflow: # Provider specific!
+    attributes:
+      - key: "com.axoflow.product"
+        action: insert
+        value: "aws-cloudwatch"
+
+receivers: # Provider specific!
   awscloudwatch:
     region: ${env:AWS_REGION}
     profile: ${env:AWS_PROFILE}
     logs:
       poll_interval: 10s
     storage: file_storage
 
-processors:
-  resource/axoflow:
-    attributes:
-      - key: "com.axoflow.product"
-        action: insert
-        value: "aws-cloudwatch"
+
+extensions:
+  health_check:
+    endpoint: ${env:POD_IP}:13133
+  file_storage:
+    directory: ${env:STORAGE_DIRECTORY}
+    create_directory: true
 
 service:
+  extensions: [health_check, file_storage]
   pipelines:
     logs:
       receivers: [awscloudwatch]
-      processors: [resource/axoflow]
+      processors: [resource/axoflow_device_id, resourcedetection/system, resource/axoflow]
+      exporters: [otlp/axorouter]

connectors/azure/README.md

@@ -7,12 +7,16 @@ This directory contains the Axoflow Azure Event Hubs connector which helps colle
 Make sure the required environment variables are set before running the connector.
 
 ```bash
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 docker run \
         --rm \
         -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \
         -e AZURE_EVENT_HUBS_CONNECTION_STRING="${AZURE_EVENT_HUBS_CONNECTION_STRING}" \
         -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \
         -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \
+        -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \
         ghcr.io/axoflow/axocloudconnectors:latest
 ```
 
@@ -33,12 +37,17 @@ kubectl create secret generic azure-event-hubs \
   --namespace cloudconnectors \
   --dry-run=client -o yaml | kubectl apply -f -
 
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 helm upgrade --install --wait --namespace cloudconnectors cloudconnectors ./charts/cloudconnectors \
   --set image.repository="axocloudconnectors" \
   --set image.tag="dev" \
   --set 'env[0].name=AXOROUTER_ENDPOINT' \
   --set 'env[0].value=axorouter.axoflow-local.svc.cluster.local:4317' \
-  --set 'env[1].name=AZURE_EVENT_HUBS_CONNECTION_STRING' \
-  --set 'env[1].valueFrom.secretKeyRef.name=azure-event-hubs' \
-  --set 'env[1].valueFrom.secretKeyRef.key=connection-string'
+  --set 'env[1].name=AXOCLOUDCONNECTOR_DEVICE_ID' \
+  --set "env[1].value=${AXOCLOUDCONNECTOR_DEVICE_ID}" \
+  --set 'env[2].name=AZURE_EVENT_HUBS_CONNECTION_STRING' \
+  --set 'env[2].valueFrom.secretKeyRef.name=azure-event-hubs' \
+  --set 'env[2].valueFrom.secretKeyRef.key=connection-string'
 ```

connectors/azure/config.yaml

@@ -1,19 +1,68 @@
-receivers:
-  azureeventhub:
-    connection: ${env:AZURE_EVENT_HUBS_CONNECTION_STRING}
-    format: azure
-    apply_semantic_conventions: true
-    storage: file_storage
+exporters:
+  otlp/axorouter:
+    endpoint: ${env:AXOROUTER_ENDPOINT}
+    retry_on_failure:
+      enabled: true
+      max_elapsed_time: 0
+    sending_queue:
+      enabled: true
+      storage: file_storage
+    tls:
+      insecure: ${env:AXOROUTER_TLS_INSECURE:-false}
+      ca_file: ${env:AXOROUTER_TLS_CA_FILE}
+      ca_pem: ${env:AXOROUTER_TLS_CA_PEM}
+      cert_file: ${env:AXOROUTER_TLS_CERT_FILE}
+      cert_pem: ${env:AXOROUTER_TLS_CERT_PEM}
+      key_file: ${env:AXOROUTER_TLS_KEY_FILE}
+      key_pem: ${env:AXOROUTER_TLS_KEY_PEM}
+      min_version: ${env:AXOROUTER_TLS_MIN_VERSION:-1.2}
+      max_version: ${env:AXOROUTER_TLS_MAX_VERSION}
+      include_system_ca_certs_pool: ${env:AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL:-false}
+      insecure_skip_verify: ${env:AXOROUTER_TLS_INSECURE_SKIP_VERIFY:-false}
 
 processors:
-  resource/axoflow:
+  resource/axoflow_device_id:
+    attributes:
+      - key: "com.axoflow.device_id"
+        action: insert
+        value: "${env:AXOCLOUDCONNECTOR_DEVICE_ID}"
+
+  resourcedetection/system:
+    detectors: ["system", "env"]
+    system:
+      hostname_sources: ["dns", "os", "cname", "lookup"]
+      resource_attributes:
+        host.name:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.id:
+          enabled: true
+
+  resource/axoflow: # Provider specific!
     attributes:
       - key: "com.axoflow.product"
         action: insert
         value: "azure-event-hubs"
 
+receivers: # Provider specific!
+  azureeventhub:
+    connection: ${env:AZURE_EVENT_HUBS_CONNECTION_STRING}
+    format: azure
+    apply_semantic_conventions: true
+    storage: file_storage
+
+extensions:
+  health_check:
+    endpoint: ${env:POD_IP}:13133
+  file_storage:
+    directory: ${env:STORAGE_DIRECTORY}
+    create_directory: true
+
 service:
+  extensions: [health_check, file_storage]
   pipelines:
     logs:
       receivers: [azureeventhub]
-      processors: [resource/axoflow]
+      processors: [resource/axoflow_device_id, resourcedetection/system, resource/axoflow]
+      exporters: [otlp/axorouter]

connectors/common-config.yaml

@@ -23,7 +23,7 @@ detect_provider() {
 
 if PROVIDER=$(detect_provider); then
     echo "Detected ${PROVIDER} configuration"
-    exec ./axoflow-otel-collector --config "/etc/axoflow-otel-collector/connectors/common-config.yaml" --config "/etc/axoflow-otel-collector/connectors/${PROVIDER}/config.yaml"
+    exec ./axoflow-otel-collector --config "/etc/axoflow-otel-collector/connectors/${PROVIDER}/config.yaml"
 fi
 
 echo "No cloud provider configuration detected. Please set environment variables for one of:"

entrypoint.sh

@@ -13,7 +13,6 @@ bin/actionlint-$(ACTIONLINT_VERSION): | $(BIN_DIR)
 	| bash -s -- "$(ACTIONLINT_VERSION)" "$(BIN_DIR)"
 	@mv bin/actionlint $@
 
-
 ####
 ## MiniKube
 ####