feat: add deviceid for host attribution

Signed-off-by: Bence Csati <bence.csati@axoflow.com>

Author: csatib02

README.md

@@ -28,6 +28,7 @@ You can find guides per connector:
 | `AXOROUTER_TLS_MAX_VERSION` | No | - | Maximum TLS version to use |
 | `AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL` | No | `false` | Whether to include system CA certificates |
 | `AXOROUTER_TLS_INSECURE_SKIP_VERIFY` | No | `false` | Whether to skip TLS certificate verification |
+| `AXOCLOUDCONNECTOR_DEVICE_ID` | Yes | - | A service id that will be used to identify the cloud connector in Axoflow |
 
 ### Azure Provider
 

connectors/aws/README.md

@@ -11,35 +11,50 @@ There are many ways you can authenticate and use the CloudWatch service:
 ### Using AWS Profile with a config file
 
 ```bash
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 docker run --rm \
+        -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \
         -e AWS_PROFILE="${AWS_PROFILE}" \
         -e AWS_REGION="${AWS_REGION}" \
         -e AWS_SDK_LOAD_CONFIG=1 \
         -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \
         -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \
+        -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \
         -v "${HOME}/.aws:/cloudconnectors/.aws:ro" \
         ghcr.io/axoflow/axocloudconnectors:latest
 ```
 
 ### Direct AWS credentials
 
 ```bash
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 docker run --rm \
+        -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \
         -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
         -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
         -e AWS_REGION="${AWS_REGION}" \
         -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \
         -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \
+        -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \
         ghcr.io/axoflow/axocloudconnectors:latest
 ```
 
 ### Using EC2 instance profile
 
 ```bash
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 docker run --rm \
+        -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \
         -e AWS_REGION="${AWS_REGION}" \
         -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \
         -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \
+        -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \
         ghcr.io/axoflow/axocloudconnectors:latest
 ```
 
@@ -75,17 +90,22 @@ kubectl create secret generic aws-credentials \
   --namespace cloudconnectors \
   --dry-run=client -o yaml | kubectl apply -f -
 
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 helm upgrade --install --wait --namespace cloudconnectors cloudconnectors ./charts/cloudconnectors \
   --set image.repository="axocloudconnectors" \
   --set image.tag="dev" \
   --set 'env[0].name=AXOROUTER_ENDPOINT' \
   --set 'env[0].value=axorouter.axoflow-local.svc.cluster.local:4317' \
-  --set 'env[1].name=AWS_REGION' \
-  --set 'env[1].value=<YOUR-AWS-REGION>' \
-  --set 'env[2].name=AWS_ACCESS_KEY_ID' \
-  --set 'env[2].valueFrom.secretKeyRef.name=aws-credentials' \
-  --set 'env[2].valueFrom.secretKeyRef.key=access-key-id' \
-  --set 'env[3].name=AWS_SECRET_ACCESS_KEY' \
+  --set 'env[1].name=AXOCLOUDCONNECTOR_DEVICE_ID' \
+  --set 'env[1].value=${AXOCLOUDCONNECTOR_DEVICE_ID}' \
+  --set 'env[2].name=AWS_REGION' \
+  --set 'env[2].value=<YOUR-AWS-REGION>' \
+  --set 'env[3].name=AWS_ACCESS_KEY_ID' \
   --set 'env[3].valueFrom.secretKeyRef.name=aws-credentials' \
-  --set 'env[3].valueFrom.secretKeyRef.key=secret-access-key'
+  --set 'env[3].valueFrom.secretKeyRef.key=access-key-id' \
+  --set 'env[4].name=AWS_SECRET_ACCESS_KEY' \
+  --set 'env[4].valueFrom.secretKeyRef.name=aws-credentials' \
+  --set 'env[4].valueFrom.secretKeyRef.key=secret-access-key'
 ```

connectors/azure/README.md

@@ -7,12 +7,16 @@ This directory contains the Axoflow Azure Event Hubs connector which helps colle
 Make sure the required environment variables are set before running the connector.
 
 ```bash
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 docker run \
         --rm \
         -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \
         -e AZURE_EVENT_HUBS_CONNECTION_STRING="${AZURE_EVENT_HUBS_CONNECTION_STRING}" \
         -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \
         -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \
+        -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \
         ghcr.io/axoflow/axocloudconnectors:latest
 ```
 
@@ -33,12 +37,17 @@ kubectl create secret generic azure-event-hubs \
   --namespace cloudconnectors \
   --dry-run=client -o yaml | kubectl apply -f -
 
+UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())")
+AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1)
+
 helm upgrade --install --wait --namespace cloudconnectors cloudconnectors ./charts/cloudconnectors \
   --set image.repository="axocloudconnectors" \
   --set image.tag="dev" \
   --set 'env[0].name=AXOROUTER_ENDPOINT' \
   --set 'env[0].value=axorouter.axoflow-local.svc.cluster.local:4317' \
-  --set 'env[1].name=AZURE_EVENT_HUBS_CONNECTION_STRING' \
-  --set 'env[1].valueFrom.secretKeyRef.name=azure-event-hubs' \
-  --set 'env[1].valueFrom.secretKeyRef.key=connection-string'
+  --set 'env[1].name=AXOCLOUDCONNECTOR_DEVICE_ID' \
+  --set 'env[1].value=${AXOCLOUDCONNECTOR_DEVICE_ID}' \
+  --set 'env[2].name=AZURE_EVENT_HUBS_CONNECTION_STRING' \
+  --set 'env[2].valueFrom.secretKeyRef.name=azure-event-hubs' \
+  --set 'env[2].valueFrom.secretKeyRef.key=connection-string'
 ```

connectors/common-config.yaml

@@ -20,6 +20,13 @@ exporters:
       include_system_ca_certs_pool: ${env:AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL:-false}
       insecure_skip_verify: ${env:AXOROUTER_TLS_INSECURE_SKIP_VERIFY:-false}
 
+processors:
+  resource/axoflow_device_id:
+    attributes:
+      - key: "com.axoflow.device_id"
+        action: insert
+        value: ${env:AXOCLOUDCONNECTOR_DEVICE_ID}
+
 extensions:
   health_check:
     endpoint: ${env:POD_IP}:13133
@@ -31,4 +38,5 @@ service:
   extensions: [health_check, file_storage]
   pipelines:
     logs:
+      processors: [resource/axoflow_device_id]
       exporters: [otlp/axorouter]

makefile.d/20-tools.mk

@@ -13,7 +13,6 @@ bin/actionlint-$(ACTIONLINT_VERSION): | $(BIN_DIR)
 	| bash -s -- "$(ACTIONLINT_VERSION)" "$(BIN_DIR)"
 	@mv bin/actionlint $@
 
-
 ####
 ## MiniKube
 ####