diff --git a/README.md b/README.md index ead857b..962eeec 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,13 @@ You can find guides per connector: | `AXOROUTER_TLS_MAX_VERSION` | No | - | Maximum TLS version to use | | `AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL` | No | `false` | Whether to include system CA certificates | | `AXOROUTER_TLS_INSECURE_SKIP_VERIFY` | No | `false` | Whether to skip TLS certificate verification | +| `AXOCLOUDCONNECTOR_DEVICE_ID` | Yes | - | A service id that will be used to identify the cloud connector in Axoflow | ### Azure Provider | Variable | Required | Default | Description | |----------|----------|---------|-------------| -| `AZURE_EVENT_HUBS_CONNECTION_STRING` | Yes | - | Azure Event Hub connection string | +| `AZURE_EVENT_HUBS_CONNECTION_STRING` | Yes | - | Azure Event Hubs connection string | ### AWS Provider diff --git a/connectors/aws/README.md b/connectors/aws/README.md index 75b8fb1..8412028 100644 --- a/connectors/aws/README.md +++ b/connectors/aws/README.md @@ -11,12 +11,17 @@ There are many ways you can authenticate and use the CloudWatch service: ### Using AWS Profile with a config file ```bash +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) + docker run --rm \ + -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \ -e AWS_PROFILE="${AWS_PROFILE}" \ -e AWS_REGION="${AWS_REGION}" \ -e AWS_SDK_LOAD_CONFIG=1 \ -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \ -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \ + -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \ -v "${HOME}/.aws:/cloudconnectors/.aws:ro" \ ghcr.io/axoflow/axocloudconnectors:latest ``` @@ -24,22 +29,32 @@ docker run --rm \ ### Direct AWS credentials ```bash +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) + docker run --rm \ + -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \ -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ -e AWS_REGION="${AWS_REGION}" \ -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \ -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \ + -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \ ghcr.io/axoflow/axocloudconnectors:latest ``` ### Using EC2 instance profile ```bash +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) + docker run --rm \ + -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \ -e AWS_REGION="${AWS_REGION}" \ -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \ -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \ + -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \ ghcr.io/axoflow/axocloudconnectors:latest ``` @@ -75,17 +90,22 @@ kubectl create secret generic aws-credentials \ --namespace cloudconnectors \ --dry-run=client -o yaml | kubectl apply -f - +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) + helm upgrade --install --wait --namespace cloudconnectors cloudconnectors ./charts/cloudconnectors \ --set image.repository="axocloudconnectors" \ --set image.tag="dev" \ --set 'env[0].name=AXOROUTER_ENDPOINT' \ --set 'env[0].value=axorouter.axoflow-local.svc.cluster.local:4317' \ - --set 'env[1].name=AWS_REGION' \ - --set 'env[1].value=' \ - --set 'env[2].name=AWS_ACCESS_KEY_ID' \ - --set 'env[2].valueFrom.secretKeyRef.name=aws-credentials' \ - --set 'env[2].valueFrom.secretKeyRef.key=access-key-id' \ - --set 'env[3].name=AWS_SECRET_ACCESS_KEY' \ + --set 'env[1].name=AXOCLOUDCONNECTOR_DEVICE_ID' \ + --set "env[1].value=${AXOCLOUDCONNECTOR_DEVICE_ID}" \ + --set 'env[2].name=AWS_REGION' \ + --set 'env[2].value=' \ + --set 'env[3].name=AWS_ACCESS_KEY_ID' \ --set 'env[3].valueFrom.secretKeyRef.name=aws-credentials' \ - --set 'env[3].valueFrom.secretKeyRef.key=secret-access-key' + --set 'env[3].valueFrom.secretKeyRef.key=access-key-id' \ + --set 'env[4].name=AWS_SECRET_ACCESS_KEY' \ + --set 'env[4].valueFrom.secretKeyRef.name=aws-credentials' \ + --set 'env[4].valueFrom.secretKeyRef.key=secret-access-key' ``` diff --git a/connectors/aws/config.yaml b/connectors/aws/config.yaml index 630f738..a6c8f11 100644 --- a/connectors/aws/config.yaml +++ b/connectors/aws/config.yaml @@ -1,4 +1,51 @@ -receivers: +exporters: + otlp/axorouter: + endpoint: ${env:AXOROUTER_ENDPOINT} + retry_on_failure: + enabled: true + max_elapsed_time: 0 + sending_queue: + enabled: true + storage: file_storage + tls: + insecure: ${env:AXOROUTER_TLS_INSECURE:-false} + ca_file: ${env:AXOROUTER_TLS_CA_FILE} + ca_pem: ${env:AXOROUTER_TLS_CA_PEM} + cert_file: ${env:AXOROUTER_TLS_CERT_FILE} + cert_pem: ${env:AXOROUTER_TLS_CERT_PEM} + key_file: ${env:AXOROUTER_TLS_KEY_FILE} + key_pem: ${env:AXOROUTER_TLS_KEY_PEM} + min_version: ${env:AXOROUTER_TLS_MIN_VERSION:-1.2} + max_version: ${env:AXOROUTER_TLS_MAX_VERSION} + include_system_ca_certs_pool: ${env:AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL:-false} + insecure_skip_verify: ${env:AXOROUTER_TLS_INSECURE_SKIP_VERIFY:-false} + +processors: + resource/axoflow_device_id: + attributes: + - key: "com.axoflow.device_id" + action: insert + value: "${env:AXOCLOUDCONNECTOR_DEVICE_ID}" + + resourcedetection/system: + detectors: ["system", "env"] + system: + hostname_sources: ["dns", "os", "cname", "lookup"] + resource_attributes: + host.name: + enabled: true + host.ip: + enabled: true + host.id: + enabled: true + + resource/axoflow: # Provider specific! + attributes: + - key: "com.axoflow.product" + action: insert + value: "aws-cloudwatch" + +receivers: # Provider specific! awscloudwatch: region: ${env:AWS_REGION} profile: ${env:AWS_PROFILE} @@ -6,15 +53,18 @@ receivers: poll_interval: 10s storage: file_storage -processors: - resource/axoflow: - attributes: - - key: "com.axoflow.product" - action: insert - value: "aws-cloudwatch" + +extensions: + health_check: + endpoint: ${env:POD_IP}:13133 + file_storage: + directory: ${env:STORAGE_DIRECTORY} + create_directory: true service: + extensions: [health_check, file_storage] pipelines: logs: receivers: [awscloudwatch] - processors: [resource/axoflow] + processors: [resource/axoflow_device_id, resourcedetection/system, resource/axoflow] + exporters: [otlp/axorouter] diff --git a/connectors/azure/README.md b/connectors/azure/README.md index 31944d1..043c53d 100644 --- a/connectors/azure/README.md +++ b/connectors/azure/README.md @@ -7,12 +7,16 @@ This directory contains the Axoflow Azure Event Hubs connector which helps colle Make sure the required environment variables are set before running the connector. ```bash +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) + docker run \ --rm \ -v "${STORAGE_DIRECTORY}":"${STORAGE_DIRECTORY}" \ -e AZURE_EVENT_HUBS_CONNECTION_STRING="${AZURE_EVENT_HUBS_CONNECTION_STRING}" \ -e AXOROUTER_ENDPOINT="${AXOROUTER_ENDPOINT}" \ -e STORAGE_DIRECTORY="${STORAGE_DIRECTORY}" \ + -e AXOCLOUDCONNECTOR_DEVICE_ID="${AXOCLOUDCONNECTOR_DEVICE_ID}" \ ghcr.io/axoflow/axocloudconnectors:latest ``` @@ -33,12 +37,17 @@ kubectl create secret generic azure-event-hubs \ --namespace cloudconnectors \ --dry-run=client -o yaml | kubectl apply -f - +UUID_FULL=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || python3 -c "import uuid; print(uuid.uuid4())") +AXOCLOUDCONNECTOR_DEVICE_ID=$(echo "$UUID_FULL" | cut -d'-' -f1) + helm upgrade --install --wait --namespace cloudconnectors cloudconnectors ./charts/cloudconnectors \ --set image.repository="axocloudconnectors" \ --set image.tag="dev" \ --set 'env[0].name=AXOROUTER_ENDPOINT' \ --set 'env[0].value=axorouter.axoflow-local.svc.cluster.local:4317' \ - --set 'env[1].name=AZURE_EVENT_HUBS_CONNECTION_STRING' \ - --set 'env[1].valueFrom.secretKeyRef.name=azure-event-hubs' \ - --set 'env[1].valueFrom.secretKeyRef.key=connection-string' + --set 'env[1].name=AXOCLOUDCONNECTOR_DEVICE_ID' \ + --set "env[1].value=${AXOCLOUDCONNECTOR_DEVICE_ID}" \ + --set 'env[2].name=AZURE_EVENT_HUBS_CONNECTION_STRING' \ + --set 'env[2].valueFrom.secretKeyRef.name=azure-event-hubs' \ + --set 'env[2].valueFrom.secretKeyRef.key=connection-string' ``` diff --git a/connectors/azure/config.yaml b/connectors/azure/config.yaml index b593eae..10092be 100644 --- a/connectors/azure/config.yaml +++ b/connectors/azure/config.yaml @@ -1,19 +1,68 @@ -receivers: - azureeventhub: - connection: ${env:AZURE_EVENT_HUBS_CONNECTION_STRING} - format: azure - apply_semantic_conventions: true - storage: file_storage +exporters: + otlp/axorouter: + endpoint: ${env:AXOROUTER_ENDPOINT} + retry_on_failure: + enabled: true + max_elapsed_time: 0 + sending_queue: + enabled: true + storage: file_storage + tls: + insecure: ${env:AXOROUTER_TLS_INSECURE:-false} + ca_file: ${env:AXOROUTER_TLS_CA_FILE} + ca_pem: ${env:AXOROUTER_TLS_CA_PEM} + cert_file: ${env:AXOROUTER_TLS_CERT_FILE} + cert_pem: ${env:AXOROUTER_TLS_CERT_PEM} + key_file: ${env:AXOROUTER_TLS_KEY_FILE} + key_pem: ${env:AXOROUTER_TLS_KEY_PEM} + min_version: ${env:AXOROUTER_TLS_MIN_VERSION:-1.2} + max_version: ${env:AXOROUTER_TLS_MAX_VERSION} + include_system_ca_certs_pool: ${env:AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL:-false} + insecure_skip_verify: ${env:AXOROUTER_TLS_INSECURE_SKIP_VERIFY:-false} processors: - resource/axoflow: + resource/axoflow_device_id: + attributes: + - key: "com.axoflow.device_id" + action: insert + value: "${env:AXOCLOUDCONNECTOR_DEVICE_ID}" + + resourcedetection/system: + detectors: ["system", "env"] + system: + hostname_sources: ["dns", "os", "cname", "lookup"] + resource_attributes: + host.name: + enabled: true + host.ip: + enabled: true + host.id: + enabled: true + + resource/axoflow: # Provider specific! attributes: - key: "com.axoflow.product" action: insert value: "azure-event-hubs" +receivers: # Provider specific! + azureeventhub: + connection: ${env:AZURE_EVENT_HUBS_CONNECTION_STRING} + format: azure + apply_semantic_conventions: true + storage: file_storage + +extensions: + health_check: + endpoint: ${env:POD_IP}:13133 + file_storage: + directory: ${env:STORAGE_DIRECTORY} + create_directory: true + service: + extensions: [health_check, file_storage] pipelines: logs: receivers: [azureeventhub] - processors: [resource/axoflow] + processors: [resource/axoflow_device_id, resourcedetection/system, resource/axoflow] + exporters: [otlp/axorouter] diff --git a/connectors/common-config.yaml b/connectors/common-config.yaml deleted file mode 100644 index c62f6ca..0000000 --- a/connectors/common-config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -exporters: - otlp/axorouter: - endpoint: ${env:AXOROUTER_ENDPOINT} - retry_on_failure: - enabled: true - max_elapsed_time: 0 - sending_queue: - enabled: true - storage: file_storage - tls: - insecure: ${env:AXOROUTER_TLS_INSECURE:-false} - ca_file: ${env:AXOROUTER_TLS_CA_FILE} - ca_pem: ${env:AXOROUTER_TLS_CA_PEM} - cert_file: ${env:AXOROUTER_TLS_CERT_FILE} - cert_pem: ${env:AXOROUTER_TLS_CERT_PEM} - key_file: ${env:AXOROUTER_TLS_KEY_FILE} - key_pem: ${env:AXOROUTER_TLS_KEY_PEM} - min_version: ${env:AXOROUTER_TLS_MIN_VERSION:-1.2} - max_version: ${env:AXOROUTER_TLS_MAX_VERSION} - include_system_ca_certs_pool: ${env:AXOROUTER_TLS_INCLUDE_SYSTEM_CA_CERTS_POOL:-false} - insecure_skip_verify: ${env:AXOROUTER_TLS_INSECURE_SKIP_VERIFY:-false} - -extensions: - health_check: - endpoint: ${env:POD_IP}:13133 - file_storage: - directory: ${env:STORAGE_DIRECTORY} - create_directory: true - -service: - extensions: [health_check, file_storage] - pipelines: - logs: - exporters: [otlp/axorouter] diff --git a/entrypoint.sh b/entrypoint.sh index c366019..f6153de 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -23,7 +23,7 @@ detect_provider() { if PROVIDER=$(detect_provider); then echo "Detected ${PROVIDER} configuration" - exec ./axoflow-otel-collector --config "/etc/axoflow-otel-collector/connectors/common-config.yaml" --config "/etc/axoflow-otel-collector/connectors/${PROVIDER}/config.yaml" + exec ./axoflow-otel-collector --config "/etc/axoflow-otel-collector/connectors/${PROVIDER}/config.yaml" fi echo "No cloud provider configuration detected. Please set environment variables for one of:" diff --git a/makefile.d/20-tools.mk b/makefile.d/20-tools.mk index d05e16d..f060167 100644 --- a/makefile.d/20-tools.mk +++ b/makefile.d/20-tools.mk @@ -13,7 +13,6 @@ bin/actionlint-$(ACTIONLINT_VERSION): | $(BIN_DIR) | bash -s -- "$(ACTIONLINT_VERSION)" "$(BIN_DIR)" @mv bin/actionlint $@ - #### ## MiniKube ####