add vector module

Author: Friz-zy

README.md

@@ -18,7 +18,7 @@ cd example
 terraform init
 terraform apply -target=module.vpc
 terraform apply
-terraform output output
+terraform output all
 ```
 
 ## Security

example/main.tf

@@ -154,6 +154,7 @@ module "eks" {
   enable_victoriametrics          = true
   enable_grafana                  = true
   enable_uptrace                  = true
+  enable_vector_agent             = true
   enable_qryn                     = false
   enable_openobserve              = false
   enable_openobserve_collector    = false

example/outputs.tf

@@ -1,4 +1,4 @@
-output "output" {
+output "all" {
   value     = module.eks
   sensitive = true
 }

main.tf

@@ -668,8 +668,6 @@ module "qryn" {
   )
 }
 
-
-
 module "openobserve" {
   source = "./modules/openobserve"
   count = var.enable_openobserve ? 1 : 0
@@ -720,6 +718,83 @@ module "openobserve_collector" {
   )
 }
 
+# https://medium.com/ibm-cloud/log-collectors-performance-benchmarking-8c5218a08fea
+# vector is still buggy =\ https://github.com/vectordotdev/vector/issues/12014
+module "vector_agent" {
+  source = "./modules/vector"
+  count = var.enable_vector_agent ? 1 : 0
+
+  create        = var.enable_vector_agent
+  chart_version = var.vector_agent_chart_version
+  namespace     = var.vector_agent_namespace
+  set           = var.vector_agent_set
+  tags          = var.tags
+
+  values = concat(
+    [templatefile("${path.module}/universal_values.yaml", {})],
+    [
+    <<-EOT
+      role: "Agent"
+
+      # customConfig -- Override Vector's default configs, if used **all** options need to be specified. This section supports
+      # using helm templates to populate dynamic values. See Vector's [configuration documentation](https://vector.dev/docs/reference/configuration/)
+      # for all options.
+      customConfig:
+        data_dir: /vector-data-dir
+        api:
+          enabled: true
+          address: 127.0.0.1:8686
+          playground: false
+
+        # https://vector.dev/docs/reference/configuration/sources/
+        sources:
+          kubernetes_logs:
+            type: kubernetes_logs
+          internal_metrics:
+            type: internal_metrics
+
+        # https://vector.dev/docs/reference/configuration/transforms/
+        transforms: {}
+
+        # https://vector.dev/docs/reference/configuration/sinks/
+        sinks:
+          prom_exporter:
+            type: prometheus_exporter
+            inputs: [internal_metrics]
+            address: 0.0.0.0:9090
+          %{ if var.enable_uptrace == true }
+          uptrace:
+            type: "http"
+            method: "post"
+            inputs:
+              - kubernetes_logs
+            encoding:
+              codec: "json"
+            framing:
+              method: "newline_delimited"
+            compression: "gzip"
+            request:
+              headers:
+                uptrace-dsn: "http://${module.uptrace[0].project_tokens[1]}@${module.uptrace[0].chart.uptrace}.${module.uptrace[0].namespace.uptrace}.svc:14318/2?grpc=14317"
+            uri: "http://${module.uptrace[0].chart.uptrace}.${module.uptrace[0].namespace.uptrace}.svc:14318/api/v1/vector/logs"
+          %{ endif }
+          %{ if var.enable_qryn == true }
+          qryn:
+            type: "loki"
+            inputs:
+              - kubernetes_logs
+            endpoint: "http://${module.qryn[0].chart.qryn}.${module.qryn[0].namespace.qryn}.svc:3100"
+            auth:
+              strategy: "basic"
+              password: "${module.qryn[0].root_password}"
+              user: "${var.admin_email}"
+          %{ endif }
+    EOT
+    ],
+    var.vector_agent_values
+  )
+}
+
 # DASHBOARD
 
 module "kubernetes_dashboard" {

modules/apisix/main.tf

@@ -4,12 +4,12 @@
 
 resource "random_password" "apisix_admin_key" {
   length           = 32
-  special          = true
+  special          = false
 }
 
 resource "random_password" "apisix_viewer_key" {
   length           = 32
-  special          = true
+  special          = false
 }
 
 module "ingress_apisix" {

modules/grafana/main.tf

@@ -17,7 +17,7 @@ locals {
 
 resource "random_password" "grafana_admin_password" {
   length           = 32
-  special          = true
+  special          = false
 }
 
 module "kubernetes_manifests" {
@@ -40,15 +40,6 @@ module "kubernetes_manifests" {
           username: "${var.admin_user}"
           password: "${local.admin_password}"
         type: Opaque
-      - kind: Secret
-        apiVersion: v1
-        metadata:
-          name: "grafana-admin-credentials"
-          namespace: "${var.namespace}"
-        stringData:
-          username: "${var.admin_user}"
-          password: "${local.admin_password}"
-        type: Opaque
     EOT
   ]
 }

modules/openobserve/main.tf

@@ -33,7 +33,7 @@ locals {
 
 resource "random_password" "openobserve_root_password" {
   length           = 32
-  special          = true
+  special          = false
 }
 
 # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket

modules/opentelemetry-operator/main.tf

@@ -8,10 +8,15 @@ module "opentelemetry_operator" {
         name  = "admissionWebhooks.certManager.enabled"
         value = "false"
       },
-      {
-        name  = "admissionWebhooks.certManager.autoGenerateCert.enabled"
-        value = "true"
-      },
+
+# Error: values don't meet the specifications of the schema(s) in the following chart(s):
+# │ opentelemetry-operator:
+# │ - admissionWebhooks.certManager: Additional property autoGenerateCert is not allowed
+#       {
+#         name  = "admissionWebhooks.certManager.autoGenerateCert.enabled"
+#         value = "true"
+#       },
+
     ],
     var.set
   )

modules/qryn/main.tf

@@ -94,7 +94,7 @@ locals {
 
 resource "random_password" "qryn_root_password" {
   length           = 32
-  special          = true
+  special          = false
 }
 
 resource "random_password" "clickhouse_password" {

modules/uptrace/main.tf

@@ -99,6 +99,31 @@ locals {
             # Overrides public URL for Vue-powered UI.
             #addr: 'https://uptrace.mydomain.com'
 
+          spans:
+            # The size of the Go chan used to buffer incoming spans.
+            # If the buffer is full, Uptrace starts to drop spans.
+            buffer_size: 100000
+
+            # The number of spans to insert in a single query.
+            #batch_size: 10000
+
+          metrics:
+            # List of attributes to drop for being noisy.
+            drop_attrs:
+              - telemetry_sdk_language
+              - telemetry_sdk_name
+              - telemetry_sdk_version
+
+            # The size of the Go chan used to buffer incoming measures.
+            # If the buffer is full, Uptrace starts to drop measures.
+            buffer_size: 100000
+
+            # The number of measures to insert in a single query.
+            #batch_size: 10000
+
+            # The size of the buffer for converting cumulative metrics to delta.
+            #cum_to_delta_size: 100000
+
           ## SMTP settings to send emails.
           ## https://uptrace.dev/get/alerting.html
           smtp_mailer:
@@ -267,18 +292,18 @@ locals {
 
 resource "random_password" "uptrace_root_password" {
   length           = 32
-  special          = true
+  special          = false
 }
 
 resource "random_password" "uptrace_project_tokens" {
   count = 2
   length           = 32
-  special          = true
+  special          = false
 }
 
 resource "random_password" "uptrace_secret_key" {
   length           = 32
-  special          = true
+  special          = false
 }
 
 resource "random_password" "clickhouse_password" {
@@ -603,9 +628,9 @@ module "grafana_operator_datasource" {
           namespace: "${var.grafana_operator_namespace}"
         stringData:
           token1: "${random_password.uptrace_project_tokens[0].result}"
-          header1: "http://${random_password.uptrace_project_tokens[0].result}@${module.uptrace.chart.uptrace}.${module.uptrace.namespace.uptrace}.svc:14318/1?grpc=14317"
+          header1: "http://${random_password.uptrace_project_tokens[0].result}@${module.uptrace.chart}.${module.uptrace.namespace}.svc:14318/1?grpc=14317"
           token2: "${random_password.uptrace_project_tokens[1].result}"
-          header2: "http://${random_password.uptrace_project_tokens[1].result}@${module.uptrace.chart.uptrace}.${module.uptrace.namespace.uptrace}.svc:14318/2?grpc=14317"
+          header2: "http://${random_password.uptrace_project_tokens[1].result}@${module.uptrace.chart}.${module.uptrace.namespace}.svc:14318/2?grpc=14317"
         type: Opaque
       - apiVersion: grafana.integreatly.org/v1beta1
         kind: GrafanaDatasource
@@ -625,7 +650,7 @@ module "grafana_operator_datasource" {
             name: "Uptrace-${var.namespace}"
             type: prometheus
             access: proxy
-            url: "http://${module.uptrace.chart.uptrace}.${module.uptrace.namespace.uptrace}.svc:14318/api/prometheus"
+            url: "http://${module.uptrace.chart}.${module.uptrace.namespace}.svc:14318/api/prometheus"
             isDefault: false
             jsonData:
               "httpHeaderName1": "uptrace-dsn"
@@ -652,7 +677,7 @@ module "grafana_operator_datasource" {
             name: "Uptrace-Tempo-${var.namespace}"
             type: tempo
             access: proxy
-            url: "http://${module.uptrace.chart.uptrace}.${module.uptrace.namespace.uptrace}.svc:14318/api/tempo"
+            url: "http://${module.uptrace.chart}.${module.uptrace.namespace}.svc:14318/api/tempo"
             isDefault: false
             jsonData:
               "httpHeaderName1": "uptrace-dsn"
@@ -679,7 +704,7 @@ module "grafana_operator_datasource" {
             name: "Uptrace-Monitoring-${var.namespace}"
             type: prometheus
             access: proxy
-            url: "http://${module.uptrace.chart.uptrace}.${module.uptrace.namespace.uptrace}.svc:14318/api/prometheus"
+            url: "http://${module.uptrace.chart}.${module.uptrace.namespace}.svc:14318/api/prometheus"
             isDefault: false
             jsonData:
               "httpHeaderName1": "uptrace-dsn"
@@ -706,7 +731,7 @@ module "grafana_operator_datasource" {
             name: "Uptrace-Monitoring-Tempo-${var.namespace}"
             type: tempo
             access: proxy
-            url: "http://${module.uptrace.chart.uptrace}.${module.uptrace.namespace.uptrace}.svc:14318/api/tempo"
+            url: "http://${module.uptrace.chart}.${module.uptrace.namespace}.svc:14318/api/tempo"
             isDefault: false
             jsonData:
               "httpHeaderName1": "uptrace-dsn"