Merge pull request #169 from sysdiglabs/staging

daviddetorres · web-flow · commit 9eb469bf5632 · 2021-07-19T14:41:25.000+03:00
July's deploy in production
diff --git a/resources/apache/INSTALL.md b/resources/apache/INSTALL.md
@@ -79,17 +79,8 @@ kubectl apply -f apache-deploy.yaml
 
 # Configuring Sysdig Agent
 
-Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml). In the configuration file, add the following code to include the 'app' label as a metric label.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-        tags:
-          app: "{kubernetes.pod.label.app}"
-```
+Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml). By default the job will be in the agent so you don't need to change anything but if you don't have it here you have the way to do that
+
 Apply the changes:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
diff --git a/resources/apache/include/apache-deploy.yaml b/resources/apache/include/apache-deploy.yaml
@@ -12,8 +12,8 @@ spec:
   template:
     metadata:
       annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "9222"
+        promcat.sysdig.com/port: "9222"
+        promcat.sysdig.com/integration_type: "apache"
       labels:
         app: apache
     spec:
diff --git a/resources/apache/include/sysdig-agent-config.yaml b/resources/apache/include/sysdig-agent-config.yaml
@@ -1,34 +1,66 @@
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sysdig-agent
+  namespace: sysdig-agent
 data:
   dragent.yaml: |
     new_k8s: true
-    k8s_cluster_name: your-cluster-name
-    jmx:
-      enabled: false
+    k8s_cluster_name: YourClusterName
     metrics_excess_log: true
-    app_checks_enabled: false
     10s_flush_enable: true
+    app_checks_enabled: false
     use_promscrape: true
+    new_k8s: true
+    promscrape_fastproto: true
     prometheus:
       enabled: true
-      histograms: true
+      prom_service_discovery: true
+      log_errors: true
+      max_metrics: 200000
+      max_metrics_per_process: 200000
+      max_tags_per_metric: 100
       ingest_raw: true
       ingest_calculated: false
-      interval: 10
-      log_errors: true
-      max_metrics: 3000
-      max_metrics_per_process: 20000
-      process_filter:
-      - include:
-          kubernetes.pod.annotation.prometheus.io/scrape: true
-          conf:
-              path: "{kubernetes.pod.annotation.prometheus.io/path}"
-              port: "{kubernetes.pod.annotation.prometheus.io/port}"
     snaplen: 512
     tags: role:cluster
-kind: ConfigMap
-metadata:
-    labels:
-      app: sysdig-agent
-    name: sysdig-agent
-    namespace: sysdig-agent
+  prometheus.yaml: |
+    global:
+      scrape_interval: 10s
+    scrape_configs:
+    - job_name: apache-default
+      tls_config:
+        insecure_skip_verify: true
+      kubernetes_sd_configs:
+      - role: pod
+      relabel_configs:
+      - action: keep
+        source_labels: [__meta_kubernetes_pod_host_ip]
+        regex: __HOSTIPS__
+      - action: keep
+        source_labels:
+        - __meta_kubernetes_pod_annotation_promcat_sysdig_com_integration_type
+        regex: "apache"
+      - action: replace
+        source_labels: [__meta_kubernetes_pod_annotation_promcat_sysdig_com_target_ns]
+        target_label: kube_namespace_name
+      - action: replace
+        source_labels: [__meta_kubernetes_pod_annotation_promcat_sysdig_com_target_workload_type]
+        target_label: kube_workload_type
+      - action: replace
+        source_labels: [__meta_kubernetes_pod_annotation_promcat_sysdig_com_target_workload_name]
+        target_label: kube_workload_name
+      - action: replace
+        replacement: true
+        target_label: sysdig_omit_source
+      - action: replace
+        source_labels: [__address__, __meta_kubernetes_pod_annotation_promcat_sysdig_com_port]
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+      - action: replace
+        source_labels: [__meta_kubernetes_pod_uid]
+        target_label: sysdig_k8s_pod_uid
+      - action: replace
+        source_labels: [__meta_kubernetes_pod_container_name]
+        target_label: sysdig_k8s_pod_container_name
diff --git a/resources/aws-alb/INSTALL.md b/resources/aws-alb/INSTALL.md
@@ -68,17 +68,7 @@ Do the following:
 
 1. In the yace deployment, include the Prometheus annotations. Add the port of the exporter as the scraping port in the annotation.    
 
-2. In the Sysdig Agent configuration, add the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
-3. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
+2. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
 ```
diff --git a/resources/aws-ebs/INSTALL.md b/resources/aws-ebs/INSTALL.md
@@ -67,16 +67,6 @@ Do the following:
 
 1. In the yace deployment, include the Prometheus annotations. Add the port of the exporter as the scraping port in the annotation.    
 
-2. In the Sysdig Agent configuration, add the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
-3. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
+2. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
diff --git a/resources/aws-elb/INSTALL.md b/resources/aws-elb/INSTALL.md
@@ -68,17 +68,7 @@ Do the following:
 
 1. In the yace deployment, include the Prometheus annotations. Add the port of the exporter as the scraping port in the annotation.    
 
-2. In the Sysdig Agent configuration, add the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
-3. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
+2. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
 ```
diff --git a/resources/aws-fargate/INSTALL.md b/resources/aws-fargate/INSTALL.md
@@ -66,17 +66,7 @@ Do the following:
 
 1. In the yace deployment, include the Prometheus annotations. Add the port of the exporter as the scraping port in the annotation.    
 
-2. In the Sysdig Agent configuration, add the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
-3. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply the changes by:
+2. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply the changes by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
 ```
diff --git a/resources/aws-lambda/INSTALL.md b/resources/aws-lambda/INSTALL.md
@@ -66,17 +66,7 @@ Do the following:
 
 1. In the yace deployment, include the Prometheus annotations. Add the port of the exporter as the scraping port in the annotation.    
 
-2. In the Sysdig Agent configuration, add the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
-3. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
+2. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
 ```
diff --git a/resources/aws-rds/INSTALL.md b/resources/aws-rds/INSTALL.md
@@ -67,17 +67,7 @@ Do the following:
 
 1. In the yace deployment, include the Prometheus annotations. Add the port of the exporter as the scraping port in the annotation.    
 
-2. In the Sysdig Agent configuration, add the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
-3. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
+2. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
 ```
diff --git a/resources/aws-s3/INSTALL.md b/resources/aws-s3/INSTALL.md
@@ -79,17 +79,7 @@ Do the following:
 
 1. In the yace deployment, include the Prometheus annotations. Add the port of the exporter as the scraping port in the annotation.    
 
-2. In the Sysdig Agent configuration, add the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
-3. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
+2. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
 ```
diff --git a/resources/aws-sqs/INSTALL.md b/resources/aws-sqs/INSTALL.md
@@ -67,17 +67,7 @@ Do the following:
 
 1. In the yace deployment, include the Prometheus annotations. Add the port of the exporter as the scraping port in the annotation.    
 
-2. In the Sysdig Agent configuration, add the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
-3. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
+2. Download the sample [Sysdig Agent configuration file](include/sysdig-agent-config.yaml) and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
 ```
diff --git a/resources/elasticsearch/INSTALL.md b/resources/elasticsearch/INSTALL.md
@@ -68,17 +68,15 @@ spec:
     metadata:
       annotations:
         promcat.sysdig.com/port: "9108"
-
         # Add here the namespace, workload type (deployment, statefulset, replicaset, daemonset) 
         # and workload name of the instance that the exporter will take data from
         promcat.sysdig.com/target_ns: elastic-namespace
         promcat.sysdig.com/target_workload_type: statefulset
         promcat.sysdig.com/target_workload_name: elasticsearch
+        promcat.sysdig.com/integration_type: elasticsearch
 ```
 This way, on the Sysdig Monitor, you can view the associated metrics corresponding to the database pods and the exporter.
 
-
-
 After you configure the Sysdig annotations, download the sample configuration file and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
diff --git a/resources/elasticsearch/include/exporter_no_credentials.yaml b/resources/elasticsearch/include/exporter_no_credentials.yaml
@@ -21,6 +21,7 @@ spec:
         promcat.sysdig.com/target_ns: elastic-namespace
         promcat.sysdig.com/target_workload_type: statefulset
         promcat.sysdig.com/target_workload_name: elasticsearch
+        promcat.sysdig.com/integration_type: elasticsearch
       labels:
         app: elasticsearch-exporter
     spec:
@@ -40,7 +41,7 @@ spec:
         env:
           - name: ES_URI
             value: http://elasticsearch:9200
-        image: quay.io/sysdig/elasticsearch-exporter:latest
+        image: quay.io/sysdig/elasticsearch-exporter:1.2.1
         imagePullPolicy: IfNotPresent
         lifecycle:
           preStop:
diff --git a/resources/elasticsearch/include/exporter_with_credentials.yaml b/resources/elasticsearch/include/exporter_with_credentials.yaml
@@ -22,6 +22,7 @@ spec:
         promcat.sysdig.com/target_ns: elastic-namespace
         promcat.sysdig.com/target_workload_type: statefulset
         promcat.sysdig.com/target_workload_name: elasticsearch
+        promcat.sysdig.com/integration_type: elasticsearch
       labels:
         app: elasticsearch-exporter
     spec:
@@ -50,7 +51,7 @@ spec:
         envFrom:
         - secretRef:
             name: elasticsearch-tls
-        image: quay.io/sysdig/elasticsearch-exporter:latest
+        image: quay.io/sysdig/elasticsearch-exporter:1.2.1
         imagePullPolicy: IfNotPresent
         lifecycle:
           preStop:
diff --git a/resources/elasticsearch/include/sysdig-agent.yaml b/resources/elasticsearch/include/sysdig-agent.yaml
@@ -1,4 +1,8 @@
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sysdig-agent
+  namespace: sysdig-agent
 data:
   dragent.yaml: |
     new_k8s: true
@@ -35,13 +39,8 @@ data:
         regex: __HOSTIPS__
       - action: keep
         source_labels:
-        - __meta_kubernetes_namespace
-        - __meta_kubernetes_pod_name
-        separator: '/'
-
-        # Add here the namespace/pod-name of your elasticsearch exporter instance
-        regex: 'elastic-namespace/elasticsearch-exporter-.+'
-
+        - __meta_kubernetes_pod_annotation_promcat_sysdig_com_integration_type
+        regex: "elasticsearch"
       - action: replace
         source_labels: [__address__, __meta_kubernetes_pod_annotation_promcat_sysdig_com_port]
         regex: ([^:]+)(?::\d+)?;(\d+)
@@ -65,11 +64,7 @@ data:
       - action: replace
         source_labels: [__meta_kubernetes_pod_container_name]
         target_label: sysdig_k8s_pod_container_name
-
-
-kind: ConfigMap
-metadata:
-    labels:
-      app: sysdig-agent
-    name: sysdig-agent
-    namespace: sysdig-agent
+      metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: (elasticsearch_cluster_health_active_primary_shards|elasticsearch_cluster_health_active_shards|elasticsearch_cluster_health_initializing_shards|elasticsearch_cluster_health_number_of_data_nodes|elasticsearch_cluster_health_number_of_nodes|elasticsearch_cluster_health_number_of_pending_tasks|elasticsearch_cluster_health_relocating_shards|elasticsearch_cluster_health_status|elasticsearch_cluster_health_unassigned_shards|elasticsearch_filesystem_data_available_bytes|elasticsearch_filesystem_data_size_bytes|elasticsearch_indices_indexing_index_time_seconds_total|elasticsearch_indices_indexing_index_total|elasticsearch_indices_merges_total_time_seconds_total|elasticsearch_indices_search_query_time_seconds|elasticsearch_indices_store_throttle_time_seconds_total|elasticsearch_jvm_gc_collection_seconds_count|elasticsearch_jvm_gc_collection_seconds_sum|elasticsearch_jvm_memory_committed_bytes|elasticsearch_jvm_memory_max_bytes|elasticsearch_jvm_memory_max_bytes  |elasticsearch_jvm_memory_pool_peak_used_bytes|elasticsearch_jvm_memory_used_bytes|elasticsearch_os_load1|elasticsearch_os_load15|elasticsearch_os_load5|elasticsearch_process_cpu_percent|elasticsearch_transport_rx_size_bytes_total|elasticsearch_transport_tx_size_bytes_total)
+        action: keep
diff --git a/resources/haproxy-ingress/README.md b/resources/haproxy-ingress/README.md
@@ -10,7 +10,7 @@ The HAProxy ingress instruments Prometheus metrics on the following scopes:
 - Backend
 - Server (disabled in the configuration provided)
 
-For a complete list of the metrics exported, see the [HAProxy exporter documentation](https://github.com/haproxy/haproxy/blob/master/contrib/prometheus-exporter/README)
+For a complete list of the metrics exported, see the [HAProxy exporter documentation](https://github.com/haproxy/haproxy/blob/master/addons/promex/README)
 
 # Number of time series generated
 Usually, for the configuration provided, the number of time series is approximately:
diff --git a/resources/mongodb/INSTALL.md b/resources/mongodb/INSTALL.md
@@ -91,16 +91,6 @@ kubectl apply -f mongodb-exporter-auth-deploy.yaml
 # SYSDIG AGENT CONFIGURATION
 In the _values.yaml_ of the Helm chart, include the Prometheus annotations to configure the port of the exporter as the scraping port.    
 
-Also, ensure that the Sysdig Agent configuration has the following lines of configuration to scrape the containers with Prometheus annotations.
-```yaml
-process_filter:
-  - include:
-      kubernetes.pod.annotation.prometheus.io/scrape: true
-      conf:
-        path: "{kubernetes.pod.annotation.prometheus.io/path}"
-        port: "{kubernetes.pod.annotation.prometheus.io/port}"
-```
-
 You can download the sample configuration file and apply it by:
 ```bash
 kubectl apply -f sysdig-agent-config.yaml
diff --git a/resources/nginx-ingress/include/sysdig-agent.yaml b/resources/nginx-ingress/include/sysdig-agent.yaml
diff --git a/resources/nginx/include/configuration.yaml b/resources/nginx/include/configuration.yaml
diff --git a/resources/nginx/include/sysdig-agent-config.yaml b/resources/nginx/include/sysdig-agent-config.yaml
diff --git a/resources/oracle-db/INSTALL.md b/resources/oracle-db/INSTALL.md
diff --git a/resources/redis/include/redis-deploy.yaml b/resources/redis/include/redis-deploy.yaml
diff --git a/resources/redis/include/sysdig-agent-config.yaml b/resources/redis/include/sysdig-agent-config.yaml
