wip

Author: c-neto

fluentbit/duplicated-logs/commands.es

@@ -0,0 +1,5 @@
+GET dummy/_search
+
+DELETE dummy
+
+

fluentbit/duplicated-logs/docker-compose.yml

@@ -0,0 +1,36 @@
+services:
+  fluentbit:
+    container_name: fluentbit
+    image: fluent/fluent-bit:latest
+    user: root
+    volumes:
+      - ./fluent-bit.yaml:/fluent-bit/fluent-bit.yaml
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./
+          target: /fluentbit/
+    command:
+      - "fluent-bit"
+      - "-c"
+      - "/fluent-bit/fluent-bit.yaml"
+
+  opensearch:
+    container_name: opensearch
+    image: opensearchproject/opensearch:2.10.0
+    volumes:
+      - ./opensearch.yml:/usr/share/opensearch/config/opensearch.yml
+    ports:
+      - 9200:9200
+    environment:
+      OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
+
+  opensearch-dashboards:
+    container_name: opensearch-dashboards
+    image: opensearchproject/opensearch-dashboards:2.10.0
+    volumes:
+      - ./opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
+    ports:
+      - 5601:5601
+    depends_on:
+      - opensearch

fluentbit/duplicated-logs/fluent-bit.yaml

@@ -0,0 +1,47 @@
+service:
+  flush: 1
+  log_level: error
+
+pipeline:
+  inputs:
+    - name: dummy
+      tag: dummy
+      samples: 3
+      dummy: |
+        {
+          "date": 1750716865.516016,
+          "message": "foobar"
+        }
+
+
+  outputs:
+    - name: stdout
+      format: json_lines
+      match: dummy
+
+    # # logs will be duplicated
+    # - name: opensearch
+    #   match: dummy
+    #   host: opensearch
+    #   port: 9200
+    #   index: dummy
+    #   write_operation: index
+    #   suppress_type_name: true
+    #   time_key: date
+    #   trace_output: true
+    #   trace_error: true
+    #   id_key: log_hash
+
+    # logs doesnt not will be duplicated
+    - name: opensearch
+      match: dummy
+      host: opensearch
+      port: 9200
+      index: dummy
+      write_operation: index
+      suppress_type_name: true
+      time_key: date
+      trace_output: true
+      trace_error: true
+      generate_id: true
+

fluentbit/duplicated-logs/opensearch.yml

@@ -0,0 +1,5 @@
+cluster.name: docker-cluster
+discovery.type: single-node
+network.host: 0.0.0.0
+
+plugins.security.disabled: true

fluentbit/duplicated-logs/opensearch_dashboards.yml

@@ -0,0 +1,8 @@
+# https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/config/opensearch_dashboards.yml
+
+server.port: 5601
+server.host: "0.0.0.0"
+server.ssl.enabled: false
+
+opensearch.hosts:
+  - "http://opensearch:9200"