wip

Author: c-neto

fluentbit/docker-compose.yml

@@ -8,18 +8,28 @@ services:
       context: log-generator
     volumes:
       - logs:/logs
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./logs
+          target: /logs
 
   fluentbit:
     container_name: fluentbit
     image: fluent/fluent-bit:latest
     user: root
     volumes:
       - logs:/logs
-      - ./fluentbit/fluent-bit.conf:/fluent-bit/fluent-bit.conf
+      - ./fluentbit/fluent-bit.yaml:/fluent-bit/fluent-bit.yaml
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./fluentbit/fluent-bit.yaml
+          target: /fluent-bit/fluent-bit.yaml
     command:
       - "fluent-bit"
       - "-c"
-      - "/fluent-bit/fluent-bit.conf"
+      - "/fluent-bit/fluent-bit.yaml"
 
   http-server:
     container_name: http-server

fluentbit/fluentbit/fluent-bit.conf

@@ -0,0 +1,30 @@
+# service:
+#   flush: 1
+#   log_level: info
+#   storage:
+#     path: /logs/fluent-bit-storage
+#     sync: full
+
+pipeline:
+  inputs:
+    - name: tail
+      path: /logs/*.log
+      storage:
+        type: filesystem
+      processors:
+        logs:
+          - name: modify
+            copy: log log_hash
+          - name: content_modifier
+            action: hash
+            key: log_hash
+  outputs:
+    - name: http
+      match: "*"
+      host: http-server
+      port: 5000
+      uri: /logs
+      format: json
+      retry_limit: false
+    - name: stdout
+      match: "*"

fluentbit/fluentbit/fluent-bit.yaml

@@ -0,0 +1,65 @@
+# lab-opensearch
+
+Lab created to test configuration Filebeat, Logstash, and OpenSearch and OpenSearch stack.
+
+- **Filebeat**: The Filebeat reads the logs line of this file [minimal-lab/filebeat/log/fake-logs.log](mininal-lab/filebeat/log/fake-logs.log) and sends to Logstash;
+- **Logstash**: Receive logs of the filebeat and parse them and forward to OpenSearch;
+- **OpenSearch**: Reveive logs (that in this step, are been named of *Documents*), and stores them in the OpenSearch internal database; 
+- **OpenSearch-Dashboards**: Connects in the OpenSearch database through web API and provides friendly web UI console to search de logs/documents. 
+
+> :warning: Study and Development's purposes
+>> The tool's configuration does not have SSL or authentication plugins enabled. 
+>> This lab was created to test purposes only, don't apply it in your production infrastructure.
+
+Project directory layout:
+
+```bash
+├── docker-compose.yml              # containers config state
+├── filebeat                        # filebeat container files
+│   ├── Dockerfile                  # filebeat dockerfile
+│   ├── filebeat.yml                # filebeat main config file
+│   └── log                         # fake log directory (shared like docker volume)
+│       └── *.log                   # fake logs 
+├── logstash                        # logstash container file
+│   ├── config                      # logstash config directory
+│   │   ├── logstash.yml            # logstash main config file
+│   │   └── pipelines.yml           # pipelines manifest file
+│   ├── Dockerfile                  # logstash Dockerfile
+│   └── pipeline                    # pipeline directory
+│       └── pipeline-1.cfg          # pipeline source code
+├── opensearch                      # opensearch container config files
+│   ├── Dockerfile                  # opensearch Dockerfile 
+│   └── opensearch.yml              # opensearch main config file 
+└── opensearch-dashboards           # opensearch-dashboards container config files
+    ├── Dockerfile                  # opensearch-dashboards Dockerfile
+    └── opensearch_dashboards.yml   # opensearch-dashboards main config file
+```
+
+## How To Run 
+
+The lab was created to run over `docker`. To set up container configuration use `docker-compose`.
+
+- To start the environment:
+
+```bash
+$ docker-compose up --build -d 
+```
+
+- To check logs:
+
+```bash
+$ docker-compose logs
+```
+
+- To destroy stack:
+
+```bash
+$ docker-compose logs
+```
+
+## How to Access the OpenSearch
+
+The resources can be accessed with the following links:
+
+- OpenSearch Dashboards: http://127.0.0.1:5601/
+- OpenSearch: http://127.0.0.1:9200/

opensearch/mininal/README.md

@@ -0,0 +1,39 @@
+version: "3.7"
+
+services:
+  opensearch:
+    container_name: opensearch
+    build:
+      context: opensearch/
+      args:
+        TAG_VERSION: "3.0.0"
+    volumes:
+      - type: bind
+        source: ./opensearch/opensearch.yml
+        target: /usr/share/opensearch/config/opensearch.yml
+    ports:
+      - 9200:9200
+    environment:
+      TIMEZONE: America/Sao_Paulo
+      TZ: America/Sao_Paulo
+      OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
+
+  opensearch-dashboards:
+    container_name: opensearch-dashboards
+    build:
+      context: opensearch-dashboards/
+      args:
+        TAG_VERSION: "3.0.0"
+    volumes:
+      - type: bind
+        source: ./opensearch-dashboards/opensearch_dashboards.yml
+        target: /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
+        read_only: true
+    ports:
+      - 5601:5601
+    environment:
+      TIMEZONE: America/Sao_Paulo
+      TZ: America/Sao_Paulo
+      LS_JAVA_OPTS: "-Xmx1g -Xms256m"
+    depends_on:
+      - opensearch

opensearch/mininal/docker-compose.yml

@@ -0,0 +1,3 @@
+ARG TAG_VERSION
+FROM opensearchproject/opensearch-dashboards:${TAG_VERSION}
+RUN /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards 

opensearch/mininal/opensearch-dashboards/Dockerfile

@@ -0,0 +1,7 @@
+# https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/config/opensearch_dashboards.yml
+
+server.port: 5601
+server.host: "0.0.0.0"
+server.ssl.enabled: false
+
+opensearch.hosts: ["http://opensearch:9200"]

opensearch/mininal/opensearch-dashboards/opensearch_dashboards.yml

@@ -0,0 +1,2 @@
+ARG TAG_VERSION
+FROM opensearchproject/opensearch:${TAG_VERSION}

opensearch/mininal/opensearch/Dockerfile

@@ -0,0 +1,5 @@
+cluster.name: docker-cluster
+discovery.type: single-node
+network.host: 0.0.0.0
+
+plugins.security.disabled: true