Adds filebeat integration for centralized logging

Implements filebeat for centralized log management in production.

This change introduces Filebeat to collect and ship logs to Elasticsearch,
enabling centralized logging and improved monitoring capabilities.
It includes configurations for Docker autodiscovery, JSON log decoding,
and integration with the Elastic Stack.

It also adds a request logging filter to log HTTP requests,
and it configures logback to output JSON format suitable for filebeat.

Adds .env* and logs/ to .gitignore to prevent committing those files.

Author: oDaviML

.github/workflows/deploy.yml

@@ -28,15 +28,26 @@ jobs:
         push: true
         tags: ${{ secrets.DOCKER_USERNAME }}/api-onibusbh:latest
 
+    - name: Copy config to EC2
+      uses: appleboy/scp-action@master
+      with:
+        host: ${{ secrets.EC2_HOST }}
+        username: ${{ secrets.EC2_USERNAME }}
+        key: ${{ secrets.EC2_SSH_KEY }}
+        source: "filebeat.prod.yml"
+        target: "/home/${{ secrets.EC2_USERNAME }}/api-onibusbh/"
+
     - name: Deploy to EC2
       uses: appleboy/ssh-action@master
       with:
         host: ${{ secrets.EC2_HOST }}
         username: ${{ secrets.EC2_USERNAME }}
         key: ${{ secrets.EC2_SSH_KEY }}
         script: |
-          docker stop api-onibusbh || true
-          docker rm api-onibusbh || true
+          WORKDIR="/home/${{ secrets.EC2_USERNAME }}/api-onibusbh"
+          
+          docker stop api-onibusbh filebeat-prod || true
+          docker rm api-onibusbh filebeat-prod || true
           
           docker pull ${{ secrets.DOCKER_USERNAME }}/api-onibusbh:latest
           
@@ -49,4 +60,17 @@ jobs:
             -e MONGO_DATABASE=${{ secrets.MONGO_DATABASE }} \
             -e MONGO_USERNAME=${{ secrets.MONGO_USERNAME }} \
             -e MONGO_PASSWORD=${{ secrets.MONGO_PASSWORD }} \
+            -e ELASTIC_HOST=${{ secrets.ELASTIC_HOST }} \
+            --restart unless-stopped \
             ${{ secrets.DOCKER_USERNAME }}/api-onibusbh:latest
+            
+          docker run -d \
+            --name filebeat-prod \
+            --user root \
+            --restart unless-stopped \
+            -e ELASTIC_HOST=${{ secrets.ELASTIC_HOST }} \
+            -v /var/lib/docker/containers:/var/lib/docker/containers:ro \
+            -v /var/run/docker.sock:/var/run/docker.sock:ro \
+            -v $WORKDIR/filebeat.prod.yml:/usr/share/filebeat/filebeat.yml:ro \
+            docker.elastic.co/beats/filebeat:8.15.0 \
+            --strict.perms=false

.gitignore

@@ -31,3 +31,11 @@ build/
 
 ### VS Code ###
 .vscode/
+
+### Environment ###
+.env
+.env.*
+
+### Logs ###
+logs/
+*.log

compose.yaml

@@ -6,4 +6,20 @@ services:
       - MONGO_INITDB_ROOT_PASSWORD=E0K64Y7sAqNYqOAx
       - MONGO_INITDB_ROOT_USERNAME=admin
     ports:
-      - "27017:27017"
+      - "27017:27017"
+
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:8.11.1
+    user: root
+    environment:
+      - ELASTIC_HOST=${ELASTIC_HOST}
+    dns:
+      - 8.8.8.8
+      - 8.8.4.4
+    volumes:
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - ./logs:/host_logs:ro
+    command: ["--strict.perms=false"]
+    restart: unless-stopped

filebeat.prod.yml

@@ -0,0 +1,29 @@
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+      templates:
+        - condition:
+            contains:
+              docker.container.name: springboot-app_prod
+          config:
+            - type: container
+              paths:
+                - /var/lib/docker/containers/${data.docker.container.id}/*.log
+              processors:
+                - decode_json_fields:
+                    fields: ["message"]
+                    target: ""
+                    overwrite_keys: true
+                    add_error_key: true
+
+processors:
+  - add_docker_metadata: ~
+  - add_cloud_metadata: ~
+  - add_tags:
+      tags: ["production", "api-onibusbh"]
+
+output.elasticsearch:
+  hosts: ['${ELASTIC_HOST}']
+  protocol: "https"
+  # ssl.verification_mode: none

filebeat.yml

@@ -0,0 +1,44 @@
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+    - /host_logs/*.json
+  json.keys_under_root: true
+  json.overwrite_keys: true
+  processors:
+    - add_tags:
+        tags: ["local_dev_ide"]
+
+- type: container
+  paths: 
+    - '/var/lib/docker/containers/*/*.log'
+  processors:
+  - add_docker_metadata:
+      host: "unix:///var/run/docker.sock"
+
+  - drop_event:
+      when:
+        not:
+          contains:
+            container.name: "springboot-app_prod"
+
+  - decode_json_fields:
+      fields: ["message"]
+      target: ""
+      overwrite_keys: true
+      process_array: false
+      max_depth: 1
+
+output.elasticsearch:
+  hosts: ["${ELASTIC_HOST:localhost:9200}"]
+  ssl.verification_mode: none
+
+  index: "api-onibusbh-prod-%{+yyyy.MM.dd}"
+
+setup.template.name: "api-onibusbh"
+setup.template.pattern: "api-onibusbh-*"
+setup.ilm.enabled: false
+
+logging.level: debug
+logging.to_stderr: true
+logging.selectors: ["*"]

src/main/java/com/dmware/api_onibusbh/config/RequestLoggingFilter.java

@@ -0,0 +1,62 @@
+package com.dmware.api_onibusbh.config;
+
+import jakarta.servlet.FilterChain;
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.stereotype.Component;
+import org.springframework.web.filter.OncePerRequestFilter;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static net.logstash.logback.argument.StructuredArguments.kv;
+
+@Component
+public class RequestLoggingFilter extends OncePerRequestFilter {
+
+    private static final Logger logger = LoggerFactory.getLogger(RequestLoggingFilter.class);
+    private static final String TRANSACTION_ID_KEY = "transaction_id";
+
+    @Override
+    protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
+            throws ServletException, IOException {
+
+        long startTime = System.currentTimeMillis();
+
+        String transactionId = request.getHeader("X-Transaction-ID");
+        if (transactionId == null || transactionId.isEmpty()) {
+            transactionId = UUID.randomUUID().toString();
+        }
+        
+        MDC.put(TRANSACTION_ID_KEY, transactionId);
+
+        try {
+            logger.info("Requisição recebida: {} {}", request.getMethod(), request.getRequestURI(),
+                    kv("method", request.getMethod()),
+                    kv("uri", request.getRequestURI()),
+                    kv("client_ip", request.getRemoteAddr()),
+                    kv("user_agent", request.getHeader("User-Agent"))
+            );
+
+            filterChain.doFilter(request, response);
+
+            long duration = System.currentTimeMillis() - startTime;
+            
+            logger.info("Requisição finalizada: {} {} - Status: {} - Tempo: {}ms", 
+                    request.getMethod(), 
+                    request.getRequestURI(), 
+                    response.getStatus(), 
+                    duration,
+                    kv("status_code", response.getStatus()),
+                    kv("duration_ms", duration)
+            );
+
+        } finally {
+            MDC.remove(TRANSACTION_ID_KEY);
+        }
+    }
+}

src/main/java/com/dmware/api_onibusbh/scheduler/CoordenadasScheduler.java

@@ -17,6 +17,7 @@
 @Component
 public class CoordenadasScheduler {
 
+    private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoordenadasScheduler.class);
     private final APIService apiService;
     private final OnibusService onibusService;
 
@@ -29,8 +30,10 @@ public CoordenadasScheduler(APIService apiService, OnibusService onibusService)
     public void fetchCoordenadasOnibus() {
         try {
             MDC.put("transaction_id", UUID.randomUUID().toString());
+            logger.info("Job de Coordenadas iniciado.");
             List<CoordenadaDTO> coordenadas = apiService.getOnibusCoordenadaBH();
             onibusService.salvaCoordenadas(coordenadas);
+            logger.info("Job de Coordenadas finalizado.");
         } finally {
             MDC.clear();
         }

src/main/java/com/dmware/api_onibusbh/scheduler/DicionarioScheduler.java

@@ -14,6 +14,7 @@
 @Component
 public class DicionarioScheduler {
 
+    private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DicionarioScheduler.class);
     private final DicionarioService dicionarioService;
 
     public DicionarioScheduler(DicionarioService dicionarioService) {
@@ -25,7 +26,9 @@ public DicionarioScheduler(DicionarioService dicionarioService) {
       public void salvaDicionarioBanco() {
             try {
                   MDC.put("transaction_id", UUID.randomUUID().toString());
+                  logger.info("Job de Dicionário iniciado.");
                   dicionarioService.salvarDicionarioBanco();
+                  logger.info("Job de Dicionário finalizado.");
             } finally {
                   MDC.clear();
             }

src/main/java/com/dmware/api_onibusbh/scheduler/LinhaScheduler.java

@@ -14,6 +14,7 @@
 @Component
 public class LinhaScheduler {
 
+    private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LinhaScheduler.class);
     private final LinhasService linhasService;
 
     public LinhaScheduler(LinhasService linhasService) {
@@ -25,7 +26,9 @@ public LinhaScheduler(LinhasService linhasService) {
     public void fetchCoordenadasOnibus() {
         try {
             MDC.put("transaction_id", UUID.randomUUID().toString());
+            logger.info("Job agendado iniciado: Atualização de Coordenadas/Linhas.");
             linhasService.salvaLinhasNormais();
+            logger.info("Job agendado finalizado: Atualização de Coordenadas/Linhas.");
         } finally {
             MDC.clear();
         }

src/main/java/com/dmware/api_onibusbh/services/DicionarioService.java

@@ -78,7 +78,7 @@ public void salvarDicionarioBanco() {
                   dicionarioRepository
                               .saveAll(modelMapper.map(dadosParaSalvar, new TypeToken<List<DicionarioEntity>>() {
                               }.getType()));
-                  logger.info("Sincronização de dicionário concluída", kv("total_processado", dadosParaSalvar.size()));
+                  logger.info("Sincronização de dicionário concluída. Total de itens processados: {}", dadosParaSalvar.size(), kv("total_processado", dadosParaSalvar.size()));
             } else {
                   logger.info("Nenhuma alteração necessária no dicionário");
             }