Merge branch 'main' into fix/conf_dirs

Author: Haarolean

documentation/compose/ui-arm64.yaml renamed to .dev/dev_arm64.yaml

@@ -1,8 +1,14 @@
+# This is a compose file designed for arm64/Apple Silicon systems
+# To adapt this to x86 please find and replace ".arm64" with empty
+
 # ARM64 supported images for kafka can be found here
 # https://hub.docker.com/r/confluentinc/cp-kafka/tags?page=1&name=arm64
 ---
-version: '2'
+version: '3.8'
+name: "kafbat-ui-dev"
+
 services:
+
   kafbat-ui:
     container_name: kafbat-ui
     image: ghcr.io/kafbat/kafka-ui:latest
@@ -12,19 +18,22 @@ services:
       - kafka0
       - schema-registry0
       - kafka-connect0
+      - ksqldb0
     environment:
       KAFKA_CLUSTERS_0_NAME: local
       KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092
       KAFKA_CLUSTERS_0_METRICS_PORT: 9997
       KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry0:8085
       KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first
       KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083
-      DYNAMIC_CONFIG_ENABLED: 'true'  # not necessary, added for tests
+      KAFKA_CLUSTERS_0_KSQLDBSERVER: http://ksqldb0:8088
+      DYNAMIC_CONFIG_ENABLED: 'true'
       KAFKA_CLUSTERS_0_AUDIT_TOPICAUDITENABLED: 'true'
       KAFKA_CLUSTERS_0_AUDIT_CONSOLEAUDITENABLED: 'true'
 
   kafka0:
-    image: confluentinc/cp-kafka:7.2.1.arm64
+    image: confluentinc/cp-kafka:7.8.0.arm64
+    user: "0:0"
     hostname: kafka0
     container_name: kafka0
     ports:
@@ -46,13 +55,12 @@ services:
       KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
       KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
       KAFKA_JMX_PORT: 9997
+#      KAFKA_JMX_HOSTNAME: localhost # uncomment this line and comment the next one if running with kafka-ui as a jar
       KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997
-    volumes:
-      - ./scripts/update_run.sh:/tmp/update_run.sh
-    command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'"
+      CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'
 
   schema-registry0:
-    image: confluentinc/cp-schema-registry:7.2.1.arm64
+    image: confluentinc/cp-schema-registry:7.8.0.arm64
     ports:
       - 8085:8085
     depends_on:
@@ -68,7 +76,7 @@ services:
       SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
 
   kafka-connect0:
-    image: confluentinc/cp-kafka-connect:7.2.1.arm64
+    image: confluentinc/cp-kafka-connect:7.8.0.arm64
     ports:
       - 8083:8083
     depends_on:
@@ -90,12 +98,30 @@ services:
       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
       CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0
-      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
+      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components,/usr/local/share/kafka/plugins,/usr/share/filestream-connectors"
+
+  ksqldb0:
+    image: confluentinc/cp-ksqldb-server:7.8.0.arm64
+    depends_on:
+      - kafka0
+      - kafka-connect0
+      - schema-registry0
+    environment:
+      KSQL_CUB_KAFKA_TIMEOUT: 120
+      KSQL_LISTENERS: http://0.0.0.0:8088
+      KSQL_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092
+      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
+      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
+      KSQL_KSQL_CONNECT_URL: http://kafka-connect0:8083
+      KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry0:8085
+      KSQL_KSQL_SERVICE_ID: my_ksql_1
+      KSQL_KSQL_HIDDEN_TOPICS: '^_.*'
+      KSQL_CACHE_MAX_BYTES_BUFFERING: 0
 
   kafka-init-topics:
-    image: confluentinc/cp-kafka:7.2.1.arm64
+    image: confluentinc/cp-kafka:7.8.0.arm64
     volumes:
-       - ./data/message.json:/data/message.json
+      - ../documentation/compose/data/message.json:/data/message.json
     depends_on:
       - kafka0
     command: "bash -c 'echo Waiting for Kafka to be ready... && \

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/CODEOWNERS

@@ -3,16 +3,16 @@
 
 # BACKEND
 /pom.xml                    @kafbat/backend
-
-/contract/         @kafbat/backend
-
-/api/              @kafbat/backend
+/contract/                  @kafbat/backend
+/api/                       @kafbat/backend
+/serde-api/                 @kafbat/backend
+/documentation/compose/     @kafbat/backend
 
 # FRONTEND
-/frontend/        @kafbat/frontend
+/frontend/                  @kafbat/frontend
 
 # TESTS
-/e2e-tests/       @kafbat/qa
+/e2e-tests/                 @kafbat/qa
 
 # INFRA
 /.github/workflows/         @kafbat/devops

.github/dependabot.yml

@@ -14,6 +14,23 @@ updates:
     - "type/dependencies"
     - "scope/backend"
 
+- package-ecosystem: docker
+  directory: "/api"
+  schedule:
+    interval: weekly
+    time: "10:00"
+    timezone: Europe/London
+  reviewers:
+    - "kafbat/backend"
+  open-pull-requests-limit: 10
+  ignore:
+  - dependency-name: "azul/zulu-openjdk-alpine"
+    # Limit dependabot pull requests to minor Java upgrades
+    update-types: ["version-update:semver-major"]
+  labels:
+    - "type/dependencies"
+    - "scope/backend"
+
 - package-ecosystem: npm
   directory: "/frontend"
   schedule:

.github/workflows/aws_publisher.yml renamed to .github/workflows/aws_ami_publish.yml

@@ -19,7 +19,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
+  build-and-test:
     uses: ./.github/workflows/backend_tests.yml
     with:
       event_name: ${{ github.event_name }}

.github/workflows/backend_main.yml

@@ -20,7 +20,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
+  build-and-test:
     uses: ./.github/workflows/backend_tests.yml
     with:
       event_name: ${{ github.event_name }}

.github/workflows/backend_pr.yml

@@ -26,14 +26,14 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          java-version: '17'
+          java-version: '21'
           distribution: 'zulu'
           cache: 'maven'
 
       - name: Cache SonarCloud packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar

.github/workflows/backend_tests.yml

@@ -27,9 +27,9 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          java-version: '17'
+          java-version: '21'
           distribution: 'zulu'
           cache: 'maven'
       - name: Build
@@ -40,12 +40,12 @@ jobs:
           export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
           echo "version=${VERSION}" >> $GITHUB_OUTPUT
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}

.github/workflows/branch-deploy.yml

@@ -6,7 +6,9 @@ on:
     types: ['labeled']
 
 permissions:
+  id-token: write
   contents: read
+  pull-requests: write
 
 jobs:
   build:
@@ -23,9 +25,9 @@ jobs:
           tag='${{ github.event.pull_request.number }}'
           echo "tag=${tag}" >> $GITHUB_OUTPUT
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          java-version: '17'
+          java-version: '21'
           distribution: 'zulu'
           cache: 'maven'
       - name: Build
@@ -36,23 +38,22 @@ jobs:
           export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
           echo "version=${VERSION}" >> $GITHUB_OUTPUT
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-
-      - name: Configure AWS credentials for Kafka-UI account
+      - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
+          role-to-assume: ${{ secrets.AWS_ROLE }}
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
@@ -65,7 +66,7 @@ jobs:
           builder: ${{ steps.buildx.outputs.name }}
           context: api
           push: true
-          tags: public.ecr.aws/kafbat/kafka-ui-custom-build:${{ steps.extract_branch.outputs.tag }}
+          tags: ${{ vars.ECR_REGISTRY }}/${{ github.repository }}:${{ steps.extract_branch.outputs.tag }}
           build-args: |
             JAR_FILE=api-${{ steps.build.outputs.version }}.jar
           cache-from: type=local,src=/tmp/.buildx-cache
@@ -75,6 +76,6 @@ jobs:
         with:
           issue-number: ${{ github.event.pull_request.number }}
           body: |
-            Image published at public.ecr.aws/kafbat/kafka-ui-custom-build:${{ steps.extract_branch.outputs.tag }}
+            Image published at ${{ vars.ECR_REGISTRY }}/${{ github.repository }}:${{ steps.extract_branch.outputs.tag }}
     outputs:
       tag: ${{ steps.extract_branch.outputs.tag }}