Merge remote-tracking branch 'origin/main' into refactor/replace-restful-react

Author: disrupted

.github/workflows/ci.yml

@@ -13,6 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       working-directory: ./backend
+      poetry-version: "1.3.2"
     strategy:
       fail-fast: false
       matrix:
@@ -28,9 +29,10 @@ jobs:
         run: sudo apt-get -y install python3-dev graphviz libgraphviz-dev pkg-config python3-wheel
 
       - name: Set up Poetry
-        uses: bakdata/ci-templates/actions/python-setup-poetry@v1.5.1
+        uses: bakdata/ci-templates/actions/python-setup-poetry@v1.5.3
         with:
           python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.poetry-version }}
           working-directory: ${{ env.working-directory }}
 
       - name: Install dependencies

.github/workflows/publish.yml

@@ -17,6 +17,7 @@ jobs:
     env:
       BACKEND_DIR: ./backend
       python-version: "3.10"
+      poetry-version: "1.3.2"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -26,9 +27,10 @@ jobs:
         run: sudo apt-get -y install python3-dev graphviz libgraphviz-dev pkg-config python3-wheel
 
       - name: Release to PyPI
-        uses: bakdata/ci-templates/actions/python-poetry-release@v1.5.1
+        uses: bakdata/ci-templates/actions/python-poetry-release@v1.5.3
         with:
           python-version: ${{ env.python-version }}
+          poetry-version: ${{ env.poetry-version }}
           working-directory: ${{ env.BACKEND_DIR }}
           pypi-token: ${{ secrets.pypi-token }}
           publish-to-test: "false"

.github/workflows/release.yml

@@ -4,9 +4,14 @@ on:
   workflow_dispatch:
     inputs:
       release-type:
-        description: "Scope of the release; See: https://python-poetry.org/docs/cli/#version"
-        default: "patch"
-        required: false
+        description: "Scope of the release; see https://python-poetry.org/docs/cli/#version"
+        type: choice
+        required: true
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
 
 jobs:
   bump-version:

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         language: system
         pass_filenames: false
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
         args: ["--settings", "backend/setup.cfg"]

backend/README.md

@@ -1,6 +1,6 @@
 # Streams Explorer
 
-> Explore Data Pipelines in Apache Kafka.
+> Explore Apache Kafka data pipelines in Kubernetes.
 
 ![streams-explorer](https://github.com/bakdata/streams-explorer/blob/main/screens/overview.png?raw=true)
 
@@ -49,6 +49,9 @@ Visit our introduction [blogpost](https://medium.com/bakdata/exploring-data-pipe
 
 ## Installation
 
+> **Prerequisites**
+> Access to a Kubernetes cluster, where streaming apps and services are deployed.
+
 ### Docker Compose
 
 1. Forward the ports to Prometheus. (Kafka Connect, Schema Registry, and other integrations are optional)
@@ -122,7 +125,7 @@ The following configuration options are available:
 
 #### General
 
-- `graph.update_interval` Render the graph every x seconds (int, **required**, default: `300`)
+- `graph.update_interval` Render the graph every x seconds (int, **required**, default: `30`)
 - `graph.layout_arguments` Arguments passed to graphviz layout (string, **required**, default: `-Grankdir=LR -Gnodesep=0.8 -Gpad=10`)
 - `graph.pipeline_distance` Increase/decrease vertical space between pipeline graphs by X pixels (int, **required**, default: `500`)
 - `graph.resolve.input_pattern_topics.all` If true topics that match (extra) input pattern(s) are connected to the streaming app in the graph containing all pipelines (bool, **required**, default: `false`)

backend/poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "streams-explorer"
-version = "2.1.1"
+version = "2.2.5"
 description = "Explore Data Pipelines in Apache Kafka."
 readme = "README.md"
 authors = ["bakdata"]
@@ -43,7 +43,7 @@ kubernetes-asyncio = "^23.6.0"
 mypy = "^0.981"
 flake8 = "^4.0.1"
 black = "^22.6.0"
-isort = "^5.10.1"
+isort = "^5.12.0"
 pytest = "^7.2.0"
 pre-commit = "^2.15.0"
 pytest-asyncio = "^0.20.1"

backend/pyproject.toml

@@ -1,6 +1,6 @@
 graph:
   # render graph every x seconds
-  update_interval: 300
+  update_interval: 30
   layout_arguments: "-Grankdir=LR -Gnodesep=0.8 -Gpad=10"
   pipeline_distance: 500
   resolve:

backend/settings.yaml

@@ -1,13 +1,16 @@
 [flake8]
-ignore =
-    # E501: line too long
-    E501,
-    # W503: line break before binary operator
-    W503,
-max-complexity = 10
 exclude =
     .git,
     __pycache__
+max-complexity = 10
+# black docs regarding flake8: https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
+# black enforces an equal amount of whitespace around slice operators. It is not PEP8 compliant.
+# black and flake8 also disagree on line length
+extend-ignore = 
+    # E203: Whitespace before ':'
+    E203,
+    # E501: Line too long
+    E501,
 
 [isort]
 profile = black

backend/setup.cfg

@@ -44,6 +44,8 @@ async def websocket_endpoint(
             await websocket.receive_text()
 
     except WebSocketDisconnect:
+        # client disconnected: browser window closed or manual page refresh
+        # continue to finally clause and remove client from client manager
         return
     finally:
         await streams_explorer.client_manager.disconnect(websocket)