Merge branch 'main' into 1218-expanding-fileselector-capabilities

Author: Vismayak

.dockerignore

@@ -0,0 +1 @@
+.venv

.github/workflows/backend.yaml

@@ -28,7 +28,7 @@ jobs:
         uses: docker/setup-buildx-action@v2
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       -
         name: Login to DockerHub
         if: github.event_name != 'pull_request'
@@ -85,7 +85,8 @@ jobs:
         name: Build and push
         uses: docker/build-push-action@v3
         with:
-          context: ./backend
+          context: .
+          file: ./backend/Dockerfile
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.backendMeta.outputs.tags }}
@@ -95,20 +96,20 @@ jobs:
         name: Build and push
         uses: docker/build-push-action@v3
         with:
-          context: ./backend
+          context: .
+          file: ./backend/heartbeat.Dockerfile
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.heartbeatMeta.outputs.tags }}
-          file: ./backend/heartbeat.Dockerfile
           labels: ${{ steps.heartbeatMeta.outputs.labels }}
 
       -
         name: Build and push
         uses: docker/build-push-action@v3
         with:
-          context: ./backend
+          context: .
+          file: ./backend/messages.Dockerfile
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.messagesMeta.outputs.tags }}
-          file: ./backend/messages.Dockerfile
           labels: ${{ steps.messagesMeta.outputs.labels }}

.github/workflows/codegen.yaml

@@ -17,38 +17,26 @@ jobs:
     timeout-minutes: 10
     runs-on: ubuntu-latest
 
-    strategy:
-      fail-fast: false
-      matrix:
-        node:
-          - "14"
-
     steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build Backend no push
         uses: docker/build-push-action@v3
         with:
-          context: ./backend
+          context: .
+          file: ./backend/Dockerfile
           platforms: linux/amd64
           push: false
           tags: github-action
 
       - name: Start containers
-        run: docker compose -f "docker-compose.test.yml" up -d --build
+        run: docker compose -f "docker-compose.yml" up -d --build
 
-      # setup node
       - name: Install node
         uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node }}
+          node-version: 16
 
       - name: Install dependencies
         working-directory: frontend
@@ -73,4 +61,4 @@ jobs:
 
       - name: Stop containers
         if: always()
-        run: docker compose -f "docker-compose.test.yml" down
+        run: docker compose -f "docker-compose.yml" down

.github/workflows/pytest.yml

@@ -1,38 +1,24 @@
 name: Pytest
+
 on: push
 
 defaults:
   run:
     working-directory: backend
 
 jobs:
-  test:
+  uv-example:
+    name: python
     runs-on: ubuntu-latest
-    timeout-minutes: 10
 
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-
-      - name: Install pipenv
-        run: |
-          python -m pip install --upgrade pipenv wheel
+      - uses: actions/checkout@v4
 
-      - id: cache-pipenv
-        uses: actions/cache@v3
-        with:
-          path: ~/.local/share/virtualenvs
-          key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
 
-      - name: Install dependencies
-        if: steps.cache-pipenv.outputs.cache-hit != 'true'
-        run: |
-          pipenv install --deploy --dev
+      - name: Install the project
+        run: uv sync --all-extras --dev
 
       - name: Start env
         run: docker compose -f ../docker-compose.dev.yml up -d
@@ -42,6 +28,5 @@ jobs:
         run: sleep 60s
         shell: bash
 
-      - name: Run test suite
-        run: |
-          pipenv run pytest -v
+      - name: Run tests
+        run: uv run pytest app/tests/

.gitignore

@@ -59,6 +59,7 @@ scripts/keycloak/data/*
 # ignore clowder chart deps
 deployments/kubernetes/charts/clowder2/charts
 deployments/kubernetes/charts/clowder2/*clowder2-software-dev.yaml
+*secret*.yaml
 
 # Environments
 .env

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     hooks:
       - id: extract-openapi
         name: extract-openapi
-        entry: pipenv run extract-openapi
+        entry: python backend/extract-openapi.py app.main:app
         language: system
         pass_filenames: false
   - repo: local
@@ -25,29 +25,21 @@ repos:
         name: prettier
         entry: bash -c "cd frontend && npx prettier src --write"
         language: system
-  - repo: https://github.com/psf/black
-    rev: 23.11.0
-    hooks:
-      - id: black
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        args: [ "--profile=black" ]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: check-yaml
         exclude: deployments/kubernetes/charts
       - id: end-of-file-fixer
         exclude: 'openapi.json|frontend/src/openapi/v2/'
       - id: trailing-whitespace
         exclude: 'openapi.json|frontend/src/openapi/v2/'
-  - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.9.0.6
-    hooks:
-      - id: shellcheck
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.9.4
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format

.python-version

@@ -0,0 +1 @@
+3.10

.run/uvicorn.run.xml

@@ -1,7 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="uvicorn" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
     <module name="clowder2" />
-    <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <envs>
@@ -14,7 +13,7 @@
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
     <option name="SCRIPT_NAME" value="uvicorn" />
-    <option name="PARAMETERS" value="app.main:app --reload --host 0.0.0.0" />
+    <option name="PARAMETERS" value="app.main:app --host 0.0.0.0 --workers 17" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="true" />

CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v2.0.0-beta.3] - 2024-07-29
+
+### Added
+
+- License management
+- Release dataset with versions
+- Enable and disable user account through Keycloak
+- Jupyterhub integration
+- Interface for creating and editing matching criteria for triggering extractors
+- Interface for editing metadata definitions
+- My dataset tab listing all the datasets created by the user
+- Drag and drop upload multiple files
+- Footer with links to documentation, source code, and contact information
+- Documentation through MKDocs
+
+### Changed
+
+- Allow public datasets and files to be searchable
+- List all the extractors with the ability to enable/disable the extractors
+- Filter listeners based on their support for file or dataset
+- Helm chart updated to support custom existing secret
+
+### Fixed
+
+- Clowder registration link on the top bar
+- Case-insensitive search
+- Download count immediately increments after download
+
 ## [v2.0.0-beta.2] - 2024-02-16
 
 ### Added