Merge branch 'refs/heads/release/v2.0-beta-3' into feature/update-user

# Conflicts:
#	backend/app/routers/authentication.py

Author: GalMunGral

.gitignore

@@ -59,6 +59,7 @@ scripts/keycloak/data/*
 # ignore clowder chart deps
 deployments/kubernetes/charts/clowder2/charts
 deployments/kubernetes/charts/clowder2/*clowder2-software-dev.yaml
+*secret*.yaml
 
 # Environments
 .env

.run/uvicorn.run.xml

@@ -1,7 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="uvicorn" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
     <module name="clowder2" />
-    <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <envs>
@@ -14,7 +13,7 @@
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
     <option name="SCRIPT_NAME" value="uvicorn" />
-    <option name="PARAMETERS" value="app.main:app --reload --host 0.0.0.0" />
+    <option name="PARAMETERS" value="app.main:app --host 0.0.0.0 --workers 17" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="true" />

CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v2.0.0-beta.3] - 2024-07-29
+
+### Added
+
+- License management
+- Release dataset with versions
+- Enable and disable user account through Keycloak
+- Jupyterhub integration
+- Interface for creating and editing matching criteria for triggering extractors
+- Interface for editing metadata definitions
+- My dataset tab listing all the datasets created by the user
+- Drag and drop upload multiple files
+- Footer with links to documentation, source code, and contact information
+- Documentation through MKDocs
+
+### Changed
+
+- Allow public datasets and files to be searchable
+- List all the extractors with the ability to enable/disable the extractors
+- Filter listeners based on their support for file or dataset
+- Helm chart updated to support custom existing secret
+
+### Fixed
+
+- Clowder registration link on the top bar
+- Case-insensitive search
+- Download count immediately increments after download
+
 ## [v2.0.0-beta.2] - 2024-02-16
 
 ### Added

backend/Dockerfile

@@ -22,4 +22,5 @@ ENV PATH="/code/.venv/bin:$PATH"
 COPY ./app /code/app
 
 # launch app using uvicorn
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"]
+# Number of recommended workers is 2 x number_of_cores +1
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80", "--workers", "17"]

backend/app/config.py

@@ -9,7 +9,7 @@ class Settings(BaseSettings):
     API_V2_STR: str = "/api/v2"
     admin_email: str = "[email protected]"
     frontend_url: str = "http://localhost:3000"
-    version: str = "2.0.0-beta.2"
+    version: str = "2.0.0-beta.3"
 
     # Unique secret for hashing API keys. Generate with `openssl rand -hex 32`
     local_auth_secret = "clowder_secret_key"

backend/app/main.py

@@ -86,7 +86,7 @@
     description="A cloud native data management framework to support any research domain. Clowder was "
     "developed to help researchers and scientists in data intensive domains manage raw data, complex "
     "metadata, and automatic data pipelines. ",
-    version="2.0.0-beta.2",
+    version="2.0.0-beta.3",
     contact={"name": "Clowder", "url": "https://clowderframework.org/"},
     license_info={
         "name": "Apache 2.0",
@@ -316,7 +316,9 @@ async def startup_beanie():
             ThumbnailDBViewList,
             LicenseDB,
         ],
-        recreate_views=True,
+        # If view exists, will not recreate
+        # When view query changes, make sure to manually drop view and recreate
+        recreate_views=False,
     )
 
 

backend/app/models/metadata.py

@@ -307,9 +307,11 @@ async def validate_context(
             detail="Context is required",
         )
     if context is not None:
-        pass
+        # TODO validate context
+        return content
     if context_url is not None:
-        pass
+        # TODO validate context
+        return content
     if definition is not None:
         if (
             md_def := await MetadataDefinitionDB.find_one(
@@ -322,7 +324,7 @@ async def validate_context(
                 status_code=400,
                 detail=f"{definition} is not valid metadata definition",
             )
-    return content
+        return content
 
 
 def deep_update(orig: dict, new: dict):

backend/app/routers/authentication.py

@@ -9,6 +9,7 @@
 )
 from app.models.datasets import DatasetDBViewList
 from app.models.users import UserDB, UserIn, UserLogin, UserOut, UserUpdate
+from app.routers.utils import save_refresh_token
 from beanie import PydanticObjectId
 from fastapi import APIRouter, Depends, HTTPException
 from keycloak.exceptions import (
@@ -71,6 +72,7 @@ async def save_user(userIn: UserIn):
 async def login(userIn: UserLogin):
     try:
         token = keycloak_openid.token(userIn.email, userIn.password)
+        await save_refresh_token(token["refresh_token"], userIn.email)
         return {"token": token["access_token"]}
     # bad credentials
     except KeycloakAuthenticationError as e:

backend/app/routers/datasets.py

@@ -73,6 +73,7 @@
 from pymongo import DESCENDING
 from rocrate.model.person import Person
 from rocrate.rocrate import ROCrate
+from starlette.concurrency import run_in_threadpool
 
 router = APIRouter()
 security = HTTPBearer()
@@ -1192,16 +1193,26 @@ async def download_dataset(
         bag_info_path = os.path.join(current_temp_dir, "bag-info.txt")
         tagmanifest_path = os.path.join(current_temp_dir, "tagmanifest-md5.txt")
 
-        with open(manifest_path, "w") as f:
-            pass  # Create empty file so no errors later if the dataset is empty
-
-        with open(bagit_path, "w") as f:
-            f.write("Bag-Software-Agent: clowder.ncsa.illinois.edu" + "\n")
-            f.write("Bagging-Date: " + str(datetime.datetime.now()) + "\n")
+        await run_in_threadpool(lambda: open(manifest_path, "w").close())
+        await run_in_threadpool(lambda: open(manifest_path, "w").close())
+        await run_in_threadpool(
+            lambda: open(bagit_path, "w").write(
+                "Bag-Software-Agent: clowder.ncsa.illinois.edu"
+                + "\n"
+                + "Bagging-Date: "
+                + str(datetime.datetime.now())
+                + "\n"
+            )
+        )
 
-        with open(bag_info_path, "w") as f:
-            f.write("BagIt-Version: 0.97" + "\n")
-            f.write("Tag-File-Character-Encoding: UTF-8" + "\n")
+        await run_in_threadpool(
+            lambda: open(bag_info_path, "w").write(
+                "BagIt-Version: 0.97"
+                + "\n"
+                + "Tag-File-Character-Encoding: UTF-8"
+                + "\n"
+            )
+        )
 
         # Write dataset metadata if found
         metadata = await MetadataDB.find(
@@ -1214,6 +1225,10 @@ async def download_dataset(
             metadata_content = json_util.dumps(metadata)
             with open(datasetmetadata_path, "w") as f:
                 f.write(metadata_content)
+            await run_in_threadpool(
+                lambda: open(datasetmetadata_path, "w").write(metadata_content)
+            )
+
             crate.add_file(
                 datasetmetadata_path,
                 dest_path="metadata/_dataset_metadata.json",
@@ -1236,16 +1251,20 @@ async def download_dataset(
                 hierarchy = await _get_folder_hierarchy(file.folder_id, "")
                 dest_folder = os.path.join(current_temp_dir, hierarchy.lstrip("/"))
                 if not os.path.isdir(dest_folder):
-                    os.makedirs(dest_folder, exist_ok=True)
+                    await run_in_threadpool(os.makedirs, dest_folder, exist_ok=True)
                 file_name = hierarchy + file_name
             current_file_path = os.path.join(current_temp_dir, file_name.lstrip("/"))
 
             content = fs.get_object(settings.MINIO_BUCKET_NAME, bytes_file_id)
             file_md5_hash = hashlib.md5(content.data).hexdigest()
-            with open(current_file_path, "wb") as f1:
-                f1.write(content.data)
-            with open(manifest_path, "a") as mpf:
-                mpf.write(file_md5_hash + " " + file_name + "\n")
+            await run_in_threadpool(
+                lambda: open(current_file_path, "wb").write(content.data)
+            )
+            await run_in_threadpool(
+                lambda: open(manifest_path, "a").write(
+                    file_md5_hash + " " + file_name + "\n"
+                )
+            )
             crate.add_file(
                 current_file_path,
                 dest_path="data/" + file_name,
@@ -1266,23 +1285,43 @@ async def download_dataset(
                     current_temp_dir, metadata_filename
                 )
                 metadata_content = json_util.dumps(metadata)
-                with open(metadata_filename_temp_path, "w") as f:
-                    f.write(metadata_content)
+                await run_in_threadpool(
+                    lambda: open(metadata_filename_temp_path, "w").write(
+                        metadata_content
+                    )
+                )
                 crate.add_file(
                     metadata_filename_temp_path,
                     dest_path="metadata/" + metadata_filename,
                     properties={"name": metadata_filename},
                 )
 
         bag_size_kb = bag_size / 1024
-
-        with open(bagit_path, "a") as f:
-            f.write("Bag-Size: " + str(bag_size_kb) + " kB" + "\n")
-            f.write("Payload-Oxum: " + str(bag_size) + "." + str(file_count) + "\n")
-            f.write("Internal-Sender-Identifier: " + dataset_id + "\n")
-            f.write("Internal-Sender-Description: " + dataset.description + "\n")
-            f.write("Contact-Name: " + user_full_name + "\n")
-            f.write("Contact-Email: " + user.email + "\n")
+        await run_in_threadpool(
+            lambda: open(bagit_path, "a").write(
+                "Bag-Size: "
+                + str(bag_size_kb)
+                + " kB"
+                + "\n"
+                + "Payload-Oxum: "
+                + str(bag_size)
+                + "."
+                + str(file_count)
+                + "\n"
+                + "Internal-Sender-Identifier: "
+                + dataset_id
+                + "\n"
+                + "Internal-Sender-Description: "
+                + dataset.description
+                + "\n"
+                + "Contact-Name: "
+                + user_full_name
+                + "\n"
+                + "Contact-Email: "
+                + user.email
+                + "\n"
+            )
+        )
         crate.add_file(
             bagit_path, dest_path="bagit.txt", properties={"name": "bagit.txt"}
         )
@@ -1296,14 +1335,33 @@ async def download_dataset(
         )
 
         # Generate tag manifest file
-        manifest_md5_hash = hashlib.md5(open(manifest_path, "rb").read()).hexdigest()
-        bagit_md5_hash = hashlib.md5(open(bagit_path, "rb").read()).hexdigest()
-        bag_info_md5_hash = hashlib.md5(open(bag_info_path, "rb").read()).hexdigest()
-
-        with open(tagmanifest_path, "w") as f:
-            f.write(bagit_md5_hash + " " + "bagit.txt" + "\n")
-            f.write(manifest_md5_hash + " " + "manifest-md5.txt" + "\n")
-            f.write(bag_info_md5_hash + " " + "bag-info.txt" + "\n")
+        manifest_md5_hash = await run_in_threadpool(
+            lambda: hashlib.md5(open(manifest_path, "rb").read()).hexdigest()
+        )
+        bagit_md5_hash = await run_in_threadpool(
+            lambda: hashlib.md5(open(bagit_path, "rb").read()).hexdigest()
+        )
+        bag_info_md5_hash = await run_in_threadpool(
+            lambda: hashlib.md5(open(bag_info_path, "rb").read()).hexdigest()
+        )
+
+        await run_in_threadpool(
+            lambda: open(tagmanifest_path, "w").write(
+                bagit_md5_hash
+                + " "
+                + "bagit.txt"
+                + "\n"
+                + manifest_md5_hash
+                + " "
+                + "manifest-md5.txt"
+                + "\n"
+                + bag_info_md5_hash
+                + " "
+                + "bag-info.txt"
+                + "\n"
+            )
+        )
+
         crate.add_file(
             tagmanifest_path,
             dest_path="tagmanifest-md5.txt",
@@ -1317,13 +1375,16 @@ async def download_dataset(
         )
         zip_name = dataset.name + version_name + ".zip"
         path_to_zip = os.path.join(current_temp_dir, zip_name)
-        crate.write_zip(path_to_zip)
-        f = open(path_to_zip, "rb", buffering=0)
-        zip_bytes = f.read()
+
+        await run_in_threadpool(crate.write_zip, path_to_zip)  # takes the most time?
+
+        f = await run_in_threadpool(open, path_to_zip, "rb", 0)
+        zip_bytes = await run_in_threadpool(f.read)
         stream = io.BytesIO(zip_bytes)
-        f.close()
+        await run_in_threadpool(f.close)
+
         try:
-            shutil.rmtree(current_temp_dir)
+            await run_in_threadpool(shutil.rmtree, current_temp_dir)
         except Exception as e:
             print("could not delete file")
             print(e)

backend/app/routers/keycloak.py

@@ -11,7 +11,8 @@
     retreive_refresh_token,
 )
 from app.models.tokens import TokenDB
-from app.models.users import UserDB, UserIn
+from app.models.users import UserDB, UserLogin
+from app.routers.utils import save_refresh_token
 from fastapi import APIRouter, HTTPException, Security
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from jose import ExpiredSignatureError, JWTError, jwt
@@ -80,10 +81,11 @@ async def logout(
 
 
 @router.post("/login")
-async def loginPost(userIn: UserIn):
+async def loginPost(userIn: UserLogin):
     """Client can use this to login when redirect is not available."""
     try:
         token = keycloak_openid.token(userIn.email, userIn.password)
+        await save_refresh_token(token["refresh_token"], userIn.email)
         return {"token": token["access_token"]}
     # bad credentials
     except KeycloakAuthenticationError as e: