Include parameters with extractor (#770)

* include parameters

* black format

* fix the content type guess

* revert

* black format

* return correct userout for api key too

* black

* add extractors yml for testing

* update the script

* use the latest tag

Author: longshuicy

backend/app/keycloak_auth.py

@@ -183,8 +183,8 @@ async def get_current_user(
                     ListenerAPIKeyDB.key == payload["key"],
                 )
             ) is not None:
-                user_out = await UserDB.find_one(UserDB.email == key.user)
-                return user_out.dict()
+                user = await UserDB.find_one(UserDB.email == key.user)
+                return UserOut(**user.dict())
             elif (
                 key := await UserAPIKeyDB.find_one(
                     UserAPIKeyDB.user == payload["user"],
@@ -203,7 +203,7 @@ async def get_current_user(
                     )
                 else:
                     user = await UserDB.find_one(UserDB.email == key.user)
-                    return user.dict()
+                    return UserOut(**user.dict())
             else:
                 raise HTTPException(
                     status_code=401,

backend/app/models/listeners.py

@@ -160,6 +160,7 @@ class EventListenerJobMessage(BaseModel):
     id: str
     datasetId: str
     job_id: str
+    parameters: Optional[dict] = None
 
 
 class EventListenerDatasetJobMessage(BaseModel):

backend/app/rabbitmq/listeners.py

@@ -7,13 +7,11 @@
 from pika.adapters.blocking_connection import BlockingChannel
 
 from app import dependencies
-from app.keycloak_auth import get_token
 from app.models.config import ConfigEntryDB
 from app.models.datasets import DatasetOut
 from app.models.files import FileOut
 from app.models.listeners import (
     EventListenerJobDB,
-    EventListenerDB,
     EventListenerJobMessage,
     EventListenerDatasetJobMessage,
 )
@@ -76,6 +74,7 @@ async def submit_file_job(
         datasetId=str(file_out.dataset_id),
         secretKey=current_secretKey,
         job_id=str(job.id),
+        parameters=parameters,
     )
     reply_to = await create_reply_queue()
     print("RABBITMQ_CLIENT: " + str(rabbitmq_client))

backend/app/routers/utils.py

@@ -1,5 +1,4 @@
 import mimetypes
-import io
 from typing import Optional
 
 from app.models.files import ContentType
@@ -19,5 +18,8 @@ def get_content_type(
     if content_type is None:
         content_type = mimetypes.guess_type(filename)
         content_type = content_type[0] if len(content_type) > 1 else content_type
+        # If still cant guess the content_type, set it to default
+        if content_type is None:
+            content_type = "application/octet-stream"
     type_main = content_type.split("/")[0] if type(content_type) is str else "N/A"
     return ContentType(content_type=content_type, main_type=type_main)

docker-compose.extractors.yml

@@ -0,0 +1,52 @@
+version: '3.7'
+
+
+services:
+  name-entity-recognition:
+    image: socialmediamacroscope/name_entity_recognition_extractor:latest
+    environment:
+      CLOWDER_VERSION: 2
+      RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F
+    networks:
+      - clowder2
+    restart: unless-stopped
+
+  network-analysis:
+    image: socialmediamacroscope/network_analysis_extractor:latest
+    environment:
+      CLOWDER_VERSION: 2
+      RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F
+    networks:
+      - clowder2
+    restart: unless-stopped
+
+  topic-modeling:
+    image: socialmediamacroscope/topic_modeling_extractor:latest
+    environment:
+      CLOWDER_VERSION: 2
+      RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F
+    networks:
+      - clowder2
+    restart: unless-stopped
+
+  natural-language-preprocessing:
+    image: socialmediamacroscope/preprocessing_extractor:latest
+    environment:
+      CLOWDER_VERSION: 2
+      RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F
+    networks:
+      - clowder2
+    restart: unless-stopped
+
+  sentiment-analysis:
+    image: socialmediamacroscope/sentiment_analysis_extractor:latest
+    environment:
+      CLOWDER_VERSION: 2
+      RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F
+    networks:
+      - clowder2
+    restart: unless-stopped
+
+
+networks:
+  clowder2: