Fixed media-types

Author: vaaale

Dockerfile

@@ -45,6 +45,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     ffmpeg \
     libsm6 \
     libxext6 \
+    media-types \
     htop \
     vim \
     wget \

Dockerfile-gpu

@@ -1,7 +1,7 @@
 FROM pytorch/pytorch:2.6.0-cuda12.6-cudnn9-devel
 #FROM pytorch/pytorch:2.6.0-cuda12.4-cudnn9-runtime
 
-RUN apt-get update && apt install ffmpeg libsm6 libxext6 nvtop wget htop vim -y
+RUN apt-get update && apt install ffmpeg libsm6 libxext6 media-types nvtop wget htop vim -y
 
 # Set working directory
 WORKDIR /app

docker-compose.yaml

@@ -1,7 +1,7 @@
 services:
 
   fetchcraft-admin:
-    image: fetchcraft
+    image: fetchcraft:latest
     container_name: fetchcraft-admin
     entrypoint: ["uv", "run", "fetchcraft-admin"]
     hostname: fetchcraft-admin
@@ -17,7 +17,7 @@ services:
     depends_on:
       - qdrant
       - mongodb
-      - fetchcraft-docling-server
+      - pgvector
     restart: unless-stopped
     networks: [ 'demo-net' ]
 
@@ -65,7 +65,7 @@ services:
     networks: [ 'demo-net' ]
 
   librechat:
-    image: librechat-new
+    image: librechat
 #    image: librechat-dev-alex2
 #    image: ghcr.io/danny-avila/librechat-dev-api:latest
     container_name: librechat
@@ -99,6 +99,7 @@ services:
   qdrant:
     image: qdrant/qdrant
     hostname: fetchcraft-qdrant
+    container_name: fetchcraft-qdrant
     ports:
       - "6333:6333"
       - "6334:6334"
@@ -170,6 +171,29 @@ services:
 #    env_file:
 #      - librechat.env
 
+  compass:
+    image: haohanyang/compass-web
+    container_name: mongodb-compass
+    environment:
+      - CW_MONGO_URI=mongodb://mongodb:27017
+    depends_on:
+      - mongodb
+    ports:
+      - 9010:8080
+    networks: [ 'demo-net' ]
+
+  pgadmin:
+    image: dpage/pgadmin4
+    container_name: pgadmin4_container
+    ports:
+      - "9020:80"
+    environment:
+      PGADMIN_DEFAULT_EMAIL: alexander.vaagan@crayon.com
+      PGADMIN_DEFAULT_PASSWORD: strong-password
+    volumes:
+      - pgadmin-data:/var/lib/pgadmin
+    networks: [ 'demo-net' ]
+
 networks:
   demo-net:
     driver: bridge
@@ -178,4 +202,5 @@ volumes:
   mongodb-data:
   qdrant-data:
   meili_data:
-  pg-data:
+  pg-data:
+  pgadmin-data:

packages/fetchcraft-core/src/fetchcraft/connector/base.py

@@ -1,3 +1,4 @@
+import logging
 import mimetypes
 from abc import ABC, abstractmethod
 from pathlib import Path
@@ -7,6 +8,8 @@
 import fsspec
 from pydantic import BaseModel, Field
 
+logger = logging.getLogger(__name__)
+
 
 class Role(BaseModel):
     name: str
@@ -24,7 +27,7 @@ class File(BaseModel, ABC):
     def __init__(self, path: Path, fs: fsspec.AbstractFileSystem, mimetype: Optional[str] = None, encoding: Optional[str] = None, **kwargs):
         if mimetype is None or encoding is None:
             _mimetype, _encoding = mimetypes.guess_type(path) or "application/octet-stream"
-            mimetype = mimetype or _mimetype or "text/plain"
+            mimetype = mimetype or _mimetype or "application/octet-stream"
             encoding = encoding or _encoding or "utf-8"
 
         super().__init__(path=path, fs=fs, mimetype=mimetype, encoding=encoding, **kwargs)

packages/fetchcraft-core/src/fetchcraft/connector/filesystem.py

@@ -67,6 +67,8 @@ def _getpwgid(gid: int) -> str:
         st = self.fs.stat(str(self.path))
         return {
             "filename": self.path.name,
+            "mimetype": self.mimetype,
+            "encoding": self.encoding,
             "source": str(self.path),
             "size": st["size"],
             "modified": st["mtime"],

packages/fetchcraft-core/src/fetchcraft/ingestion/transformations.py

@@ -127,6 +127,7 @@ async def process(
         file_content = base64.b64decode(record.content)
 
         # Select parser based on mimetype
+        logger.info(f"Getting parser for {file_path} -> {mimetype}")
         parser = self.parser_map.get(mimetype, self.parser_map.get("default", None))
 
         if not parser:
@@ -143,11 +144,8 @@ async def process(
 
         # Build metadata to pass to parser
         parser_metadata = {
-            "source": source,
             "path": file_path,
-            "mimetype": mimetype,
-            # Include any additional metadata from the record
-            **dict(record),
+            **record.metadata()
         }
 
         # Check if parser is remote (async callback-based)