fix(Weights loader): Add debug output and expose Docker log file

Author: danactive

.editorconfig

@@ -20,3 +20,7 @@ indent_size = 2
 [Makefile]
 indent_style = tab
 indent_size = 4
+
+[*.py]
+indent_style = space
+indent_size = 4

.gitignore

@@ -42,3 +42,4 @@ coverage
 tmp
 public/resized
 models
+weights.log

Makefile

@@ -1,18 +1,31 @@
+login-check:
+	huggingface-cli whoami
+
 login:
 	@echo "🔐 Logging in to Hugging Face CLI..."
 	huggingface-cli login
 	@echo "Verify account"
 	huggingface-cli whoami
 
 load-weights:
+	@echo "Part 1/3 Build docker image"
 	docker build \
 		--build-arg MODEL_REPO="$(MODEL_REPO)" \
 		--build-arg FILENAMES="$(FILENAMES)" \
 		-f apps/load-weights/Dockerfile \
 		-t weights-loader .
-	docker create --name extract-model weights-loader
+	@echo "Part 2/3 Extracting weights from Hugging Face using cached credential"
+	docker run --name extract-model \
+		-v $$HOME/.cache/huggingface:/root/.cache/huggingface \
+		-e MODEL_REPO="$(MODEL_REPO)" \
+		-e FILENAMES="$(FILENAMES)" \
+		weights-loader \
+		sh -c 'python /dock/hugging-offline.py --repo-id "$$MODEL_REPO" --filenames "$$FILENAMES" > /dock/build.log 2>&1'
+	@echo "Part 3/3 Copying weights to local directory and display log"
 	docker cp extract-model:/dock/models/. ./models/
+	docker cp extract-model:/dock/build.log ./weights.log || true
 	docker rm extract-model
+	cat ./weights.log || true
 
 build-ai-api:
 	docker build -f apps/api/Dockerfile -t ai-api .

apps/api/README.md

@@ -36,6 +36,6 @@ This project serves a high-accuracy image classification API using a **Vision Tr
 - **Aesthetic scoring** using a trained multilayer perceptron (MLP)
 - Deterministic evaluation
 - A custom regression MLP head trained on:
-  	- AVA dataset (aesthetics annotated)
+  	- [AVA dataset (aesthetics annotated)](https://github.com/microsoft/LMOps/blob/main/promptist/aesthetic/ava%2Blogos-l14-linearMSE.pth)
 	- Logos subset (for composition)
 - [Code sample](https://github.com/christophschuhmann/improved-aesthetic-predictor/tree/main)

apps/api/main.py

@@ -38,7 +38,6 @@ async def classify_endpoint(req: Request):
 @main_py_app.post("/scores")
 async def score_endpoint(req: Request):
     try:
-        score = await score_aesthetic(req)
-        return {"aesthetic_score": round(score, 3)}
+        return {"aesthetic_score": await score_aesthetic(req)}
     except Exception as e:
         return error_response(e)

apps/load-weights/Dockerfile

@@ -5,8 +5,6 @@ ARG FILENAMES
 
 ENV MODEL_REPO="${MODEL_REPO}"
 ENV FILENAMES="${FILENAMES}"
-# Ensures Python output shows up immediately
-ENV PYTHONUNBUFFERED=1
 
 WORKDIR /dock
 
@@ -17,4 +15,3 @@ COPY apps/load-weights/hugging-offline.py /dock/hugging-offline.py
 
 RUN echo "MODEL_REPO: '$MODEL_REPO'"
 RUN echo "FILENAMES: '$FILENAMES'"
-RUN python /dock/hugging-offline.py --repo-id "$MODEL_REPO" --filenames $(echo $FILENAMES)

apps/load-weights/README.md

@@ -5,17 +5,12 @@ Hugging Face timm https://huggingface.co/timm/convnext_large_mlp.laion2b_ft_augr
 
 
 ## Image classifier
-1. Build container `make build-load-weights MODEL_REPO=timm/eva02_large_patch14_clip_336.merged2b_ft_inat21 FILENAMES="pytorch_model.bin"`
-1. Run container
-```
-docker run --rm \
-  -v "$HOME/.cache/huggingface":/root/.cache/huggingface \
-  weights-loader
-```
-
-Then copy from the Hugging Face cache to the repo
-- `mkdir -p models/timm_eva02_large_patch14_clip_336.merged2b_ft_inat21`
-- ```
-cp $HOME/.cache/huggingface/hub/models--timm--eva02_large_patch14_clip_336.merged2b_ft_inat21/snapshots/*/pytorch_model.bin \
-   models/timm_eva02_large_patch14_clip_336.merged2b_ft_inat21/
-```
+
+`make load-weights MODEL_REPO=timm/eva02_large_patch14_clip_336.merged2b_ft_inat21 FILENAMES="pytorch_model.bin"`
+
+## Aesthetic Scorer
+
+`make load-weights MODEL_REPO=apple/DFN5B-CLIP-ViT-H-14-378 FILENAMES="open_clip_pytorch_model.bin open_clip_config.json"`
+
+
+`make load-weights MODEL_REPO=CrowsonKB/simulacra-aesthetic-model FILENAMES="simulacra_aesthetic_model.pth"`

apps/load-weights/hugging-offline.py

@@ -7,13 +7,27 @@
 import os
 import sys
 import logging
-import traceback
+import subprocess
 
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, force=True)
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
 
+def log_huggingface_whoami():
+    logger.info("🔑 Checking Hugging Face authentication with 'huggingface-cli whoami'...")
+    try:
+        result = subprocess.run(
+            ["huggingface-cli", "whoami"],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        logger.info(f"Hugging Face user: {result.stdout.strip()}")
+    except Exception as e:
+        logger.error(f"❌ Unable to verify Hugging Face authentication: {e}")
+
 def download_weights(repo_id: str, filenames: list[str]):
+    log_huggingface_whoami()
     # Create destination path
     dest_dir = os.path.join("models", repo_id.replace("/", "_"))
     os.makedirs(dest_dir, exist_ok=True)
@@ -22,28 +36,37 @@ def download_weights(repo_id: str, filenames: list[str]):
         logger.info(f"📥 Downloading {filename} from {repo_id} on Hugging Face Hub...")
         try:
             path = hf_hub_download(repo_id=repo_id, filename=filename)
-
             # Move and rename
             dest_path = os.path.join(dest_dir, filename)
             shutil.copy(path, dest_path)
+            logger.info(f"✅ Saved {filename} to {dest_path}")
         except (RepositoryNotFoundError, HfHubHTTPError) as e:
             logger.error(f"❌ Failed to fetch '{filename}' from '{repo_id}': {e}")
             continue
 
-    logger.info(f"✅ All saved to: {dest_dir}")
+    # Verification step
+    missing = []
+    for filename in filenames:
+        dest_path = os.path.join(dest_dir, filename)
+        if not os.path.isfile(dest_path):
+            missing.append(filename)
+    if missing:
+        logger.error(f"❌ The following files are missing after download: {missing}")
+    else:
+        logger.info(f"✅ All expected files present in: {dest_dir}")
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Download Hugging Face model weights")
-    parser.add_argument(
-        "--repo-id",
-        required=True,
-        help="Hugging Face repository ID (e.g., openai/clip-vit-base-patch32)"
-    )
-    parser.add_argument(
-        "--filenames",
-        nargs="+",
-        default=["pytorch_model.bin", "config.json"],
-        help="List of filenames to download (space-separated)"
-    )
-    args = parser.parse_args()
-    download_weights(args.repo_id, args.filenames)
+  parser = argparse.ArgumentParser(description="Download Hugging Face model weights")
+  parser.add_argument(
+    "--repo-id",
+    required=True,
+    help="Hugging Face repository ID (e.g., openai/clip-vit-base-patch32)"
+  )
+  parser.add_argument(
+    "--filenames",
+    nargs="+",
+    default=["pytorch_model.bin", "config.json"],
+    help="List of filenames to download (space-separated)"
+  )
+  args = parser.parse_args()
+  download_weights(args.repo_id, args.filenames)