Merge remote-tracking branch 'jaychao/pr-jay-24' into merge_pr

Author: bchou9

.github/workflows/ci-quick.yml

@@ -18,6 +18,11 @@ env:
   NODE_VERSION: '20.x'
   PYTHON_VERSION: '3.10'
 
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
 jobs:
   # ============================================
   # Quick validation (lint, type check, fast tests)
@@ -106,8 +111,6 @@ jobs:
             "RESILIENTDB_GRAPHQL_URI=https://cloud.resilientdb.com/graphql" \
             "JWT_SECRET=test-secret-key-do-not-use-in-production" > .env
       
-      - name: Run backend unit tests only (fast)
-        # wait for mongodb to be ready on localhost:27017
       - name: Wait for MongoDB
         run: |
           for i in {1..30}; do
@@ -136,12 +139,14 @@ jobs:
             --ci
       
       - name: Build check (frontend)
+        env:
+          CI: false
         run: |
           cd frontend
           npm run build
       
       - name: Comment PR with quick results
-        if: github.event_name == 'pull_request' && always()
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && always()
         uses: actions/github-script@v7
         with:
           script: |

README.md

@@ -56,6 +56,7 @@ def filter(self, record):
 from routes.frontend import frontend_bp
 from routes.analytics import analytics_bp
 from routes.export import export_bp
+from routes.ai_assistant import ai_assistant_bp
 from services.db import redis_client
 from services.canvas_counter import get_canvas_draw_count
 from services.graphql_service import commit_transaction_via_graphql
@@ -215,6 +216,7 @@ def handle_all_exceptions(e):
 app.register_blueprint(submit_room_line_bp)
 app.register_blueprint(admin_bp)
 app.register_blueprint(export_bp)
+app.register_blueprint(ai_assistant_bp)
 
 # Register versioned API v1 blueprints for external applications
 from api_v1.auth import auth_v1_bp

backend/app.py

@@ -0,0 +1,208 @@
+from flask import Blueprint, request, jsonify
+# Import style transfer function as well
+from services.llm_service import (
+    prompt_to_drawings,
+    complete_shape_from_canvas,
+    beautify_canvas_state,
+    style_transfer_canvas,
+)
+from services.llm_service import recognize_objects_in_box
+# from services.image_generation_service import (
+#     text_to_image as img_text_to_image,
+# )
+import logging
+import base64
+import io
+
+ai_assistant_bp = Blueprint('ai_assistant', __name__)
+logger = logging.getLogger(__name__)
+
+
+@ai_assistant_bp.route('/api/ai_assistant/drawing', methods=['POST'])
+def text_to_drawings():
+    """
+    Body: { "prompt": "<natural language description>", canvasState: {json object} }
+    Returns: Parsed drawing JSON (shape/color/size/position/...) or an error payload.
+    """
+    try:
+        payload = request.get_json(silent=True) or {}
+        prompt = payload.get("prompt")
+        canvasState = payload.get("canvasState") or {}
+
+        if not isinstance(prompt, str) or not prompt.strip():
+            return jsonify({"error": "bad_request", "detail": "Missing or invalid 'prompt' (string)."}), 400
+
+        logger.info("AI drawing requested")
+        result = prompt_to_drawings(prompt.strip(), canvasState)
+
+        print(f"\n\nModel result: {result}\n\n")
+
+        # If services returned an error, surface it with 502 (bad upstream)
+        if isinstance(result, dict) and "error" in result:
+            logger.warning("AI drawing failed: %s", result)
+            return jsonify({"error": "upstream_model_error", "detail": result}), 502
+
+        return jsonify(result), 200
+    except Exception as e:
+        logger.exception("Unhandled error in /drawing")
+        return jsonify({"error": "server_error", "detail": str(e)}), 500
+
+
+@ai_assistant_bp.route('/api/ai_assistant/complete', methods=['POST'])
+def shape_completion():
+    """
+    Body: { "canvasState": { ... } }
+    Returns: { complete, confidence, object{ color, lineWidth, pathData{...} } } or an error payload.
+    """
+    try:
+        payload = request.get_json(silent=True) or {}
+        canvas_state = payload.get("canvasState")
+        if not isinstance(canvas_state, dict):
+            return jsonify({"error": "bad_request", "detail": "Missing or invalid 'canvas_state' (object)."}), 400
+
+        logger.info("AI shape completion requested")
+        suggestion = complete_shape_from_canvas(canvas_state)
+
+        if not isinstance(canvas_state, dict):
+            return jsonify({
+                "error": "bad_request",
+                "detail": "Missing or invalid 'canvasState' (object)."
+            }), 400
+
+        return jsonify(suggestion), 200
+    except Exception as e:
+        logger.exception("Unhandled error in /complete")
+        return jsonify({"error": "server_error", "detail": str(e)}), 500
+
+
+@ai_assistant_bp.route('/api/ai_assistant/image', methods=['POST'])
+def text_to_image():
+    """
+    TODO: To be implemented
+    Body: { "prompt": "<string>", "width"?: int, "height"?: int, "style"?: str }
+    Returns: { "imageDataUrl": "data:image/png;base64,..." }
+    """
+    try:
+        payload = request.get_json(silent=True) or {}
+        prompt = payload.get("prompt", "")
+        width = payload.get("width") or 512
+        height = payload.get("height") or 512
+        style = payload.get("style") or "default"
+
+        if not isinstance(prompt, str) or not prompt.strip():
+            return jsonify({
+                "error": "bad_request",
+                "detail": "Missing or invalid 'prompt' (string)."
+            }), 400
+
+            logger.info("AI text-to-image requested")
+
+            # Try to generate via image_generation_service
+            try:
+                from services.image_generation_service import text_to_image as img_text_to_image
+                pil_image = img_text_to_image(prompt.strip(), width=width, height=height, style=style)
+            except Exception as e:
+                logger.exception("Image generation failed: %s", e)
+                return jsonify({"error": "image_generation_failed", "detail": str(e)}), 502
+
+            buf = io.BytesIO()
+            pil_image.save(buf, format="PNG")
+            buf.seek(0)
+            encoded = base64.b64encode(buf.read()).decode("utf-8")
+            data_url = f"data:image/png;base64,{encoded}"
+
+            return jsonify({"imageDataUrl": data_url}), 200
+
+    except Exception as e:
+        logger.exception("Unhandled error in /image")
+        return jsonify({"error": "server_error", "detail": str(e)}), 500
+
+
+@ai_assistant_bp.route("/api/ai_assistant/beautify", methods=["POST"])
+def beautify_sketch():
+    try:
+        payload = request.get_json(silent=True) or {}
+        canvas_state = payload.get("canvasState")
+
+        if not isinstance(canvas_state, dict):
+            return jsonify({
+                "error": "bad_request",
+                "detail": "Missing or invalid 'canvasState' (object)."
+            }), 400
+
+        result = beautify_canvas_state(canvas_state)
+        # print("\n\ncanvas_state!!!", canvas_state, "\n\n")
+        # print("\n\nResult!!!", result, "\n\n")
+
+        if not isinstance(result, dict) or "objects" not in result:
+            logger.warning("Beautify returned invalid payload: %r", result)
+            return jsonify({
+                "error": "upstream_model_error",
+                "detail": "Beautify model returned invalid payload."
+            }), 502
+
+        return jsonify(result), 200
+
+    except Exception as e:
+        logger.exception("Unhandled error in /beautify")
+        return jsonify({"error": "server_error", "detail": str(e)}), 500
+
+
+@ai_assistant_bp.route('/api/ai_assistant/style', methods=['POST'])
+def style_transfer():
+    """
+    Body: { "canvasState": {...}, "stylePrompt": "<string describing style e.g. 'Van Gogh oil painting'" }
+    Returns: { "objects": [...] } or error
+    """
+    try:
+        payload = request.get_json(silent=True) or {}
+        canvas_state = payload.get('canvasState')
+        style_prompt = payload.get('stylePrompt')
+
+        if not isinstance(canvas_state, dict):
+            return jsonify({"error": "bad_request", "detail": "Missing or invalid 'canvasState' (object)."}), 400
+        if not isinstance(style_prompt, str) or not style_prompt.strip():
+            return jsonify({"error": "bad_request", "detail": "Missing or invalid 'stylePrompt' (string)."}), 400
+
+        logger.info('AI style transfer requested')
+        result = style_transfer_canvas(canvas_state, style_prompt.strip())
+
+        # If the model returned an error payload, log it and return a safe
+        # fallback: the original canvas objects so the client can continue.
+        if isinstance(result, dict) and "error" in result:
+            logger.warning('Style transfer model error, falling back to original canvas: %s', result)
+            original_objects = canvas_state.get("objects") or canvas_state.get("drawings") or []
+            return jsonify({"objects": original_objects}), 200
+
+        # Normal successful response
+        return jsonify(result), 200
+    except Exception as e:
+        logger.exception('Unhandled error in /style')
+        return jsonify({"error": "server_error", "detail": str(e)}), 500
+
+
+@ai_assistant_bp.route('/api/ai_assistant/recognize', methods=['POST'])
+def recognize():
+    """
+    Body: { "canvasObjects": [...], "box": { x,y,width,height }, "bounds": { width, height } }
+    Returns: { label, confidence, explanation } or error
+    """
+    try:
+        payload = request.get_json(silent=True) or {}
+        canvas_objects = payload.get('canvasObjects') or payload.get('objects') or []
+        box = payload.get('box') or {}
+        bounds = payload.get('bounds') or {}
+
+        if not isinstance(canvas_objects, list):
+            return jsonify({"error": "bad_request", "detail": "'canvasObjects' must be a list."}), 400
+
+        result = recognize_objects_in_box(canvas_objects, box, bounds)
+
+        if isinstance(result, dict) and 'error' in result:
+            logger.warning('Recognition upstream error: %s', result)
+            return jsonify({"error": "upstream_model_error", "detail": result}), 502
+
+        return jsonify(result), 200
+    except Exception as e:
+        logger.exception('Unhandled error in /recognize')
+        return jsonify({"error": "server_error", "detail": str(e)}), 500

backend/routes/ai_assistant.py

@@ -0,0 +1,61 @@
+
+from config import OPENAI_API_KEY
+
+
+def text_to_image(prompt: str, width: int = 512, height: int = 512, style: str = "default"):
+	"""
+	Generate an image for the given prompt. Returns a PIL.Image object on success.
+	This function attempts to use the OpenAI Images API if available; otherwise,
+	it falls back to generating a very small placeholder image so the endpoint
+	returns something useful for UI development.
+	"""
+	# Try OpenAI Images (if package available)
+	try:
+		from openai import OpenAI
+		from PIL import Image
+		import base64
+		import io
+
+		client = OpenAI(api_key=OPENAI_API_KEY)
+
+		# Attempt to use images.generate if available on the client
+		try:
+			resp = client.images.generate(
+				model="gpt-image-1",
+				prompt=prompt,
+				size=f"{width}x{height}"
+			)
+			# The response may contain base64 data depending on the SDK version
+			b64 = None
+			if isinstance(resp, dict) and resp.get("data") and isinstance(resp["data"], list):
+				item = resp["data"][0]
+				if isinstance(item, dict) and item.get("b64_json"):
+					b64 = item.get("b64_json")
+			if b64:
+				img_bytes = base64.b64decode(b64)
+				return Image.open(io.BytesIO(img_bytes))
+		except Exception:
+			# Fall back to other approaches below
+			pass
+
+	except Exception:
+		# openai or PIL not available - fall through to placeholder
+		pass
+
+	# Placeholder fallback: create a simple blank image (Pillow may be missing)
+	try:
+		from PIL import Image, ImageDraw, ImageFont
+		img = Image.new("RGBA", (width, height), (255, 255, 255, 255))
+		draw = ImageDraw.Draw(img)
+		# Draw a small placeholder label in the center if fonts available
+		try:
+			f = ImageFont.load_default()
+			text = "AI\nImage"
+			w, h = draw.multiline_textsize(text, font=f)
+			draw.multiline_text(((width - w) / 2, (height - h) / 2), text, fill=(120, 120, 120), font=f, align="center")
+		except Exception:
+			pass
+		return img
+	except Exception as e:
+		raise RuntimeError("No image generation backend available: " + str(e))
+