feat: support web search in collection (#1451)

Author: earayu

Makefile

@@ -216,7 +216,7 @@ evaluate:
 # OpenAPI and model generation
 .PHONY: merge-openapi generate-models generate-frontend-sdk
 merge-openapi:
-	@cd aperag && redocly bundle ./api/openapi.yaml > ./api/openapi.merged.yaml
+	@cd aperag && npx --yes @redocly/cli bundle ./api/openapi.yaml > ./api/openapi.merged.yaml
 
 generate-models: merge-openapi
 	@datamodel-codegen \

aperag/api/components/schemas/document.yaml

@@ -251,3 +251,89 @@ confirmDocumentsResponse:
   required:
     - confirmed_count
     - failed_count
+
+fetchUrlRequest:
+  type: object
+  properties:
+    urls:
+      type: array
+      items:
+        type: string
+        format: uri
+      minItems: 1
+      maxItems: 10
+      description: List of URLs to fetch and import (max 10)
+      example:
+        - "https://example.com/article1"
+        - "https://example.com/article2"
+  required:
+    - urls
+
+fetchUrlResultItem:
+  type: object
+  properties:
+    url:
+      type: string
+      description: The source URL
+    fetch_status:
+      type: string
+      enum:
+        - success
+        - error
+      description: Whether the URL was fetched successfully
+    document_id:
+      type: string
+      description: ID of the created document (only present on success)
+    filename:
+      type: string
+      description: Filename of the created document (only present on success)
+    size:
+      type: integer
+      description: Size of the created document in bytes (only present on success)
+    status:
+      type: string
+      description: Document status (only present on success)
+    error:
+      type: string
+      description: Error message (only present on failure)
+  required:
+    - url
+    - fetch_status
+
+fetchUrlResponse:
+  type: object
+  properties:
+    results:
+      type: array
+      items:
+        $ref: '#/fetchUrlResultItem'
+      description: Results for each URL
+    total:
+      type: integer
+      description: Total number of URLs processed
+    succeeded:
+      type: integer
+      description: Number of URLs successfully fetched
+    failed:
+      type: integer
+      description: Number of URLs that failed
+  required:
+    - results
+    - total
+    - succeeded
+    - failed
+
+stagedDocumentsResponse:
+  type: object
+  properties:
+    documents:
+      type: array
+      items:
+        $ref: '#/uploadDocumentResponse'
+      description: List of staged (UPLOADED) documents awaiting confirmation
+    total:
+      type: integer
+      description: Total number of staged documents
+  required:
+    - documents
+    - total

aperag/api/openapi.yaml

@@ -73,6 +73,10 @@ paths:
     $ref: './paths/collections.yaml#/upload_document'
   /collections/{collection_id}/documents/confirm:
     $ref: './paths/collections.yaml#/confirm_documents'
+  /collections/{collection_id}/documents/fetch-url:
+    $ref: './paths/collections.yaml#/fetch_url_document'
+  /collections/{collection_id}/documents/staged:
+    $ref: './paths/collections.yaml#/list_staged_documents'
   /collections/{collection_id}/searches:
     $ref: './paths/collections.yaml#/searches'
   /collections/{collection_id}/searches/{search_id}:

aperag/api/paths/collections.yaml

@@ -664,6 +664,98 @@ confirm_documents:
             schema:
               $ref: '../components/schemas/common.yaml#/failResponse'
 
+fetch_url_document:
+  post:
+    summary: Fetch documents from URLs
+    description: |
+      Fetch web page content from one or more URLs and create UPLOADED documents.
+      Each URL is fetched using the web read service (JINA with Trafilatura fallback).
+      Successfully fetched URLs produce UPLOADED documents in the staging area,
+      identical to file uploads. Use the confirm endpoint to move them to PENDING and start indexing.
+    security:
+      - BearerAuth: []
+    parameters:
+      - name: collection_id
+        in: path
+        required: true
+        schema:
+          type: string
+    requestBody:
+      required: true
+      content:
+        application/json:
+          schema:
+            $ref: '../components/schemas/document.yaml#/fetchUrlRequest'
+          examples:
+            single_url:
+              summary: Single URL
+              value:
+                urls:
+                  - "https://example.com/article"
+            multiple_urls:
+              summary: Multiple URLs
+              value:
+                urls:
+                  - "https://example.com/article1"
+                  - "https://example.com/article2"
+    responses:
+      '200':
+        description: URL fetch completed (partial success is also 200)
+        content:
+          application/json:
+            schema:
+              $ref: '../components/schemas/document.yaml#/fetchUrlResponse'
+      '400':
+        description: Bad request - invalid URLs or too many URLs
+        content:
+          application/json:
+            schema:
+              $ref: '../components/schemas/common.yaml#/failResponse'
+      '401':
+        description: Unauthorized
+        content:
+          application/json:
+            schema:
+              $ref: '../components/schemas/common.yaml#/failResponse'
+      '404':
+        description: Collection not found
+        content:
+          application/json:
+            schema:
+              $ref: '../components/schemas/common.yaml#/failResponse'
+
+list_staged_documents:
+  get:
+    summary: List staged documents
+    description: Returns all UPLOADED (staged) documents for the collection that are awaiting confirmation.
+    security:
+      - BearerAuth: []
+    parameters:
+      - name: collection_id
+        in: path
+        required: true
+        schema:
+          type: string
+    responses:
+      '200':
+        description: Staged documents retrieved successfully
+        content:
+          application/json:
+            schema:
+              $ref: '../components/schemas/document.yaml#/stagedDocumentsResponse'
+      '401':
+        description: Unauthorized
+        content:
+          application/json:
+            schema:
+              $ref: '../components/schemas/common.yaml#/failResponse'
+      '404':
+        description: Collection not found
+        content:
+          application/json:
+            schema:
+              $ref: '../components/schemas/common.yaml#/failResponse'
+
 searches:
   get:
     summary: Get search history

aperag/schema/view_models.py

@@ -14,14 +14,23 @@
 
 # generated by datamodel-codegen:
 #   filename:  openapi.merged.yaml
-#   timestamp: 2026-03-04T08:41:38+00:00
+#   timestamp: 2026-03-09T07:45:40+00:00
 
 from __future__ import annotations
 
 from datetime import datetime
 from typing import Any, Literal, Optional, Union
 
-from pydantic import BaseModel, ConfigDict, EmailStr, Field, RootModel, confloat, conint
+from pydantic import (
+    AnyUrl,
+    BaseModel,
+    ConfigDict,
+    EmailStr,
+    Field,
+    RootModel,
+    confloat,
+    conint,
+)
 
 
 class ModelSpec(BaseModel):
@@ -874,6 +883,51 @@ class ConfirmDocumentsResponse(BaseModel):
     )
 
 
+class FetchUrlRequest(BaseModel):
+    urls: list[AnyUrl] = Field(
+        ...,
+        description='List of URLs to fetch and import (max 10)',
+        examples=[['https://example.com/article1', 'https://example.com/article2']],
+    )
+
+
+class FetchUrlResultItem(BaseModel):
+    url: str = Field(..., description='The source URL')
+    fetch_status: Literal['success', 'error'] = Field(
+        ..., description='Whether the URL was fetched successfully'
+    )
+    document_id: Optional[str] = Field(
+        None, description='ID of the created document (only present on success)'
+    )
+    filename: Optional[str] = Field(
+        None, description='Filename of the created document (only present on success)'
+    )
+    size: Optional[int] = Field(
+        None,
+        description='Size of the created document in bytes (only present on success)',
+    )
+    status: Optional[str] = Field(
+        None, description='Document status (only present on success)'
+    )
+    error: Optional[str] = Field(
+        None, description='Error message (only present on failure)'
+    )
+
+
+class FetchUrlResponse(BaseModel):
+    results: list[FetchUrlResultItem] = Field(..., description='Results for each URL')
+    total: int = Field(..., description='Total number of URLs processed')
+    succeeded: int = Field(..., description='Number of URLs successfully fetched')
+    failed: int = Field(..., description='Number of URLs that failed')
+
+
+class StagedDocumentsResponse(BaseModel):
+    documents: list[UploadDocumentResponse] = Field(
+        ..., description='List of staged (UPLOADED) documents awaiting confirmation'
+    )
+    total: int = Field(..., description='Total number of staged documents')
+
+
 class VectorSearchParams(BaseModel):
     topk: Optional[int] = Field(None, description='Top K results')
     similarity: Optional[confloat(ge=0.0, le=1.0)] = Field(