Skip to content

Commit a2c36b4

Browse files
HenryHengZJHenryHengZJ
authored
Feature/Add new doc store upsert and refresh API (#3556)
add new doc store upsert and refresh API
1 parent 36496b1 commit a2c36b4

File tree

15 files changed

+1417
-796
lines changed

15 files changed

+1417
-796
lines changed

‎packages/api-documentation/src/yml/swagger.yml‎

Lines changed: 130 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,10 @@ paths:
305305
type: string
306306
format: binary
307307
description: Files to be uploaded
308+
base64:
309+
type: boolean
310+
default: false
311+
description: Return contents of the files in base64 format
308312
required:
309313
- files
310314
required: true
@@ -618,171 +622,109 @@ paths:
618622
'500':
619623
description: Internal server error
620624

621-
/document-store/loader/preview:
625+
/document-store/upsert/{id}:
622626
post:
623627
tags:
624628
- document-store
625629
security:
626630
- bearerAuth: []
627-
summary: Preview document chunks
628-
description: Preview document chunks from loader
629-
operationId: previewChunking
631+
summary: Upsert new document to document store
632+
description: Upsert new document to document store
633+
operationId: upsertDocument
634+
parameters:
635+
- in: path
636+
name: id
637+
required: true
638+
schema:
639+
type: string
640+
format: uuid
641+
description: Document store ID
630642
requestBody:
631643
content:
632644
application/json:
633645
schema:
634-
$ref: '#/components/schemas/DocumentStoreLoaderForPreview'
635-
required: true
636-
responses:
637-
'200':
638-
description: Successfully preview chunks
639-
content:
640-
application/json:
641-
schema:
642-
type: object
643-
properties:
644-
chunks:
645-
type: array
646-
items:
647-
$ref: '#/components/schemas/Document'
648-
totalChunks:
649-
type: integer
650-
example: 10
651-
previewChunkCount:
652-
type: integer
653-
example: 5
654-
'400':
655-
description: Invalid request body
656-
'500':
657-
description: Internal server error
658-
659-
/document-store/loader/process:
660-
post:
661-
tags:
662-
- document-store
663-
security:
664-
- bearerAuth: []
665-
summary: Process loading & chunking operation
666-
description: Process loading & chunking operation of document from loader
667-
operationId: processChunking
668-
requestBody:
669-
content:
670-
application/json:
646+
$ref: '#/components/schemas/DocumentStoreLoaderForUpsert'
647+
multipart/form-data:
671648
schema:
672649
type: object
673-
required:
674-
- storeId
675-
- id
676650
properties:
677-
storeId:
678-
type: string
679-
description: Document store ID
680-
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
681-
id:
651+
files:
652+
type: array
653+
items:
654+
type: string
655+
format: binary
656+
description: Files to be uploaded
657+
loader:
682658
type: string
683-
description: Document loader ID. If your URL is /document-stores/{storeId}/{id}, then id is the last part of the URL
684-
example: 'c427e569-b81a-469a-b14c-fa73dd5bae49'
685-
required: true
686-
responses:
687-
'200':
688-
description: Successfully process chunking operation
689-
content:
690-
application/json:
691-
schema:
692-
$ref: '#/components/schemas/DocumentStoreFileChunkPagedResponse'
693-
694-
'400':
695-
description: Invalid request body
696-
'500':
697-
description: Internal server error
698-
699-
/document-store/vectorstore/save:
700-
post:
701-
tags:
702-
- document-store
703-
security:
704-
- bearerAuth: []
705-
summary: Save upsert configuration of document store
706-
description: Save upsert configuration of document store
707-
operationId: saveVectorStoreConfig
708-
requestBody:
709-
content:
710-
application/json:
711-
schema:
712-
type: object
713-
required:
714-
- storeId
715-
properties:
716-
storeId:
659+
nullable: true
660+
example: '{"name":"plainText","config":{"text":"why the sky is blue"}}'
661+
description: Loader configurations
662+
splitter:
717663
type: string
718-
description: Document store ID
719-
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
720-
embeddingName:
664+
nullable: true
665+
example: '{"name":"recursiveCharacterTextSplitter","config":{"chunkSize":2000}}'
666+
description: Splitter configurations
667+
embedding:
721668
type: string
722-
description: Name of the embedding
723-
example: 'openAIEmbeddings'
724-
embeddingConfig:
725-
type: object
726-
description: Configuration of the embedding
727-
example: { 'model': 'text-embedding-ada-002', 'credential': '1eba5808-c55b-4817-a285-b0c92846a7ad' }
728-
vectorStoreName:
669+
nullable: true
670+
example: '{"name":"openAIEmbeddings","config":{"modelName":"text-embedding-ada-002"}}'
671+
description: Embedding configurations
672+
vectorStore:
729673
type: string
730-
description: Name of the vector store
731-
example: 'faiss'
732-
vectorStoreConfig:
733-
type: object
734-
description: Configuration of the embedding
735-
example: { 'basePath': './faiss' }
736-
recordManagerName:
674+
nullable: true
675+
example: '{"name":"faiss"}'
676+
description: Vector Store configurations
677+
recordManager:
737678
type: string
738-
description: Name of the record manager
739-
example: 'SQLiteRecordManager'
740-
recordManagerConfig:
741-
type: object
742-
description: Configuration of the embedding
743-
example: { 'databaseFilePath': './recordManager.db' }
679+
nullable: true
680+
example: '{"name":"postgresRecordManager"}'
681+
description: Record Manager configurations
682+
required:
683+
- files
744684
required: true
745685
responses:
746686
'200':
747-
description: Successfully save upsert configuration of document store
687+
description: Successfully execute upsert operation
748688
content:
749689
application/json:
750690
schema:
751-
$ref: '#/components/schemas/DocumentStore'
691+
$ref: '#/components/schemas/VectorUpsertResponse'
752692

753693
'400':
754694
description: Invalid request body
755695
'500':
756696
description: Internal server error
757697

758-
/document-store/vectorstore/insert:
698+
/document-store/refresh/{id}:
759699
post:
760700
tags:
761701
- document-store
762702
security:
763703
- bearerAuth: []
764-
summary: Upsert chunks from document store
765-
description: Upsert chunks from document store using the saved configuration
766-
operationId: insertIntoVectorStore
704+
summary: Re-process and upsert all documents in document store
705+
description: Re-process and upsert all existing documents in document store
706+
operationId: refreshDocument
707+
parameters:
708+
- in: path
709+
name: id
710+
required: true
711+
schema:
712+
type: string
713+
format: uuid
714+
description: Document store ID
767715
requestBody:
768716
content:
769717
application/json:
770718
schema:
771-
type: object
772-
required:
773-
- storeId
774-
properties:
775-
storeId:
776-
type: string
777-
description: Document store ID
778-
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
719+
$ref: '#/components/schemas/DocumentStoreLoaderForRefresh'
779720
required: true
780721
responses:
781722
'200':
782-
description: Successfully save upsert configuration of document store
723+
description: Successfully execute refresh operation
783724
content:
784725
application/json:
785-
schema:
726+
type: array
727+
items:
786728
$ref: '#/components/schemas/VectorUpsertResponse'
787729

788730
'400':
@@ -2220,6 +2162,72 @@ components:
22202162
description:
22212163
type: string
22222164

2165+
DocumentStoreLoaderForUpsert:
2166+
type: object
2167+
properties:
2168+
docId:
2169+
type: string
2170+
format: uuid
2171+
description: Document ID within the store. If provided, existing configuration from the document will be used for the new document
2172+
loader:
2173+
type: object
2174+
properties:
2175+
name:
2176+
type: string
2177+
example: plainText
2178+
description: Name of the loader (camelCase)
2179+
config:
2180+
type: object
2181+
description: Configuration for the loader
2182+
splitter:
2183+
type: object
2184+
properties:
2185+
name:
2186+
type: string
2187+
example: recursiveCharacterTextSplitter
2188+
description: Name of the text splitter (camelCase)
2189+
config:
2190+
type: object
2191+
description: Configuration for the text splitter
2192+
embedding:
2193+
type: object
2194+
properties:
2195+
name:
2196+
type: string
2197+
example: openAIEmbeddings
2198+
description: Name of the embedding generator (camelCase)
2199+
config:
2200+
type: object
2201+
description: Configuration for the embedding generator
2202+
vectorStore:
2203+
type: object
2204+
properties:
2205+
name:
2206+
type: string
2207+
example: faiss
2208+
description: Name of the vector store (camelCase)
2209+
config:
2210+
type: object
2211+
description: Configuration for the vector store
2212+
recordManager:
2213+
type: object
2214+
properties:
2215+
name:
2216+
type: string
2217+
example: postgresRecordManager
2218+
description: Name of the record manager (camelCase)
2219+
config:
2220+
type: object
2221+
description: Configuration for the record manager
2222+
2223+
DocumentStoreLoaderForRefresh:
2224+
type: object
2225+
properties:
2226+
items:
2227+
type: array
2228+
items:
2229+
$ref: '#/components/schemas/DocumentStoreLoaderForUpsert'
2230+
22232231
ChatMessageFeedback:
22242232
type: object
22252233
properties:

0 commit comments

Comments
 (0)