diff --git a/fern/apis/fai/openapi.json b/fern/apis/fai/openapi.json index 90a1f79a8..013020a8f 100644 --- a/fern/apis/fai/openapi.json +++ b/fern/apis/fai/openapi.json @@ -3365,6 +3365,437 @@ "internal" ] } + }, + "/sources/website/{domain}/index": { + "post": { + "tags": [ + "Website" + ], + "summary": "Index Website", + "description": "Start crawling and indexing a website.\nReturns a job_id to track the crawling progress.", + "operationId": "index_website", + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Domain" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IndexWebsiteRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IndexWebsiteResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "x-fern-audiences": [ + "customers" + ], + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/sources/website/{domain}/status": { + "get": { + "tags": [ + "Website" + ], + "summary": "Get Website Status", + "description": "Get the status of a website crawling job.", + "operationId": "get_website_status", + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Domain" + } + }, + { + "name": "job_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The job ID returned from the index endpoint", + "title": "Job Id" + }, + "description": "The job ID returned from the index endpoint" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetWebsiteStatusResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "x-fern-audiences": [ + "customers" + ], + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/sources/website/{domain}/{website_id}": { + "get": { + "tags": [ + "Website" + ], + "summary": "Get Website By Id", + "description": "Get a single indexed website page by ID.", + "operationId": "get_website_by_id", + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Domain" + } + }, + { + "name": "website_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Website Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetWebsiteResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "x-fern-audiences": [ + "customers" + ], + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/sources/website/{domain}": { + "get": { + "tags": [ + "Website" + ], + "summary": "Get Websites", + "description": "List all indexed website pages for a domain with pagination.", + "operationId": "get_websites", + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Domain" + } + }, + { + "name": "page", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "description": "The page number for pagination", + "default": 1, + "title": "Page" + }, + "description": "The page number for pagination" + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "description": "The number of sources per page", + "default": 100, + "title": "Limit" + }, + "description": "The number of sources per page" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetWebsitesResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "x-fern-audiences": [ + "customers" + ], + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/sources/website/{domain}/reindex": { + "post": { + "tags": [ + "Website" + ], + "summary": "Reindex Website", + "description": "Re-crawl a website by starting a new crawl job. The job will delete old pages before indexing.\nUses the configuration from the original index request.", + "operationId": "reindex_website", + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Domain" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReindexWebsiteRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReindexWebsiteResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "x-fern-audiences": [ + "customers" + ], + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/sources/website/{domain}/delete": { + "delete": { + "tags": [ + "Website" + ], + "summary": "Delete Website", + "description": "Delete all pages from a specific website by base URL.", + "operationId": "delete_website", + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Domain" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteWebsiteRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteWebsiteResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "x-fern-audiences": [ + "customers" + ], + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/sources/website/{domain}/delete-all": { + "delete": { + "tags": [ + "Website" + ], + "summary": "Delete All Websites", + "description": "Delete all indexed website pages for a domain.", + "operationId": "delete_all_websites", + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Domain" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteAllWebsitesResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "x-fern-audiences": [ + "customers" + ], + "security": [ + { + "bearerAuth": [] + } + ] + } } }, "components": { @@ -3779,6 +4210,26 @@ ], "title": "CreateQueryResponse" }, + "DeleteAllWebsitesResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success", + "description": "Whether all websites were successfully deleted" + }, + "pages_deleted": { + "type": "integer", + "title": "Pages Deleted", + "description": "Total number of pages deleted" + } + }, + "type": "object", + "required": [ + "success", + "pages_deleted" + ], + "title": "DeleteAllWebsitesResponse" + }, "DeleteDocumentRequest": { "properties": { "document_id": { @@ -3821,6 +4272,40 @@ ], "title": "DeleteGuidanceResponse" }, + "DeleteWebsiteRequest": { + "properties": { + "base_url": { + "type": "string", + "title": "Base Url", + "description": "The base URL of the website to delete (deletes all pages from this source)" + } + }, + "type": "object", + "required": [ + "base_url" + ], + "title": "DeleteWebsiteRequest" + }, + "DeleteWebsiteResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success", + "description": "Whether the website was successfully deleted" + }, + "pages_deleted": { + "type": "integer", + "title": "Pages Deleted", + "description": "Number of pages deleted" + } + }, + "type": "object", + "required": [ + "success", + "pages_deleted" + ], + "title": "DeleteWebsiteResponse" + }, "DiscordIntegrationResponse": { "properties": { "integration_url": { @@ -4428,6 +4913,89 @@ ], "title": "GetSettingsResponse" }, + "GetWebsiteResponse": { + "properties": { + "website": { + "$ref": "#/components/schemas/Website", + "description": "The requested website" + } + }, + "type": "object", + "required": [ + "website" + ], + "title": "GetWebsiteResponse" + }, + "GetWebsiteStatusResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id" + }, + "status": { + "type": "string", + "title": "Status", + "description": "Job status: PENDING, PROCESSING, COMPLETED, or FAILED" + }, + "base_url": { + "type": "string", + "title": "Base Url" + }, + "pages_indexed": { + "type": "integer", + "title": "Pages Indexed", + "description": "Number of pages successfully indexed" + }, + "pages_failed": { + "type": "integer", + "title": "Pages Failed", + "description": "Number of pages that failed to index" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error", + "description": "Error message if the job failed" + } + }, + "type": "object", + "required": [ + "job_id", + "status", + "base_url", + "pages_indexed", + "pages_failed" + ], + "title": "GetWebsiteStatusResponse" + }, + "GetWebsitesResponse": { + "properties": { + "websites": { + "items": { + "$ref": "#/components/schemas/Website" + }, + "type": "array", + "title": "Websites", + "description": "List of indexed website pages for the domain" + }, + "pagination": { + "$ref": "#/components/schemas/PaginationResponse", + "description": "Pagination information for the website list" + } + }, + "type": "object", + "required": [ + "websites", + "pagination" + ], + "title": "GetWebsitesResponse" + }, "GroupBy": { "type": "string", "enum": [ @@ -4539,32 +5107,202 @@ }, "type": "object", "required": [ - "repo_urls" + "repo_urls" + ], + "title": "IndexGithubRequest" + }, + "IndexGithubResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "Job ID for tracking indexing progress" + }, + "repo_urls": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Repo Urls", + "description": "GitHub repository URLs being indexed" + } + }, + "type": "object", + "required": [ + "job_id", + "repo_urls" + ], + "title": "IndexGithubResponse" + }, + "IndexWebsiteRequest": { + "properties": { + "base_url": { + "type": "string", + "title": "Base Url", + "description": "The base URL to start indexing from (e.g., 'https://docs.example.com')" + }, + "domain_filter": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain Filter", + "description": "Domain to filter crawling (e.g., 'docs.example.com'). Defaults to base_url domain." + }, + "path_filter": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path Filter", + "description": "Path prefix to restrict crawling (e.g., '/docs'). Only URLs starting with this will be crawled." + }, + "url_pattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url Pattern", + "description": "Regex pattern to filter URLs (e.g., 'https://example\\.com/(docs|api)/.*')." + }, + "chunk_size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Chunk Size", + "description": "Size of text chunks for splitting documents", + "default": 1000 + }, + "chunk_overlap": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Chunk Overlap", + "description": "Overlap between consecutive chunks", + "default": 200 + }, + "min_content_length": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Min Content Length", + "description": "Minimum content length to index a page", + "default": 100 + }, + "max_pages": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Pages", + "description": "Maximum number of pages to crawl. None means unlimited." + }, + "delay": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Delay", + "description": "Delay in seconds between requests", + "default": 1.0 + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version", + "description": "Version to tag all indexed pages with" + }, + "product": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Product", + "description": "Product to tag all indexed pages with" + }, + "authed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Authed", + "description": "Whether indexed pages should be auth-gated" + } + }, + "type": "object", + "required": [ + "base_url" ], - "title": "IndexGithubRequest" + "title": "IndexWebsiteRequest" }, - "IndexGithubResponse": { + "IndexWebsiteResponse": { "properties": { "job_id": { "type": "string", "title": "Job Id", - "description": "Job ID for tracking indexing progress" + "description": "ID to track the indexing job status" }, - "repo_urls": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Repo Urls", - "description": "GitHub repository URLs being indexed" + "base_url": { + "type": "string", + "title": "Base Url", + "description": "The base URL being indexed" } }, "type": "object", "required": [ "job_id", - "repo_urls" + "base_url" ], - "title": "IndexGithubResponse" + "title": "IndexWebsiteResponse" }, "IndexingCallbackRequest": { "properties": { @@ -4959,6 +5697,172 @@ ], "title": "ReconstructIndexResponse" }, + "ReindexWebsiteRequest": { + "properties": { + "base_url": { + "type": "string", + "title": "Base Url", + "description": "The base URL to re-crawl (will delete old pages and re-index)" + }, + "domain_filter": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain Filter", + "description": "Domain to filter crawling (e.g., 'docs.example.com'). If not provided, uses previous config." + }, + "path_filter": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path Filter", + "description": "Path prefix to restrict crawling (e.g., '/docs'). If not provided, uses previous config." + }, + "url_pattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url Pattern", + "description": "Regex pattern to filter URLs (e.g., 'https://example\\.com/(docs|api)/.*'). If not provided, uses previous config." + }, + "chunk_size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Chunk Size", + "description": "Size of text chunks for splitting documents. If not provided, uses previous config." + }, + "chunk_overlap": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Chunk Overlap", + "description": "Overlap between consecutive chunks. If not provided, uses previous config." + }, + "min_content_length": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Min Content Length", + "description": "Minimum content length to index a page. If not provided, uses previous config." + }, + "max_pages": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Pages", + "description": "Maximum number of pages to crawl. If not provided, uses previous config." + }, + "delay": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Delay", + "description": "Delay in seconds between requests. If not provided, uses previous config." + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version", + "description": "Version to tag all indexed pages with. If not provided, uses previous config." + }, + "product": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Product", + "description": "Product to tag all indexed pages with. If not provided, uses previous config." + }, + "authed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Authed", + "description": "Whether indexed pages should be auth-gated. If not provided, uses previous config." + } + }, + "type": "object", + "required": [ + "base_url" + ], + "title": "ReindexWebsiteRequest" + }, + "ReindexWebsiteResponse": { + "properties": { + "job_id": { + "type": "string", + "title": "Job Id", + "description": "ID to track the re-crawling job status" + }, + "base_url": { + "type": "string", + "title": "Base Url", + "description": "The base URL being re-crawled" + } + }, + "type": "object", + "required": [ + "job_id", + "base_url" + ], + "title": "ReindexWebsiteResponse" + }, "ScribeEditCallbackRequest": { "properties": { "editing_id": { @@ -5558,6 +6462,114 @@ "NO_CHANGE" ], "title": "VersionBump" + }, + "Website": { + "properties": { + "website_id": { + "type": "string", + "title": "Website Id" + }, + "domain": { + "type": "string", + "title": "Domain" + }, + "base_url": { + "type": "string", + "title": "Base Url" + }, + "page_url": { + "type": "string", + "title": "Page Url" + }, + "chunk": { + "type": "string", + "title": "Chunk" + }, + "document": { + "type": "string", + "title": "Document" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + }, + "product": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Product" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "authed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Authed" + }, + "created_at": { + "type": "string", + "format": "date-time", + "title": "Created At" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "Updated At" + } + }, + "type": "object", + "required": [ + "website_id", + "domain", + "base_url", + "page_url", + "chunk", + "document", + "created_at", + "updated_at" + ], + "title": "Website" } }, "securitySchemes": {