Skip to content

v6: Backups API #346

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bevzzz wants to merge 9 commits into
base: v6-collections
Choose a base branch
from
+2,227 −26
Open

v6: Backups API #346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d3adf35
chore(api): update contracts and generated models
bevzzz Mar 4, 2026
7b7c766
feat(backup): support backups in lower-level client
bevzzz Mar 4, 2026
75f3720
feat(internal/testkit): add tickingContext and ErrorIs utils
bevzzz Mar 4, 2026
a539cbc
feat(backup): add public backups API
bevzzz Mar 4, 2026
50094a9
chore: add better assert message
bevzzz Mar 4, 2026
dcf308f
fix(internal/testkit): do not require Map, Slice, and Chan to be poin…
bevzzz Mar 4, 2026
52f7fb1
test(internal/api): test BackupInfo unmarshaling
bevzzz Mar 4, 2026
bf7399c
chore(backup): make StartedAt and CompletedAt pointer variables
bevzzz Mar 4, 2026
21e7c3b
feat(backup): add Backup namespace to client
bevzzz Mar 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion api/proto/v1/base_search.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ message Hybrid {
repeated string properties = 2;
// protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED
repeated float vector = 3 [deprecated = true]; // will be removed in the future, use vectors
float alpha = 4;
float alpha = 4 [deprecated = true]; // deprecated in 1.36.0 - use alpha_param
enum FusionType {
FUSION_TYPE_UNSPECIFIED = 0;
FUSION_TYPE_RANKED = 1;
Expand All @@ -63,6 +63,7 @@ message Hybrid {
NearVector near_vector = 9; // same as above. Use the target vector in the hybrid message
Targets targets = 10;
optional SearchOperatorOptions bm25_search_operator = 11;
optional float alpha_param = 12;

// only vector distance, but keep it extendable
oneof threshold {
Expand Down
301 changes: 300 additions & 1 deletion api/rest/schema.check.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,154 @@
},
"type": "object"
},
"ExportCreateRequest": {
"description": "Request to create a new export operation",
"type": "object",
"required": ["id"],
"properties": {
"id": {
"type": "string",
"description": "Unique identifier for this export. Must be URL-safe."
},
"include": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of collection names to include in the export. Cannot be used with 'exclude'."
},
"exclude": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of collection names to exclude from the export. Cannot be used with 'include'."
},
"config": {
"type": "object",
"description": "Backend-specific configuration",
"properties": {
"bucket": {
"type": "string",
"description": "Bucket, container, or volume name for cloud storage backends"
},
"path": {
"type": "string",
"description": "Path prefix within the bucket or filesystem"
}
}
}
}
},
"ExportCreateResponse": {
"description": "Response from creating an export operation",
"type": "object",
"properties": {
"id": {
"type": "string",
"description": "Unique identifier for this export"
},
"backend": {
"type": "string",
"description": "The backend storage system used"
},
"path": {
"type": "string",
"description": "Full path where the export is being written"
},
"status": {
"type": "string",
"description": "Current status of the export",
"enum": ["STARTED"]
},
"startedAt": {
"type": "string",
"format": "date-time",
"description": "When the export started"
},
"classes": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of collections being exported"
}
}
},
"ExportStatusResponse": {
"description": "Current status of an export operation",
"type": "object",
"properties": {
"id": {
"type": "string",
"description": "Unique identifier for this export"
},
"backend": {
"type": "string",
"description": "The backend storage system used"
},
"path": {
"type": "string",
"description": "Full path where the export is stored"
},
"status": {
"type": "string",
"description": "Current status of the export",
"enum": ["STARTED", "TRANSFERRING", "SUCCESS", "FAILED"]
},
"startedAt": {
"type": "string",
"format": "date-time",
"description": "When the export started"
},
"tookInMs": {
"type": "integer",
"format": "int64",
"description": "Duration of the export in milliseconds"
},
"classes": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of collections in this export"
},
"shardStatus": {
"type": "object",
"additionalProperties": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/ShardProgress"
}
},
"description": "Per-shard progress: className -> shardName -> status"
},
"error": {
"type": "string",
"description": "Error message if export failed"
}
}
},
"ShardProgress": {
"description": "Progress information for exporting a single shard",
"type": "object",
"properties": {
"status": {
"type": "string",
"description": "Status of this shard's export",
"enum": ["STARTED", "TRANSFERRING", "SUCCESS", "FAILED"]
},
"objectsExported": {
"type": "integer",
"format": "int64",
"description": "Number of objects exported from this shard"
},
"error": {
"type": "string",
"description": "Error message if this shard's export failed"
}
}
},
"GraphQLError": {
"description": "An error response caused by a GraphQL query.",
"properties": {
Expand Down Expand Up @@ -1728,8 +1876,10 @@
"STARTED",
"TRANSFERRING",
"TRANSFERRED",
"FINALIZING",
"SUCCESS",
"FAILED",
"CANCELLING",
"CANCELED"
]
},
Expand Down Expand Up @@ -1777,8 +1927,10 @@
"STARTED",
"TRANSFERRING",
"TRANSFERRED",
"FINALIZING",
"SUCCESS",
"FAILED",
"CANCELLING",
"CANCELED"
]
}
Expand Down Expand Up @@ -1900,6 +2052,12 @@
"items": {
"type": "string"
}
},
"incremental_base_backup_id": {
"description": "The ID of an existing backup to use as the base for a file-based incremental backup. If set, only files that have changed since the base backup will be included in the new backup.",
"type": "string",
"x-nullable": true,
"default": null
}
}
},
Expand Down Expand Up @@ -1941,8 +2099,10 @@
"STARTED",
"TRANSFERRING",
"TRANSFERRED",
"FINALIZING",
"SUCCESS",
"FAILED",
"CANCELLING",
"CANCELED"
]
}
Expand Down Expand Up @@ -1972,8 +2132,10 @@
"STARTED",
"TRANSFERRING",
"TRANSFERRED",
"FINALIZING",
"SUCCESS",
"FAILED",
"CANCELLING",
"CANCELED"
]
},
Expand Down Expand Up @@ -2064,8 +2226,10 @@
"STARTED",
"TRANSFERRING",
"TRANSFERRED",
"FINALIZING",
"SUCCESS",
"FAILED",
"CANCELLING",
"CANCELED"
]
}
Expand Down Expand Up @@ -3345,7 +3509,7 @@
},
"description": "# Introduction<br/> Weaviate is an open source, AI-native vector database that helps developers create intuitive and reliable AI-powered applications. <br/> ### Base Path <br/>The base path for the Weaviate server is structured as `[YOUR-WEAVIATE-HOST]:[PORT]/v1`. As an example, if you wish to access the `schema` endpoint on a local instance, you would navigate to `http://localhost:8080/v1/schema`. Ensure you replace `[YOUR-WEAVIATE-HOST]` and `[PORT]` with your actual server host and port number respectively. <br/> ### Questions? <br/>If you have any comments or questions, please feel free to reach out to us at the community forum [https://forum.weaviate.io/](https://forum.weaviate.io/). <br/>### Issues? <br/>If you find a bug or want to file a feature request, please open an issue on our GitHub repository for [Weaviate](https://github.com/weaviate/weaviate). <br/>### Need more documentation? <br/>For a quickstart, code examples, concepts and more, please visit our [documentation page](https://docs.weaviate.io/weaviate).",
"title": "Weaviate REST API",
"version": "1.36.0-rc.0"
"version": "1.37.0-dev"
},
"parameters": {
"CommonAfterParameterQuery": {
Expand Down Expand Up @@ -9199,6 +9363,137 @@
}
}
},
"/export/{backend}": {
"post": {
"summary": "Start a new export",
"description": "Initiates an export operation that writes collections to Parquet files on the specified backend storage (S3, GCS, Azure, or filesystem). Each collection is exported to a separate Parquet file.",
"operationId": "export.create",
"x-serviceIds": [
"weaviate.local.export.create"
],
"tags": [
"export"
],
"parameters": [
{
"name": "backend",
"in": "path",
"required": true,
"type": "string",
"description": "The backend storage system to use for the export (e.g., `filesystem`, `gcs`, `s3`, `azure`)."
},
{
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/ExportCreateRequest"
}
}
],
"responses": {
"200": {
"description": "Successfully started export operation",
"schema": {
"$ref": "#/definitions/ExportCreateResponse"
}
},
"401": {
"description": "Unauthorized or invalid credentials"
},
"403": {
"description": "Forbidden - insufficient permissions",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
},
"422": {
"description": "Invalid export request",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
},
"500": {
"description": "Internal server error occurred while starting export",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
}
}
}
},
"/export/{backend}/{id}": {
"get": {
"summary": "Get export status",
"description": "Retrieves the current status of an export operation, including progress for each collection being exported.",
"operationId": "export.status",
"x-serviceIds": [
"weaviate.local.export.status"
],
"tags": [
"export"
],
"parameters": [
{
"name": "backend",
"in": "path",
"required": true,
"type": "string",
"description": "The backend storage system where the export is stored."
},
{
"name": "id",
"in": "path",
"required": true,
"type": "string",
"description": "The unique identifier of the export."
},
{
"name": "bucket",
"in": "query",
"required": false,
"type": "string",
"description": "Optional bucket name where the export is stored. If not specified, uses the backend's default bucket."
},
{
"name": "path",
"in": "query",
"required": false,
"type": "string",
"description": "Optional path prefix within the bucket. If not specified, uses the backend's default path."
}
],
"responses": {
"200": {
"description": "Successfully retrieved export status",
"schema": {
"$ref": "#/definitions/ExportStatusResponse"
}
},
"401": {
"description": "Unauthorized or invalid credentials"
},
"403": {
"description": "Forbidden - insufficient permissions",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
},
"404": {
"description": "Export not found",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
},
"500": {
"description": "Internal server error occurred while retrieving export status",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
}
}
}
},
"/cluster/statistics": {
"get": {
"summary": "Get cluster statistics",
Expand Down Expand Up @@ -9538,6 +9833,10 @@
"name": "backups",
"description": "Operations related to creating and managing backups of Weaviate data. This feature allows you to create snapshots of your collections and store them on external storage backends such as cloud object storage (S3, GCS, Azure) or a shared filesystem. These endpoints enable you to initiate backup and restore processes, monitor their status, list available backups on a backend, and delete unwanted backups. Backups are essential for disaster recovery, data migration, and maintaining point-in-time copies of your vector database."
},
{
"name": "exports",
"description": "Operations for exporting Weaviate data to Parquet format on external storage backends (S3, GCS, Azure, or filesystem). Exports provide a way to extract your vector data and object properties into a standardized columnar format for data analysis, archival, or migration. Each collection is exported to a separate Parquet file containing object IDs, vectors, properties, and metadata."
},
{
"name": "users",
"description": "Endpoints for user account management in Weaviate. This includes operations specific to Weaviate-managed database users (`db` users), such as creation (which generates an API key), listing, deletion, activation/deactivation, and API key rotation. It also provides operations applicable to any authenticated user (`db` or `oidc`), like retrieving their own information (username and assigned roles).<br/><br/>**User Types:**<br/>* **`db` users:** Managed entirely within Weaviate (creation, deletion, API keys). Use these endpoints for full lifecycle management.<br/>* **`oidc` users:** Authenticated via an external OpenID Connect provider. Their lifecycle (creation, credentials) is managed externally, but their role assignments *within Weaviate* are managed via the `authz` endpoints."
Expand Down
Loading