diff --git a/.changeset/fine-pans-sit.md b/.changeset/fine-pans-sit.md new file mode 100644 index 000000000000..63a5e74a048a --- /dev/null +++ b/.changeset/fine-pans-sit.md @@ -0,0 +1,10 @@ +--- +"wrangler": minor +--- + +Add table-level compaction commands for R2 Data Catalog: + +- `wrangler r2 bucket catalog compaction enable [namespace] [table]` +- `wrangler r2 bucket catalog compaction disable [namespace] [table]` + +This allows you to enable and disable automatic file compaction for a specific R2 data catalog table. diff --git a/packages/wrangler/src/__tests__/r2.test.ts b/packages/wrangler/src/__tests__/r2.test.ts index c5c056090186..265bec23a771 100644 --- a/packages/wrangler/src/__tests__/r2.test.ts +++ b/packages/wrangler/src/__tests__/r2.test.ts @@ -1312,8 +1312,8 @@ describe("r2", () => { Control settings for automatic file compaction maintenance jobs for your R2 data catalog [open-beta] COMMANDS - wrangler r2 bucket catalog compaction enable Enable automatic file compaction for your R2 data catalog [open-beta] - wrangler r2 bucket catalog compaction disable Disable automatic file compaction for your R2 data catalog [open-beta] + wrangler r2 bucket catalog compaction enable [namespace] [table] Enable automatic file compaction for your R2 data catalog or a specific table [open-beta] + wrangler r2 bucket catalog compaction disable [namespace] [table] Disable automatic file compaction for your R2 data catalog or a specific table [open-beta] GLOBAL FLAGS -c, --config Path to Wrangler configuration file [string] @@ -1368,7 +1368,10 @@ describe("r2", () => { " ⛅️ wrangler x.x.x ────────────────── - ✨ Successfully enabled file compaction for the data catalog for bucket 'testBucket'." + ✨ Successfully enabled file compaction for the data catalog for bucket 'testBucket'. + + Compaction will automatically combine small files into larger ones to improve query performance. + For more details, refer to: https://developers.cloudflare.com/r2/data-catalog/about-compaction/" ` ); }); @@ -1381,12 +1384,14 @@ describe("r2", () => { ); expect(std.out).toMatchInlineSnapshot(` " - wrangler r2 bucket catalog compaction enable + wrangler r2 bucket catalog compaction enable [namespace] [table] - Enable automatic file compaction for your R2 data catalog [open-beta] + Enable automatic file compaction for your R2 data catalog or a specific table [open-beta] POSITIONALS - bucket The name of the bucket which contains the catalog [string] [required] + bucket The name of the bucket which contains the catalog [string] [required] + namespace The namespace containing the table (optional, for table-level compaction) [string] + table The name of the table (optional, for table-level compaction) [string] GLOBAL FLAGS -c, --config Path to Wrangler configuration file [string] @@ -1398,7 +1403,7 @@ describe("r2", () => { OPTIONS --target-size The target size for compacted files in MB (allowed values: 64, 128, 256, 512) [number] [default: 128] - --token A cloudflare api token with access to R2 and R2 Data Catalog which will be used to read/write files for compaction. [string] [required]" + --token A cloudflare api token with access to R2 and R2 Data Catalog (required for catalog-level compaction settings only) [string]" `); expect(std.err).toMatchInlineSnapshot(` "X [ERROR] Not enough non-option arguments: got 0, need at least 1 @@ -1407,13 +1412,99 @@ describe("r2", () => { `); }); - it("should error if --token is not provided", async () => { + it("should error if --token is not provided for catalog-level", async () => { await expect( runWrangler( "r2 bucket catalog compaction enable testBucket --target-size 512" ) ).rejects.toThrowErrorMatchingInlineSnapshot( - `[Error: Missing required argument: token]` + `[Error: Token is required for catalog-level compaction. Use --token flag to provide a Cloudflare API token.]` + ); + }); + + it("should enable table compaction without token", async () => { + msw.use( + http.post( + "*/accounts/some-account-id/r2-catalog/testBucket/namespaces/testNamespace/tables/testTable/maintenance-configs", + async ({ request }) => { + const body = await request.json(); + expect(request.method).toEqual("POST"); + expect(body).toEqual({ + compaction: { + state: "enabled", + }, + }); + return HttpResponse.json( + createFetchResult({ success: true }, true) + ); + }, + { once: true } + ) + ); + await runWrangler( + "r2 bucket catalog compaction enable testBucket testNamespace testTable" + ); + expect(std.out).toMatchInlineSnapshot( + ` + " + ⛅️ wrangler x.x.x + ────────────────── + ✨ Successfully enabled file compaction for table 'testNamespace.testTable' in bucket 'testBucket'." + ` + ); + }); + + it("should enable table compaction with custom target size", async () => { + msw.use( + http.post( + "*/accounts/some-account-id/r2-catalog/testBucket/namespaces/testNamespace/tables/testTable/maintenance-configs", + async ({ request }) => { + const body = await request.json(); + expect(request.method).toEqual("POST"); + expect(body).toEqual({ + compaction: { + state: "enabled", + target_size_mb: 256, + }, + }); + return HttpResponse.json( + createFetchResult({ success: true }, true) + ); + }, + { once: true } + ) + ); + await runWrangler( + "r2 bucket catalog compaction enable testBucket testNamespace testTable --target-size 256" + ); + expect(std.out).toMatchInlineSnapshot( + ` + " + ⛅️ wrangler x.x.x + ────────────────── + ✨ Successfully enabled file compaction for table 'testNamespace.testTable' in bucket 'testBucket'." + ` + ); + }); + + it("should error if only namespace is provided", async () => { + await expect( + runWrangler( + "r2 bucket catalog compaction enable testBucket testNamespace" + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Table name is required when namespace is specified]` + ); + }); + + it("should error if only table is provided", async () => { + // This test ensures that if table is passed as namespace position, it errors properly + await expect( + runWrangler( + 'r2 bucket catalog compaction enable testBucket "" testTable' + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Namespace is required when table is specified]` ); }); }); @@ -1429,12 +1520,14 @@ describe("r2", () => { ); expect(std.out).toMatchInlineSnapshot(` " - wrangler r2 bucket catalog compaction disable + wrangler r2 bucket catalog compaction disable [namespace] [table] - Disable automatic file compaction for your R2 data catalog [open-beta] + Disable automatic file compaction for your R2 data catalog or a specific table [open-beta] POSITIONALS - bucket The name of the bucket which contains the catalog [string] [required] + bucket The name of the bucket which contains the catalog [string] [required] + namespace The namespace containing the table (optional, for table-level compaction) [string] + table The name of the table (optional, for table-level compaction) [string] GLOBAL FLAGS -c, --config Path to Wrangler configuration file [string] @@ -1503,6 +1596,60 @@ describe("r2", () => { Disable cancelled." `); }); + + it("should disable table compaction when confirmed", async () => { + setIsTTY(true); + mockConfirm({ + text: "Are you sure you want to disable file compaction for table 'testNamespace.testTable' in bucket 'testBucket'?", + result: true, + }); + msw.use( + http.post( + "*/accounts/some-account-id/r2-catalog/testBucket/namespaces/testNamespace/tables/testTable/maintenance-configs", + async ({ request }) => { + const body = await request.json(); + expect(request.method).toEqual("POST"); + expect(body).toEqual({ + compaction: { + state: "disabled", + }, + }); + return HttpResponse.json( + createFetchResult({ success: true }, true) + ); + }, + { once: true } + ) + ); + await runWrangler( + "r2 bucket catalog compaction disable testBucket testNamespace testTable" + ); + expect(std.out).toMatchInlineSnapshot( + ` + " + ⛅️ wrangler x.x.x + ────────────────── + Successfully disabled file compaction for table 'testNamespace.testTable' in bucket 'testBucket'." + ` + ); + }); + + it("should cancel table compaction disable when rejected", async () => { + setIsTTY(true); + mockConfirm({ + text: "Are you sure you want to disable file compaction for table 'testNamespace.testTable' in bucket 'testBucket'?", + result: false, + }); + await runWrangler( + "r2 bucket catalog compaction disable testBucket testNamespace testTable" + ); + expect(std.out).toMatchInlineSnapshot(` + " + ⛅️ wrangler x.x.x + ────────────────── + Disable cancelled." + `); + }); }); }); }); diff --git a/packages/wrangler/src/r2/catalog.ts b/packages/wrangler/src/r2/catalog.ts index 8a2810ec081b..c0a762284764 100644 --- a/packages/wrangler/src/r2/catalog.ts +++ b/packages/wrangler/src/r2/catalog.ts @@ -1,6 +1,7 @@ import { createCommand, createNamespace } from "../core/create-command"; import { confirm } from "../dialogs"; import { getCloudflareApiEnvironmentFromEnv } from "../environment-variables/misc-variables"; +import { UserError } from "../errors"; import { logger } from "../logger"; import { APIError } from "../parse"; import { requireAuth } from "../user"; @@ -8,8 +9,10 @@ import formatLabelledValues from "../utils/render-labelled-values"; import { disableR2Catalog, disableR2CatalogCompaction, + disableR2CatalogTableCompaction, enableR2Catalog, enableR2CatalogCompaction, + enableR2CatalogTableCompaction, getR2Catalog, upsertR2DataCatalogCredential, } from "./helpers"; @@ -170,17 +173,29 @@ export const r2BucketCatalogCompactionNamespace = createNamespace({ export const r2BucketCatalogCompactionEnableCommand = createCommand({ metadata: { - description: "Enable automatic file compaction for your R2 data catalog", + description: + "Enable automatic file compaction for your R2 data catalog or a specific table", status: "open-beta", owner: "Product: R2 Data Catalog", }, - positionalArgs: ["bucket"], + positionalArgs: ["bucket", "namespace", "table"], args: { bucket: { describe: "The name of the bucket which contains the catalog", type: "string", demandOption: true, }, + namespace: { + describe: + "The namespace containing the table (optional, for table-level compaction)", + type: "string", + demandOption: false, + }, + table: { + describe: "The name of the table (optional, for table-level compaction)", + type: "string", + demandOption: false, + }, "target-size": { describe: "The target size for compacted files in MB (allowed values: 64, 128, 256, 512)", @@ -190,63 +205,141 @@ export const r2BucketCatalogCompactionEnableCommand = createCommand({ }, token: { describe: - "A cloudflare api token with access to R2 and R2 Data Catalog which will be used to read/write files for compaction.", - demandOption: true, + "A cloudflare api token with access to R2 and R2 Data Catalog (required for catalog-level compaction settings only)", + demandOption: false, type: "string", }, }, async handler(args, { config }) { const accountId = await requireAuth(config); - await upsertR2DataCatalogCredential( - config, - accountId, - args.bucket, - args.token - ); + // Validate namespace and table are provided together + if (args.namespace && !args.table) { + throw new UserError("Table name is required when namespace is specified"); + } + if (!args.namespace && args.table) { + throw new UserError("Namespace is required when table is specified"); + } - await enableR2CatalogCompaction( - config, - accountId, - args.bucket, - args.targetSize - ); + if (args.namespace && args.table) { + // Table-level compaction + await enableR2CatalogTableCompaction( + config, + accountId, + args.bucket, + args.namespace, + args.table, + args.targetSize !== 128 ? args.targetSize : undefined + ); - logger.log( - `✨ Successfully enabled file compaction for the data catalog for bucket '${args.bucket}'.` - ); + logger.log( + `✨ Successfully enabled file compaction for table '${args.namespace}.${args.table}' in bucket '${args.bucket}'.` + ); + } else { + // Catalog-level compaction - token is required + if (!args.token) { + throw new UserError( + "Token is required for catalog-level compaction. Use --token flag to provide a Cloudflare API token." + ); + } + + await upsertR2DataCatalogCredential( + config, + accountId, + args.bucket, + args.token + ); + + await enableR2CatalogCompaction( + config, + accountId, + args.bucket, + args.targetSize + ); + + logger.log( + `✨ Successfully enabled file compaction for the data catalog for bucket '${args.bucket}'. + +Compaction will automatically combine small files into larger ones to improve query performance. +For more details, refer to: https://developers.cloudflare.com/r2/data-catalog/about-compaction/` + ); + } }, }); export const r2BucketCatalogCompactionDisableCommand = createCommand({ metadata: { - description: "Disable automatic file compaction for your R2 data catalog", + description: + "Disable automatic file compaction for your R2 data catalog or a specific table", status: "open-beta", owner: "Product: R2 Data Catalog", }, - positionalArgs: ["bucket"], + positionalArgs: ["bucket", "namespace", "table"], args: { bucket: { describe: "The name of the bucket which contains the catalog", type: "string", demandOption: true, }, + namespace: { + describe: + "The namespace containing the table (optional, for table-level compaction)", + type: "string", + demandOption: false, + }, + table: { + describe: "The name of the table (optional, for table-level compaction)", + type: "string", + demandOption: false, + }, }, async handler(args, { config }) { const accountId = await requireAuth(config); - const confirmedDisable = await confirm( - `Are you sure you want to disable file compaction for the data catalog for bucket '${args.bucket}'?` - ); - if (!confirmedDisable) { - logger.log("Disable cancelled."); - return; + // Validate namespace and table are provided together + if (args.namespace && !args.table) { + throw new UserError("Table name is required when namespace is specified"); + } + if (!args.namespace && args.table) { + throw new UserError("Namespace is required when table is specified"); } - await disableR2CatalogCompaction(config, accountId, args.bucket); + if (args.namespace && args.table) { + // Table-level compaction + const confirmedDisable = await confirm( + `Are you sure you want to disable file compaction for table '${args.namespace}.${args.table}' in bucket '${args.bucket}'?` + ); + if (!confirmedDisable) { + logger.log("Disable cancelled."); + return; + } - logger.log( - `Successfully disabled file compaction for the data catalog for bucket '${args.bucket}'.` - ); + await disableR2CatalogTableCompaction( + config, + accountId, + args.bucket, + args.namespace, + args.table + ); + + logger.log( + `Successfully disabled file compaction for table '${args.namespace}.${args.table}' in bucket '${args.bucket}'.` + ); + } else { + // Catalog-level compaction + const confirmedDisable = await confirm( + `Are you sure you want to disable file compaction for the data catalog for bucket '${args.bucket}'?` + ); + if (!confirmedDisable) { + logger.log("Disable cancelled."); + return; + } + + await disableR2CatalogCompaction(config, accountId, args.bucket); + + logger.log( + `Successfully disabled file compaction for the data catalog for bucket '${args.bucket}'.` + ); + } }, }); diff --git a/packages/wrangler/src/r2/helpers.ts b/packages/wrangler/src/r2/helpers.ts index 6b50c6a4ff3b..8770180d6c14 100644 --- a/packages/wrangler/src/r2/helpers.ts +++ b/packages/wrangler/src/r2/helpers.ts @@ -684,6 +684,72 @@ export async function upsertR2DataCatalogCredential( ); } +/** + * Enable compaction for a specific table in the R2 catalog + */ +export async function enableR2CatalogTableCompaction( + complianceConfig: ComplianceConfig, + accountId: string, + bucketName: string, + namespace: string, + tableName: string, + targetSizeMb?: number +): Promise { + const body: { + compaction: { + state: string; + target_size_mb?: number; + }; + } = { + compaction: { + state: "enabled", + }, + }; + + if (targetSizeMb !== undefined) { + body.compaction.target_size_mb = targetSizeMb; + } + + return await fetchResult( + complianceConfig, + `/accounts/${accountId}/r2-catalog/${bucketName}/namespaces/${namespace}/tables/${tableName}/maintenance-configs`, + { + method: "POST", + body: JSON.stringify(body), + headers: { + "Content-Type": "application/json", + }, + } + ); +} + +/** + * Disable compaction for a specific table in the R2 catalog + */ +export async function disableR2CatalogTableCompaction( + complianceConfig: ComplianceConfig, + accountId: string, + bucketName: string, + namespace: string, + tableName: string +): Promise { + return await fetchResult( + complianceConfig, + `/accounts/${accountId}/r2-catalog/${bucketName}/namespaces/${namespace}/tables/${tableName}/maintenance-configs`, + { + method: "POST", + body: JSON.stringify({ + compaction: { + state: "disabled", + }, + }), + headers: { + "Content-Type": "application/json", + }, + } + ); +} + export type R2EventableOperation = | "PutObject" | "DeleteObject"