Skip to content

Commit ffa2600

Browse files
nagrahamnagraham
authored
[R2 Data Catalog] Add wrangler commands for the R2 Data Catalog compaction feature (#10479)
This change adds two new Wrangler sub-commands within R2 Data Catalog to enable/disable automatic compaction for an Iceberg Table in their catalog. Compaction optimizes the size of data files to improve read performance.
1 parent e2b838f commit ffa2600

File tree

5 files changed

+399
-1
lines changed

5 files changed

+399
-1
lines changed

‎.changeset/flat-mice-prove.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"wrangler": minor
3+
---
4+
5+
feat: Add wrangler commands for the R2 Data Catalog compaction feature

‎packages/wrangler/src/__tests__/r2.test.ts‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -969,6 +969,7 @@ describe("r2", () => {
969969
wrangler r2 bucket catalog enable <bucket> Enable the data catalog on an R2 bucket [open-beta]
970970
wrangler r2 bucket catalog disable <bucket> Disable the data catalog for an R2 bucket [open-beta]
971971
wrangler r2 bucket catalog get <bucket> Get the status of the data catalog for an R2 bucket [open-beta]
972+
wrangler r2 bucket catalog compaction Manage compaction maintenance for tables in your R2 data catalog [private-beta]
972973
973974
GLOBAL FLAGS
974975
-c, --config Path to Wrangler configuration file [string]
@@ -1231,6 +1232,220 @@ For more details, refer to: https://developers.cloudflare.com/r2/api/s3/tokens/"
12311232
`);
12321233
});
12331234
});
1235+
1236+
describe("compaction", () => {
1237+
it("should show the correct help when an invalid command is passed", async () => {
1238+
await expect(() =>
1239+
runWrangler("r2 bucket catalog compaction foo")
1240+
).rejects.toThrowErrorMatchingInlineSnapshot(
1241+
`[Error: Unknown argument: foo]`
1242+
);
1243+
expect(std.err).toMatchInlineSnapshot(`
1244+
"X [ERROR] Unknown argument: foo
1245+
1246+
"
1247+
`);
1248+
expect(std.out).toMatchInlineSnapshot(`
1249+
"
1250+
wrangler r2 bucket catalog compaction
1251+
1252+
Manage compaction maintenance for tables in your R2 data catalog [private-beta]
1253+
1254+
COMMANDS
1255+
wrangler r2 bucket catalog compaction enable <bucket> Enable compaction maintenance for a table in the R2 data catalog [private-beta]
1256+
wrangler r2 bucket catalog compaction disable <bucket> Disable compaction maintenance for a table in the R2 data catalog [private-beta]
1257+
1258+
GLOBAL FLAGS
1259+
-c, --config Path to Wrangler configuration file [string]
1260+
--cwd Run as if Wrangler was started in the specified directory instead of the current working directory [string]
1261+
-e, --env Environment to use for operations, and for selecting .env and .dev.vars files [string]
1262+
--env-file Path to an .env file to load - can be specified multiple times - values from earlier files are overridden by values in later files [array]
1263+
-h, --help Show help [boolean]
1264+
-v, --version Show version number [boolean]"
1265+
`);
1266+
});
1267+
1268+
describe("enable", () => {
1269+
it("should enable compaction for the given table", async () => {
1270+
msw.use(
1271+
http.post(
1272+
"*/accounts/some-account-id/r2-catalog/testBucket/namespaces/testNamespace/tables/testTable/maintenance-configs",
1273+
async ({ request }) => {
1274+
const body = await request.json();
1275+
expect(body).toEqual({
1276+
configuration_type: "compaction",
1277+
configuration: {},
1278+
state: "enabled",
1279+
});
1280+
return HttpResponse.json(
1281+
createFetchResult({ success: true }, true)
1282+
);
1283+
},
1284+
{ once: true }
1285+
)
1286+
);
1287+
await runWrangler(
1288+
"r2 bucket catalog compaction enable testBucket --table testTable --namespace testNamespace"
1289+
);
1290+
expect(std.out).toMatchInlineSnapshot(
1291+
`"✨ Successfully enabled compaction maintenance for table 'testTable' in namespace 'testNamespace' of bucket 'testBucket'."`
1292+
);
1293+
});
1294+
1295+
it("should error if no bucket name is given", async () => {
1296+
await expect(
1297+
runWrangler("r2 bucket catalog compaction enable")
1298+
).rejects.toThrowErrorMatchingInlineSnapshot(
1299+
`[Error: Not enough non-option arguments: got 0, need at least 1]`
1300+
);
1301+
expect(std.out).toMatchInlineSnapshot(`
1302+
"
1303+
wrangler r2 bucket catalog compaction enable <bucket>
1304+
1305+
Enable compaction maintenance for a table in the R2 data catalog [private-beta]
1306+
1307+
POSITIONALS
1308+
bucket The name of the bucket [string] [required]
1309+
1310+
GLOBAL FLAGS
1311+
-c, --config Path to Wrangler configuration file [string]
1312+
--cwd Run as if Wrangler was started in the specified directory instead of the current working directory [string]
1313+
-e, --env Environment to use for operations, and for selecting .env and .dev.vars files [string]
1314+
--env-file Path to an .env file to load - can be specified multiple times - values from earlier files are overridden by values in later files [array]
1315+
-h, --help Show help [boolean]
1316+
-v, --version Show version number [boolean]
1317+
1318+
OPTIONS
1319+
--table The name of the table to enable compaction for [string] [required]
1320+
--namespace The namespace containing the table [string] [required]"
1321+
`);
1322+
expect(std.err).toMatchInlineSnapshot(`
1323+
"X [ERROR] Not enough non-option arguments: got 0, need at least 1
1324+
1325+
"
1326+
`);
1327+
});
1328+
1329+
it("should error if --table is not provided", async () => {
1330+
await expect(
1331+
runWrangler(
1332+
"r2 bucket catalog compaction enable testBucket --namespace testNamespace"
1333+
)
1334+
).rejects.toThrowErrorMatchingInlineSnapshot(
1335+
`[Error: Missing required argument: table]`
1336+
);
1337+
});
1338+
1339+
it("should error if --namespace is not provided", async () => {
1340+
await expect(
1341+
runWrangler(
1342+
"r2 bucket catalog compaction enable testBucket --table testTable"
1343+
)
1344+
).rejects.toThrowErrorMatchingInlineSnapshot(
1345+
`[Error: Missing required argument: namespace]`
1346+
);
1347+
});
1348+
});
1349+
1350+
describe("disable", () => {
1351+
const { setIsTTY } = useMockIsTTY();
1352+
1353+
it("should error if no bucket name is given", async () => {
1354+
await expect(
1355+
runWrangler("r2 bucket catalog compaction disable")
1356+
).rejects.toThrowErrorMatchingInlineSnapshot(
1357+
`[Error: Not enough non-option arguments: got 0, need at least 1]`
1358+
);
1359+
expect(std.out).toMatchInlineSnapshot(`
1360+
"
1361+
wrangler r2 bucket catalog compaction disable <bucket>
1362+
1363+
Disable compaction maintenance for a table in the R2 data catalog [private-beta]
1364+
1365+
POSITIONALS
1366+
bucket The name of the bucket [string] [required]
1367+
1368+
GLOBAL FLAGS
1369+
-c, --config Path to Wrangler configuration file [string]
1370+
--cwd Run as if Wrangler was started in the specified directory instead of the current working directory [string]
1371+
-e, --env Environment to use for operations, and for selecting .env and .dev.vars files [string]
1372+
--env-file Path to an .env file to load - can be specified multiple times - values from earlier files are overridden by values in later files [array]
1373+
-h, --help Show help [boolean]
1374+
-v, --version Show version number [boolean]
1375+
1376+
OPTIONS
1377+
--table The name of the table to disable compaction for [string] [required]
1378+
--namespace The namespace containing the table [string] [required]"
1379+
`);
1380+
expect(std.err).toMatchInlineSnapshot(`
1381+
"X [ERROR] Not enough non-option arguments: got 0, need at least 1
1382+
1383+
"
1384+
`);
1385+
});
1386+
1387+
it("should disable compaction for the given table with confirmation", async () => {
1388+
setIsTTY(true);
1389+
mockConfirm({
1390+
text: "Are you sure you want to disable compaction maintenance for table 'testTable' in namespace 'testNamespace' of bucket 'testBucket'?",
1391+
result: true,
1392+
});
1393+
msw.use(
1394+
http.put(
1395+
"*/accounts/some-account-id/r2-catalog/testBucket/namespaces/testNamespace/tables/testTable/maintenance-configs/compaction",
1396+
async ({ request }) => {
1397+
const body = await request.json();
1398+
expect(body).toEqual({
1399+
state: "disabled",
1400+
});
1401+
return HttpResponse.json(
1402+
createFetchResult({ success: true }, true)
1403+
);
1404+
},
1405+
{ once: true }
1406+
)
1407+
);
1408+
await runWrangler(
1409+
"r2 bucket catalog compaction disable testBucket --table testTable --namespace testNamespace"
1410+
);
1411+
expect(std.out).toMatchInlineSnapshot(
1412+
`"Successfully disabled compaction maintenance for table 'testTable' in namespace 'testNamespace' of bucket 'testBucket'."`
1413+
);
1414+
});
1415+
1416+
it("should cancel disable when confirmation is rejected", async () => {
1417+
setIsTTY(true);
1418+
mockConfirm({
1419+
text: "Are you sure you want to disable compaction maintenance for table 'testTable' in namespace 'testNamespace' of bucket 'testBucket'?",
1420+
result: false,
1421+
});
1422+
await runWrangler(
1423+
"r2 bucket catalog compaction disable testBucket --table testTable --namespace testNamespace"
1424+
);
1425+
expect(std.out).toMatchInlineSnapshot(`"Disable cancelled."`);
1426+
});
1427+
1428+
it("should error if --table is not provided", async () => {
1429+
await expect(
1430+
runWrangler(
1431+
"r2 bucket catalog compaction disable testBucket --namespace testNamespace"
1432+
)
1433+
).rejects.toThrowErrorMatchingInlineSnapshot(
1434+
`[Error: Missing required argument: table]`
1435+
);
1436+
});
1437+
1438+
it("should error if --namespace is not provided", async () => {
1439+
await expect(
1440+
runWrangler(
1441+
"r2 bucket catalog compaction disable testBucket --table testTable"
1442+
)
1443+
).rejects.toThrowErrorMatchingInlineSnapshot(
1444+
`[Error: Missing required argument: namespace]`
1445+
);
1446+
});
1447+
});
1448+
});
12341449
});
12351450

12361451
describe("notification", () => {

‎packages/wrangler/src/index.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,9 @@ import {
190190
r2BucketUpdateStorageClassCommand,
191191
} from "./r2/bucket";
192192
import {
193+
r2BucketCatalogCompactionDisableCommand,
194+
r2BucketCatalogCompactionEnableCommand,
195+
r2BucketCatalogCompactionNamespace,
193196
r2BucketCatalogDisableCommand,
194197
r2BucketCatalogEnableCommand,
195198
r2BucketCatalogGetCommand,
@@ -842,6 +845,18 @@ export function createCLIParser(argv: string[]) {
842845
command: "wrangler r2 bucket catalog get",
843846
definition: r2BucketCatalogGetCommand,
844847
},
848+
{
849+
command: "wrangler r2 bucket catalog compaction",
850+
definition: r2BucketCatalogCompactionNamespace,
851+
},
852+
{
853+
command: "wrangler r2 bucket catalog compaction enable",
854+
definition: r2BucketCatalogCompactionEnableCommand,
855+
},
856+
{
857+
command: "wrangler r2 bucket catalog compaction disable",
858+
definition: r2BucketCatalogCompactionDisableCommand,
859+
},
845860
{
846861
command: "wrangler r2 bucket notification",
847862
definition: r2BucketNotificationNamespace,

‎packages/wrangler/src/r2/catalog.ts‎

Lines changed: 108 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,13 @@ import { logger } from "../logger";
55
import { APIError } from "../parse";
66
import { requireAuth } from "../user";
77
import formatLabelledValues from "../utils/render-labelled-values";
8-
import { disableR2Catalog, enableR2Catalog, getR2Catalog } from "./helpers";
8+
import {
9+
disableR2Catalog,
10+
disableR2CatalogCompaction,
11+
enableR2Catalog,
12+
enableR2CatalogCompaction,
13+
getR2Catalog,
14+
} from "./helpers";
915

1016
export const r2BucketCatalogNamespace = createNamespace({
1117
metadata: {
@@ -152,3 +158,104 @@ export const r2BucketCatalogGetCommand = createCommand({
152158
}
153159
},
154160
});
161+
162+
export const r2BucketCatalogCompactionNamespace = createNamespace({
163+
metadata: {
164+
description:
165+
"Manage compaction maintenance for tables in your R2 data catalog",
166+
status: "private-beta",
167+
owner: "Product: R2 Data Catalog",
168+
},
169+
});
170+
171+
export const r2BucketCatalogCompactionEnableCommand = createCommand({
172+
metadata: {
173+
description:
174+
"Enable compaction maintenance for a table in the R2 data catalog",
175+
status: "private-beta",
176+
owner: "Product: R2 Data Catalog",
177+
},
178+
positionalArgs: ["bucket"],
179+
args: {
180+
bucket: {
181+
describe: "The name of the bucket",
182+
type: "string",
183+
demandOption: true,
184+
},
185+
table: {
186+
describe: "The name of the table to enable compaction for",
187+
type: "string",
188+
demandOption: true,
189+
},
190+
namespace: {
191+
describe: "The namespace containing the table",
192+
type: "string",
193+
demandOption: true,
194+
},
195+
},
196+
async handler(args, { config }) {
197+
const accountId = await requireAuth(config);
198+
199+
await enableR2CatalogCompaction(
200+
config,
201+
accountId,
202+
args.bucket,
203+
args.namespace,
204+
args.table
205+
);
206+
207+
logger.log(
208+
`✨ Successfully enabled compaction maintenance for table '${args.table}' in namespace '${args.namespace}' of bucket '${args.bucket}'.`
209+
);
210+
},
211+
});
212+
213+
export const r2BucketCatalogCompactionDisableCommand = createCommand({
214+
metadata: {
215+
description:
216+
"Disable compaction maintenance for a table in the R2 data catalog",
217+
status: "private-beta",
218+
owner: "Product: R2 Data Catalog",
219+
},
220+
positionalArgs: ["bucket"],
221+
args: {
222+
bucket: {
223+
describe: "The name of the bucket",
224+
type: "string",
225+
demandOption: true,
226+
},
227+
table: {
228+
describe: "The name of the table to disable compaction for",
229+
type: "string",
230+
demandOption: true,
231+
},
232+
namespace: {
233+
describe: "The namespace containing the table",
234+
type: "string",
235+
demandOption: true,
236+
},
237+
},
238+
async handler(args, { config }) {
239+
const accountId = await requireAuth(config);
240+
241+
const confirmedDisable = await confirm(
242+
`Are you sure you want to disable compaction maintenance for table '${args.table}' in namespace '${args.namespace}' of bucket '${args.bucket}'?`
243+
);
244+
if (!confirmedDisable) {
245+
logger.log("Disable cancelled.");
246+
return;
247+
}
248+
249+
await disableR2CatalogCompaction(
250+
config,
251+
accountId,
252+
args.bucket,
253+
args.namespace,
254+
args.table
255+
);
256+
257+
logger.log(
258+
`Successfully disabled compaction maintenance for table '${args.table}' in namespace '${args.namespace}' of bucket '${args.bucket}'.`
259+
);
260+
},
261+
});

0 commit comments

Comments
 (0)