diff --git a/.changeset/fifty-foxes-win.md b/.changeset/fifty-foxes-win.md new file mode 100644 index 000000000000..5007a849bf1a --- /dev/null +++ b/.changeset/fifty-foxes-win.md @@ -0,0 +1,5 @@ +--- +"wrangler": minor +--- + +Added new pipelines commands (pipelines, streams, sinks, setup), moved old pipelines commands behind --legacy diff --git a/packages/wrangler/src/__tests__/pipelines.test.ts b/packages/wrangler/src/__tests__/pipelines.test.ts index 4bb4bb953c61..faffcadd20fd 100644 --- a/packages/wrangler/src/__tests__/pipelines.test.ts +++ b/packages/wrangler/src/__tests__/pipelines.test.ts @@ -1,82 +1,51 @@ +import { writeFileSync } from "node:fs"; import { http, HttpResponse } from "msw"; import { describe, expect, it } from "vitest"; -import { normalizeOutput } from "../../e2e/helpers/normalize"; -import { __testSkipDelays } from "../pipelines"; -import { endEventLoop } from "./helpers/end-event-loop"; import { mockAccountId, mockApiToken } from "./helpers/mock-account-id"; import { mockConsoleMethods } from "./helpers/mock-console"; +import { mockConfirm } from "./helpers/mock-dialogs"; +import { useMockIsTTY } from "./helpers/mock-istty"; import { msw } from "./helpers/msw"; import { runInTempDir } from "./helpers/run-in-tmp"; import { runWrangler } from "./helpers/run-wrangler"; -import type { HttpSource, Pipeline, PipelineEntry } from "../pipelines/client"; +import type { Pipeline, SchemaField, Sink, Stream } from "../pipelines/types"; -describe("pipelines", () => { +describe("wrangler pipelines", () => { const std = mockConsoleMethods(); mockAccountId(); mockApiToken(); runInTempDir(); - const samplePipeline = { - id: "0001", - version: 1, - name: "my-pipeline", - metadata: {}, - source: [ - { - type: "binding", - format: "json", - }, - { - type: "http", - format: "json", - authentication: false, - cors: { - origins: ["*"], - }, - }, - ], - transforms: [], - destination: { - type: "r2", - format: "json", - batch: { - max_bytes: 100000000, - max_duration_s: 300, - max_rows: 100000, - }, - compression: { - type: "none", - }, - path: { - bucket: "bucket", - }, - }, - endpoint: "https://0001.pipelines.cloudflarestorage.com", - } satisfies Pipeline; - - function mockCreateR2TokenFailure(bucket: string) { + const accountId = "some-account-id"; + + function mockValidateSqlRequest(sql: string, isValid = true) { const requests = { count: 0 }; msw.use( - http.get( - "*/accounts/:accountId/r2/buckets/:bucket", - async ({ params }) => { - expect(params.accountId).toEqual("some-account-id"); - expect(params.bucket).toEqual(bucket); + http.post( + `*/accounts/${accountId}/pipelines/v1/validate_sql`, + async ({ request }) => { requests.count++; - return HttpResponse.json( - { - success: false, - errors: [ - { - code: 10006, - message: "The specified bucket does not exist.", - }, - ], - messages: [], - result: null, + const body = (await request.json()) as { sql: string }; + expect(body.sql).toBe(sql); + + if (!isValid) { + const error = { + notes: [{ text: "Invalid SQL syntax near 'INVALID'" }], + }; + throw error; + } + + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: { + tables: { + test_stream: { type: "stream" }, + test_sink: { type: "sink" }, + }, }, - { status: 404 } - ); + }); }, { once: true } ) @@ -84,51 +53,50 @@ describe("pipelines", () => { return requests; } - function mockCreateRequest( - name: string, - status: number = 200, - error?: object - ) { - const requests: { count: number; body: Pipeline | null } = { - count: 0, - body: null, - }; + function mockCreatePipelineRequest(expectedRequest: { + name: string; + sql: string; + }) { + const requests = { count: 0 }; msw.use( http.post( - "*/accounts/:accountId/pipelines", - async ({ request, params }) => { - expect(params.accountId).toEqual("some-account-id"); - const config = (await request.json()) as Pipeline; - expect(config.name).toEqual(name); - requests.body = config; + `*/accounts/${accountId}/pipelines/v1/pipelines`, + async ({ request }) => { requests.count++; - const pipeline: Pipeline = { - ...config, - id: "0001", - name: name, - endpoint: "foo", - }; - - // API will set defaults if not provided - if (!pipeline.destination.batch.max_rows) { - pipeline.destination.batch.max_rows = 10_000_000; - } - if (!pipeline.destination.batch.max_bytes) { - pipeline.destination.batch.max_bytes = 100_000_000; - } - if (!pipeline.destination.batch.max_duration_s) { - pipeline.destination.batch.max_duration_s = 300; - } - - return HttpResponse.json( - { - success: !error, - errors: error ? [error] : [], - messages: [], - result: pipeline, + const body = (await request.json()) as { name: string; sql: string }; + expect(body.name).toBe(expectedRequest.name); + expect(body.sql).toBe(expectedRequest.sql); + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: { + id: "pipeline_123", + name: expectedRequest.name, + sql: expectedRequest.sql, + status: "active", + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + tables: [ + { + id: "stream_456", + name: "test_stream", + type: "stream", + version: 1, + latest: 1, + href: "/accounts/some-account-id/pipelines/v1/streams/stream_456", + }, + { + id: "sink_789", + name: "test_sink", + type: "sink", + version: 1, + latest: 1, + href: "/accounts/some-account-id/pipelines/v1/sinks/sink_789", + }, + ], }, - { status } - ); + }); }, { once: true } ) @@ -136,24 +104,19 @@ describe("pipelines", () => { return requests; } - function mockListRequest(entries: PipelineEntry[]) { + function mockGetPipelineRequest(pipelineId: string, pipeline: Pipeline) { const requests = { count: 0 }; msw.use( http.get( - "*/accounts/:accountId/pipelines", - async ({ params }) => { + `*/accounts/${accountId}/pipelines/v1/pipelines/${pipelineId}`, + () => { requests.count++; - expect(params.accountId).toEqual("some-account-id"); - - return HttpResponse.json( - { - success: true, - errors: [], - messages: [], - result: entries, - }, - { status: 200 } - ); + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: pipeline, + }); }, { once: true } ) @@ -161,30 +124,19 @@ describe("pipelines", () => { return requests; } - function mockGetRequest( - name: string, - pipeline: Pipeline | null, - status: number = 200, - error?: object - ) { + function mockGetStreamRequest(streamId: string, stream: Stream) { const requests = { count: 0 }; msw.use( http.get( - "*/accounts/:accountId/pipelines/:name", - async ({ params }) => { + `*/accounts/${accountId}/pipelines/v1/streams/${streamId}`, + () => { requests.count++; - expect(params.accountId).toEqual("some-account-id"); - expect(params.name).toEqual(name); - - return HttpResponse.json( - { - success: !error, - errors: error ? [error] : [], - messages: [], - result: pipeline, - }, - { status } - ); + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: stream, + }); }, { once: true } ) @@ -192,39 +144,29 @@ describe("pipelines", () => { return requests; } - function mockUpdateRequest( - name: string, - pipeline: Pipeline | null, - status: number = 200, - error?: object - ) { - const requests: { count: number; body: Pipeline | null } = { - count: 0, - body: null, - }; + function mockListPipelinesRequest(pipelines: Pipeline[]) { + const requests = { count: 0 }; msw.use( - http.put( - "*/accounts/:accountId/pipelines/:name", - async ({ params, request }) => { + http.get( + `*/accounts/${accountId}/pipelines/v1/pipelines`, + ({ request }) => { requests.count++; - requests.body = (await request.json()) as Pipeline; - expect(params.accountId).toEqual("some-account-id"); - expect(params.name).toEqual(name); - - // update strips creds, so enforce this - if (pipeline?.destination) { - pipeline.destination.credentials = undefined; - } - - return HttpResponse.json( - { - success: !error, - errors: error ? [error] : [], - messages: [], - result: pipeline, + const url = new URL(request.url); + const page = Number(url.searchParams.get("page") || 1); + const perPage = Number(url.searchParams.get("per_page") || 20); + + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: pipelines, + result_info: { + page, + per_page: perPage, + count: pipelines.length, + total_count: pipelines.length, }, - { status } - ); + }); }, { once: true } ) @@ -232,29 +174,19 @@ describe("pipelines", () => { return requests; } - function mockDeleteRequest( - name: string, - status: number = 200, - error?: object - ) { + function mockDeletePipelineRequest(pipelineId: string) { const requests = { count: 0 }; msw.use( http.delete( - "*/accounts/:accountId/pipelines/:name", - async ({ params }) => { + `*/accounts/${accountId}/pipelines/v1/pipelines/${pipelineId}`, + () => { requests.count++; - expect(params.accountId).toEqual("some-account-id"); - expect(params.name).toEqual(name); - - return HttpResponse.json( - { - success: !error, - errors: error ? [error] : [], - messages: [], - result: null, - }, - { status } - ); + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: null, + }); }, { once: true } ) @@ -262,510 +194,1435 @@ describe("pipelines", () => { return requests; } - beforeAll(() => { - __testSkipDelays(); - }); + describe("pipelines create", () => { + it("should error when neither --sql nor --sql-file is provided", async () => { + await expect( + runWrangler("pipelines create my_pipeline") + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Either --sql or --sql-file must be provided]` + ); + }); + + it("should create pipeline with inline SQL", async () => { + const sql = "INSERT INTO test_sink SELECT * FROM test_stream;"; + const validateRequest = mockValidateSqlRequest(sql); + const createRequest = mockCreatePipelineRequest({ + name: "my_pipeline", + sql, + }); + const getPipelineRequest = mockGetPipelineRequest("pipeline_123", { + id: "pipeline_123", + name: "my_pipeline", + sql, + status: "active", + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + tables: [ + { + id: "stream_456", + name: "test_stream", + type: "stream", + version: 1, + latest: 1, + href: "/accounts/some-account-id/pipelines/v1/streams/stream_456", + }, + { + id: "sink_789", + name: "test_sink", + type: "sink", + version: 1, + latest: 1, + href: "/accounts/some-account-id/pipelines/v1/sinks/sink_789", + }, + ], + }); + const getStreamRequest = mockGetStreamRequest("stream_456", { + id: "stream_456", + name: "test_stream", + version: 1, + endpoint: "https://pipelines.cloudflare.com/stream_456", + format: { type: "json", unstructured: true }, + schema: null, + http: { + enabled: true, + authentication: false, + }, + worker_binding: { enabled: true }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }); + + await runWrangler(`pipelines create my_pipeline --sql "${sql}"`); + + expect(validateRequest.count).toBe(1); + expect(createRequest.count).toBe(1); + expect(getPipelineRequest.count).toBe(1); + expect(getStreamRequest.count).toBe(1); - it("shows usage details", async () => { - await runWrangler("pipelines"); - await endEventLoop(); - - expect(std.err).toMatchInlineSnapshot(`""`); - expect(std.out).toMatchInlineSnapshot(` - "wrangler pipelines - - 🚰 Manage Cloudflare Pipelines [open-beta] - - COMMANDS - wrangler pipelines create Create a new pipeline [open-beta] - wrangler pipelines list List all pipelines [open-beta] - wrangler pipelines get Get a pipeline's configuration [open-beta] - wrangler pipelines update Update a pipeline [open-beta] - wrangler pipelines delete Delete a pipeline [open-beta] - - GLOBAL FLAGS - -c, --config Path to Wrangler configuration file [string] - --cwd Run as if Wrangler was started in the specified directory instead of the current working directory [string] - -e, --env Environment to use for operations, and for selecting .env and .dev.vars files [string] - --env-file Path to an .env file to load - can be specified multiple times - values from earlier files are overridden by values in later files [array] - -h, --help Show help [boolean] - -v, --version Show version number [boolean]" - `); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toContain("🌀 Validating SQL..."); + expect(std.out).toContain( + "✅ SQL validated successfully. References tables: test_stream, test_sink" + ); + expect(std.out).toContain("🌀 Creating pipeline 'my_pipeline'..."); + expect(std.out).toContain( + "✨ Successfully created pipeline 'my_pipeline' with id 'pipeline_123'." + ); + expect(std.out).toContain( + "Send your first event to stream 'test_stream':" + ); + expect(std.out).toContain("Worker Integration:"); + expect(std.out).toContain("HTTP Endpoint:"); + }); + + it("should create pipeline from SQL file", async () => { + const sql = "INSERT INTO test_sink SELECT * FROM test_stream;"; + const sqlFile = "pipeline.sql"; + writeFileSync(sqlFile, sql); + + const validateRequest = mockValidateSqlRequest(sql); + const createRequest = mockCreatePipelineRequest({ + name: "my_pipeline", + sql, + }); + const getPipelineRequest = mockGetPipelineRequest("pipeline_123", { + id: "pipeline_123", + name: "my_pipeline", + sql, + status: "active", + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + tables: [ + { + id: "stream_456", + name: "test_stream", + type: "stream", + version: 1, + latest: 1, + href: "/accounts/some-account-id/pipelines/v1/streams/stream_456", + }, + { + id: "sink_789", + name: "test_sink", + type: "sink", + version: 1, + latest: 1, + href: "/accounts/some-account-id/pipelines/v1/sinks/sink_789", + }, + ], + }); + const getStreamRequest = mockGetStreamRequest("stream_456", { + id: "stream_456", + name: "test_stream", + version: 1, + endpoint: "https://pipelines.cloudflare.com/stream_456", + format: { type: "json", unstructured: true }, + schema: null, + http: { + enabled: true, + authentication: false, + }, + worker_binding: { enabled: true }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }); + + await runWrangler(`pipelines create my_pipeline --sql-file ${sqlFile}`); + + expect(validateRequest.count).toBe(1); + expect(createRequest.count).toBe(1); + expect(getPipelineRequest.count).toBe(1); + expect(getStreamRequest.count).toBe(1); + + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toContain("🌀 Validating SQL..."); + expect(std.out).toContain("✅ SQL validated successfully."); + expect(std.out).toContain( + "✨ Successfully created pipeline 'my_pipeline' with id 'pipeline_123'." + ); + }); + + it("should error when SQL validation fails", async () => { + const sql = "INVALID SQL QUERY"; + const validateRequest = mockValidateSqlRequest(sql, false); + + await expect( + runWrangler(`pipelines create my_pipeline --sql "${sql}"`) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: SQL validation failed: Invalid SQL syntax near 'INVALID']` + ); + + expect(validateRequest.count).toBe(1); + }); + + it("should show wrangler version message on authentication error", async () => { + const sql = "INSERT INTO test_sink SELECT * FROM test_stream;"; + const validateRequest = mockValidateSqlRequest(sql); + + // Mock create returns auth error (code 10000) + msw.use( + http.post( + `*/accounts/${accountId}/pipelines/v1/pipelines`, + () => { + return HttpResponse.json( + { + success: false, + errors: [{ code: 10000, message: "Authentication error" }], + messages: [], + result: null, + }, + { status: 403 } + ); + }, + { once: true } + ) + ); + + await expect( + runWrangler(`pipelines create my_pipeline --sql "${sql}"`) + ).rejects.toThrowErrorMatchingInlineSnapshot( + ` + [Error: Your account does not have access to the new Pipelines API. To use the legacy Pipelines API, please run: + + npx wrangler@4.36.0 pipelines create my_pipeline + + This will use an older version of Wrangler that supports the legacy API.] + ` + ); + + expect(validateRequest.count).toBe(1); + }); }); - describe("create", () => { - it("should show usage details", async () => { - await runWrangler("pipelines create -h"); - await endEventLoop(); + describe("pipelines list", () => { + it("should list pipelines", async () => { + const mockPipelines: Pipeline[] = [ + { + id: "pipeline_1", + name: "pipeline_one", + sql: "INSERT INTO sink1 SELECT * FROM stream1;", + status: "active", + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }, + { + id: "pipeline_2", + name: "pipeline_two", + sql: "INSERT INTO sink2 SELECT * FROM stream2;", + status: "active", + created_at: "2024-01-02T00:00:00Z", + modified_at: "2024-01-02T00:00:00Z", + }, + ]; + const listRequest = mockListPipelinesRequest(mockPipelines); + + await runWrangler("pipelines list"); + + expect(listRequest.count).toBe(1); expect(std.err).toMatchInlineSnapshot(`""`); expect(std.out).toMatchInlineSnapshot(` - "wrangler pipelines create - - Create a new pipeline [open-beta] - - Source settings - --source Space separated list of allowed sources. Options are 'http' or 'worker' [array] [default: [\\"http\\",\\"worker\\"]] - --require-http-auth Require Cloudflare API Token for HTTPS endpoint authentication [boolean] [default: false] - --cors-origins CORS origin allowlist for HTTP endpoint (use * for any origin). Defaults to an empty array [array] - - Batch hints - --batch-max-mb Maximum batch size in megabytes before flushing. Defaults to 100 MB if unset. Minimum: 1, Maximum: 100 [number] - --batch-max-rows Maximum number of rows per batch before flushing. Defaults to 10,000,000 if unset. Minimum: 100, Maximum: 10,000,000 [number] - --batch-max-seconds Maximum age of batch in seconds before flushing. Defaults to 300 if unset. Minimum: 1, Maximum: 300 [number] - - Destination settings - --r2-bucket Destination R2 bucket name [string] [required] - --r2-access-key-id R2 service Access Key ID for authentication. Leave empty for OAuth confirmation. [string] - --r2-secret-access-key R2 service Secret Access Key for authentication. Leave empty for OAuth confirmation. [string] - --r2-prefix Prefix for storing files in the destination bucket. Default is no prefix [string] [default: \\"\\"] - --compression Compression format for output files [string] [choices: \\"none\\", \\"gzip\\", \\"deflate\\"] [default: \\"gzip\\"] - - Pipeline settings - --shard-count Number of shards for the pipeline. More shards handle higher request volume; fewer shards produce larger output files. Defaults to 2 if unset. Minimum: 1, Maximum: 15 [number] - - POSITIONALS - pipeline The name of the new pipeline [string] [required] - - GLOBAL FLAGS - -c, --config Path to Wrangler configuration file [string] - --cwd Run as if Wrangler was started in the specified directory instead of the current working directory [string] - -e, --env Environment to use for operations, and for selecting .env and .dev.vars files [string] - --env-file Path to an .env file to load - can be specified multiple times - values from earlier files are overridden by values in later files [array] - -h, --help Show help [boolean] - -v, --version Show version number [boolean]" + "┌─┬─┬─┬─┐ + │ Name │ ID │ Created │ Modified │ + ├─┼─┼─┼─┤ + │ pipeline_one │ pipeline_1 │ 1/1/2024 │ 1/1/2024 │ + ├─┼─┼─┼─┤ + │ pipeline_two │ pipeline_2 │ 1/2/2024 │ 1/2/2024 │ + └─┴─┴─┴─┘" `); }); - it("should create a pipeline with explicit credentials", async () => { - const requests = mockCreateRequest("my-pipeline"); - await runWrangler( - "pipelines create my-pipeline --r2-bucket test-bucket --r2-access-key-id my-key --r2-secret-access-key my-secret" - ); - expect(requests.count).toEqual(1); - expect(std.out).toMatchInlineSnapshot(` - "🌀 Creating pipeline named \\"my-pipeline\\" - ✅ Successfully created pipeline \\"my-pipeline\\" with ID 0001 + it("should handle empty pipelines list", async () => { + const listRequest = mockListPipelinesRequest([]); - Id: 0001 - Name: my-pipeline - Sources: - HTTP: - Endpoint: foo - Authentication: off - Format: JSON - Worker: - Format: JSON - Destination: - Type: R2 - Bucket: test-bucket - Format: newline-delimited JSON - Compression: GZIP - Batch hints: - Max bytes: 100 MB - Max duration: 300 seconds - Max records: 10,000,000 + await runWrangler("pipelines list"); + + expect(listRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(`"No pipelines found."`); + }); + + it("should merge new and legacy pipelines with Type column for legacy", async () => { + const mockNewPipelines: Pipeline[] = [ + { + id: "pipeline_1", + name: "new_pipeline", + sql: "INSERT INTO sink1 SELECT * FROM stream1;", + status: "active", + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }, + ]; + + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/v1/pipelines`, + () => { + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: mockNewPipelines, + result_info: { + page: 1, + per_page: 20, + count: mockNewPipelines.length, + total_count: mockNewPipelines.length, + }, + }); + }, + { once: true } + ) + ); - 🎉 You can now send data to your pipeline! - To access your new Pipeline in your Worker, add the following snippet to your configuration file: + const mockLegacyPipelines = [ { - \\"pipelines\\": [ - { - \\"pipeline\\": \\"my-pipeline\\", - \\"binding\\": \\"PIPELINE\\" - } - ] - } + id: "legacy_123", + name: "legacy_pipeline", + endpoint: "https://pipelines.cloudflare.com/legacy", + }, + ]; + + msw.use( + http.get( + `*/accounts/${accountId}/pipelines`, + () => { + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: mockLegacyPipelines, + }); + }, + { once: true } + ) + ); - Send data to your pipeline's HTTP endpoint: + await runWrangler("pipelines list"); + + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.warn).toMatchInlineSnapshot(` + "▲ [WARNING] 🚧 \`wrangler pipelines list\` is an open-beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose + + + ▲ [WARNING] ⚠️ You have legacy pipelines. Consider creating new pipelines by running 'wrangler pipelines setup'. - curl \\"foo\\" -d '[{\\"foo\\": \\"bar\\"}]' " `); + expect(std.out).toMatchInlineSnapshot(` + "┌─┬─┬─┬─┬─┐ + │ Name │ ID │ Created │ Modified │ Type │ + ├─┼─┼─┼─┼─┤ + │ new_pipeline │ pipeline_1 │ 1/1/2024 │ 1/1/2024 │ │ + ├─┼─┼─┼─┼─┤ + │ legacy_pipeline │ legacy_123 │ N/A │ N/A │ Legacy │ + └─┴─┴─┴─┴─┘" + `); }); + }); - it("should fail a missing bucket", async () => { - const requests = mockCreateR2TokenFailure("bad-bucket"); + describe("pipelines get", () => { + it("should error when no pipeline ID provided", async () => { await expect( - runWrangler("pipelines create bad-pipeline --r2-bucket bad-bucket") - ).rejects.toThrowError(); + runWrangler("pipelines get") + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Not enough non-option arguments: got 0, need at least 1]` + ); + }); + + it("should get pipeline details", async () => { + const mockPipeline: Pipeline = { + id: "pipeline_123", + name: "my_pipeline", + sql: "INSERT INTO test_sink SELECT * FROM test_stream;", + status: "active", + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + tables: [ + { + id: "stream_456", + name: "test_stream", + type: "stream", + version: 1, + latest: 1, + href: "/accounts/some-account-id/pipelines/v1/streams/stream_456", + }, + { + id: "sink_789", + name: "test_sink", + type: "sink", + version: 1, + latest: 1, + href: "/accounts/some-account-id/pipelines/v1/sinks/sink_789", + }, + ], + }; + + const getRequest = mockGetPipelineRequest("pipeline_123", mockPipeline); - await endEventLoop(); + await runWrangler("pipelines get pipeline_123"); - expect(normalizeOutput(std.err)).toMatchInlineSnapshot(` - "X [ERROR] The R2 bucket [bad-bucket] doesn't exist" + expect(getRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "General: + ID: pipeline_123 + Name: my_pipeline + Created At: 1/1/2024, 12:00:00 AM + Modified At: 1/1/2024, 12:00:00 AM + + Pipeline SQL: + INSERT INTO test_sink SELECT * FROM test_stream; + + Connected Streams: + ┌─┬─┐ + │ Name │ ID │ + ├─┼─┤ + │ test_stream │ stream_456 │ + └─┴─┘ + + Connected Sinks: + ┌─┬─┐ + │ Name │ ID │ + ├─┼─┤ + │ test_sink │ sink_789 │ + └─┴─┘" `); - expect(std.out).toMatchInlineSnapshot(`""`); - expect(requests.count).toEqual(1); }); - it("should create a pipeline with auth", async () => { - const requests = mockCreateRequest("my-pipeline"); - await runWrangler( - "pipelines create my-pipeline --require-http-auth --r2-bucket test-bucket --r2-access-key-id my-key --r2-secret-access-key my-secret" + it("should fall back to legacy API when pipeline not found in new API", async () => { + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/v1/pipelines/my-legacy-pipeline`, + () => { + return HttpResponse.json( + { + success: false, + errors: [{ code: 1000, message: "Pipeline not found" }], + messages: [], + result: null, + }, + { status: 404 } + ); + }, + { once: true } + ) ); - expect(requests.count).toEqual(1); - // contain http source and include auth - expect(requests.body?.source[0].type).toEqual("http"); - expect((requests.body?.source[0] as HttpSource).authentication).toEqual( - true - ); - }); + const mockLegacyPipeline = { + id: "legacy_123", + name: "my-legacy-pipeline", + endpoint: "https://pipelines.cloudflare.com/legacy", + source: [{ type: "http", format: "json" }], + destination: { + type: "r2", + format: "json", + path: { bucket: "my-bucket", prefix: "data/" }, + batch: { + max_duration_s: 300, + max_bytes: 100000000, + max_rows: 10000000, + }, + compression: { type: "gzip" }, + }, + transforms: [], + metadata: { shards: 2 }, + }; - it("should create a pipeline without http", async () => { - const requests = mockCreateRequest("my-pipeline"); - await runWrangler( - "pipelines create my-pipeline --source worker --r2-bucket test-bucket --r2-access-key-id my-key --r2-secret-access-key my-secret" + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/my-legacy-pipeline`, + () => { + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: mockLegacyPipeline, + }); + }, + { once: true } + ) ); - expect(requests.count).toEqual(1); - // only contains binding source - expect(requests.body?.source.length).toEqual(1); - expect(requests.body?.source[0].type).toEqual("binding"); - }); - }); + await runWrangler("pipelines get my-legacy-pipeline"); - it("list - should list pipelines", async () => { - const requests = mockListRequest([ - { - name: "foo", - id: "0001", - endpoint: "https://0001.pipelines.cloudflarestorage.com", - }, - ]); - await runWrangler("pipelines list"); - - expect(std.err).toMatchInlineSnapshot(`""`); - expect(std.out).toMatchInlineSnapshot(` - "┌─┬─┬─┐ - │ name │ id │ endpoint │ - ├─┼─┼─┤ - │ foo │ 0001 │ https://0001.pipelines.cloudflarestorage.com │ - └─┴─┴─┘" - `); - expect(requests.count).toEqual(1); - }); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.warn).toMatchInlineSnapshot(` + "▲ [WARNING] 🚧 \`wrangler pipelines get\` is an open-beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose - describe("get", () => { - it("should get pipeline pretty", async () => { - const requests = mockGetRequest("foo", samplePipeline); - await runWrangler("pipelines get foo"); - expect(std.err).toMatchInlineSnapshot(`""`); + ▲ [WARNING] ⚠️ This is a legacy pipeline. Consider creating a new pipeline by running 'wrangler pipelines setup'. + + " + `); expect(std.out).toMatchInlineSnapshot(` - "Id: 0001 - Name: my-pipeline + "Id: legacy_123 + Name: my-legacy-pipeline Sources: HTTP: - Endpoint: https://0001.pipelines.cloudflarestorage.com + Endpoint: https://pipelines.cloudflare.com/legacy Authentication: off - CORS Origins: * Format: JSON - Worker: - Format: JSON Destination: Type: R2 - Bucket: bucket + Bucket: my-bucket Format: newline-delimited JSON - Compression: NONE + Prefix: data/ + Compression: GZIP Batch hints: Max bytes: 100 MB Max duration: 300 seconds - Max records: 100,000 + Max records: 10,000,000 " `); - expect(requests.count).toEqual(1); }); + }); - it("should get pipeline json", async () => { - const requests = mockGetRequest("foo", samplePipeline); - await runWrangler("pipelines get foo --format=json"); + describe("pipelines delete", () => { + const { setIsTTY } = useMockIsTTY(); + it("should error when no pipeline ID provided", async () => { + await expect( + runWrangler("pipelines delete") + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Not enough non-option arguments: got 0, need at least 1]` + ); + }); + + it("should prompt for confirmation before delete", async () => { + const mockPipeline: Pipeline = { + id: "pipeline_123", + name: "my_pipeline", + sql: "INSERT INTO test_sink SELECT * FROM test_stream;", + status: "active", + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }; + + const getRequest = mockGetPipelineRequest("pipeline_123", mockPipeline); + const deleteRequest = mockDeletePipelineRequest("pipeline_123"); + + setIsTTY(true); + mockConfirm({ + text: "Are you sure you want to delete the pipeline 'my_pipeline' (pipeline_123)?", + result: true, + }); + + await runWrangler("pipelines delete pipeline_123"); + expect(getRequest.count).toBe(1); + expect(deleteRequest.count).toBe(1); expect(std.err).toMatchInlineSnapshot(`""`); expect(std.out).toMatchInlineSnapshot(` - "{ - \\"id\\": \\"0001\\", - \\"version\\": 1, - \\"name\\": \\"my-pipeline\\", - \\"metadata\\": {}, - \\"source\\": [ - { - \\"type\\": \\"binding\\", - \\"format\\": \\"json\\" - }, - { - \\"type\\": \\"http\\", - \\"format\\": \\"json\\", - \\"authentication\\": false, - \\"cors\\": { - \\"origins\\": [ - \\"*\\" - ] - } - } - ], - \\"transforms\\": [], - \\"destination\\": { - \\"type\\": \\"r2\\", - \\"format\\": \\"json\\", - \\"batch\\": { - \\"max_bytes\\": 100000000, - \\"max_duration_s\\": 300, - \\"max_rows\\": 100000 - }, - \\"compression\\": { - \\"type\\": \\"none\\" - }, - \\"path\\": { - \\"bucket\\": \\"bucket\\" - } - }, - \\"endpoint\\": \\"https://0001.pipelines.cloudflarestorage.com\\" - }" + "✨ Successfully deleted pipeline 'my_pipeline' with id 'pipeline_123'." `); - expect(requests.count).toEqual(1); }); - it("should fail on missing pipeline", async () => { - const requests = mockGetRequest("bad-pipeline", null, 404, { - code: 1000, - message: "Pipeline does not exist", + it("should fall back to legacy API when deleting pipeline not in new API", async () => { + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/v1/pipelines/my-legacy-pipeline`, + () => { + return HttpResponse.json( + { + success: false, + errors: [{ code: 1000, message: "Pipeline not found" }], + messages: [], + result: null, + }, + { status: 404 } + ); + }, + { once: true } + ) + ); + + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/my-legacy-pipeline`, + () => { + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: { + id: "legacy_123", + name: "my-legacy-pipeline", + endpoint: "https://pipelines.cloudflare.com/legacy", + }, + }); + }, + { once: true } + ) + ); + + msw.use( + http.delete( + `*/accounts/${accountId}/pipelines/my-legacy-pipeline`, + () => { + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: null, + }); + }, + { once: true } + ) + ); + + setIsTTY(true); + mockConfirm({ + text: "Are you sure you want to delete the legacy pipeline 'my-legacy-pipeline'?", + result: true, }); - await expect( - runWrangler("pipelines get bad-pipeline") - ).rejects.toThrowError(); - await endEventLoop(); + await runWrangler("pipelines delete my-legacy-pipeline"); expect(std.err).toMatchInlineSnapshot(`""`); - expect(normalizeOutput(std.out)).toMatchInlineSnapshot(` - "X [ERROR] A request to the Cloudflare API (/accounts/some-account-id/pipelines/bad-pipeline) failed. - Pipeline does not exist [code: 1000] - If you think this is a bug, please open an issue at: - https://github.com/cloudflare/workers-sdk/issues/new/choose" + expect(std.out).toMatchInlineSnapshot(` + "✨ Successfully deleted legacy pipeline 'my-legacy-pipeline'." `); - expect(requests.count).toEqual(1); }); }); - describe("update", () => { - it("should update a pipeline", async () => { - const pipeline: Pipeline = samplePipeline; - mockGetRequest(pipeline.name, pipeline); + describe("pipelines update", () => { + it("should error when trying to update V1 pipeline", async () => { + const mockPipeline: Pipeline = { + id: "pipeline_123", + name: "my_pipeline", + sql: "INSERT INTO test_sink SELECT * FROM test_stream;", + status: "active", + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }; - const update = JSON.parse(JSON.stringify(pipeline)); - update.destination.compression.type = "gzip"; - const updateReq = mockUpdateRequest(update.name, update); + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/v1/pipelines/pipeline_123`, + () => { + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: mockPipeline, + }); + }, + { once: true } + ) + ); - await runWrangler("pipelines update my-pipeline --compression gzip"); - expect(updateReq.count).toEqual(1); + await expect( + runWrangler("pipelines update pipeline_123 --batch-max-mb 50") + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Pipelines created with the V1 API cannot be updated. To modify your pipeline, delete and recreate it with your new SQL.]` + ); }); - it("should update a pipeline with new bucket", async () => { - const pipeline: Pipeline = samplePipeline; - mockGetRequest(pipeline.name, pipeline); + it("should update legacy pipeline with warning", async () => { + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/v1/pipelines/my-legacy-pipeline`, + () => { + return HttpResponse.json( + { + success: false, + errors: [{ code: 1000, message: "Pipeline not found" }], + messages: [], + result: null, + }, + { status: 404 } + ); + }, + { once: true } + ) + ); - const update = JSON.parse(JSON.stringify(pipeline)); - update.destination.path.bucket = "new_bucket"; - update.destination.credentials = { - endpoint: "https://some-account-id.r2.cloudflarestorage.com", - access_key_id: "service-token-id", - secret_access_key: "my-secret-access-key", + const mockLegacyPipeline = { + id: "legacy_123", + name: "my-legacy-pipeline", + endpoint: "https://pipelines.cloudflare.com/legacy", + source: [{ type: "http", format: "json" }], + destination: { + type: "r2", + format: "json", + path: { bucket: "my-bucket", prefix: "data/" }, + batch: { + max_duration_s: 300, + max_bytes: 100000000, + max_rows: 10000000, + }, + compression: { type: "gzip" }, + }, + transforms: [], + metadata: { shards: 2 }, }; - const updateReq = mockUpdateRequest(update.name, update); + + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/my-legacy-pipeline`, + () => { + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: mockLegacyPipeline, + }); + }, + { once: true } + ) + ); + + msw.use( + http.put( + `*/accounts/${accountId}/pipelines/my-legacy-pipeline`, + () => { + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: { + ...mockLegacyPipeline, + destination: { + ...mockLegacyPipeline.destination, + batch: { + ...mockLegacyPipeline.destination.batch, + max_bytes: 50000000, + }, + }, + }, + }); + }, + { once: true } + ) + ); await runWrangler( - "pipelines update my-pipeline --r2-bucket new-bucket --r2-access-key-id service-token-id --r2-secret-access-key my-secret-access-key" + "pipelines update my-legacy-pipeline --batch-max-mb 50" ); - expect(updateReq.count).toEqual(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.warn).toMatchInlineSnapshot(` + "▲ [WARNING] 🚧 \`wrangler pipelines update\` is an open-beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose + + + ▲ [WARNING] ⚠️ Updating legacy pipeline. Consider recreating with 'wrangler pipelines setup'. + + " + `); + expect(std.out).toMatchInlineSnapshot(` + "🌀 Updating pipeline \\"my-legacy-pipeline\\" + ✨ Successfully updated pipeline \\"my-legacy-pipeline\\" with ID legacy_123 + " + `); }); + }); - it("should update a pipeline with new credential", async () => { - const pipeline: Pipeline = samplePipeline; - mockGetRequest(pipeline.name, pipeline); + describe("pipelines streams create", () => { + const { setIsTTY } = useMockIsTTY(); + function mockCreateStreamRequest(expectedRequest: { + name: string; + hasSchema?: boolean; + }) { + const requests = { count: 0 }; + msw.use( + http.post( + `*/accounts/${accountId}/pipelines/v1/streams`, + async ({ request }) => { + requests.count++; + const body = (await request.json()) as { + name: string; + schema?: { fields: SchemaField[] }; + }; + expect(body.name).toBe(expectedRequest.name); + + const schema = expectedRequest.hasSchema + ? { + fields: [ + { name: "id", type: "string", required: true }, + { + name: "timestamp", + type: "timestamp", + required: true, + unit: "millisecond", + }, + ], + } + : null; + + const format = expectedRequest.hasSchema + ? { type: "json" } + : { type: "json", unstructured: true }; + + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: { + id: "stream_123", + name: expectedRequest.name, + version: 1, + endpoint: `https://pipelines.cloudflare.com/${expectedRequest.name}`, + format, + schema, + http: { + enabled: true, + authentication: true, + }, + worker_binding: { enabled: true }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }, + }); + }, + { once: true } + ) + ); + return requests; + } + + it("should error when no stream name provided", async () => { + await expect( + runWrangler("pipelines streams create") + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Not enough non-option arguments: got 0, need at least 1]` + ); + }); + + it("should create stream with default settings", async () => { + setIsTTY(true); + mockConfirm({ + text: "No schema file provided. Do you want to create stream without a schema (unstructured JSON)?", + result: true, + }); + + const createRequest = mockCreateStreamRequest({ + name: "my_stream", + }); + + await runWrangler("pipelines streams create my_stream"); - const update = JSON.parse(JSON.stringify(pipeline)); - update.destination.path.bucket = "new-bucket"; - update.destination.credentials = { - endpoint: "https://some-account-id.r2.cloudflarestorage.com", - access_key_id: "new-key", - secret_access_key: "new-secret", + expect(createRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "🌀 Creating stream 'my_stream'... + ✨ Successfully created stream 'my_stream' with id 'stream_123'. + + Creation Summary: + General: + Name: my_stream + + HTTP Ingest: + Enabled: Yes + Authentication: Yes + Endpoint: https://pipelines.cloudflare.com/my_stream + CORS Origins: None + + Input Schema: Unstructured JSON (single 'value' column)" + `); + }); + + it("should create stream with schema from file", async () => { + const schemaFile = "schema.json"; + const schema = { + fields: [ + { name: "id", type: "string", required: true }, + { + name: "timestamp", + type: "timestamp", + required: true, + unit: "millisecond", + }, + ], }; - const updateReq = mockUpdateRequest(update.name, update); + writeFileSync(schemaFile, JSON.stringify(schema)); + + const createRequest = mockCreateStreamRequest({ + name: "my_stream", + hasSchema: true, + }); await runWrangler( - "pipelines update my-pipeline --r2-bucket new-bucket --r2-access-key-id new-key --r2-secret-access-key new-secret" + `pipelines streams create my_stream --schema-file ${schemaFile}` ); - expect(updateReq.count).toEqual(1); + expect(createRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "🌀 Creating stream 'my_stream'... + ✨ Successfully created stream 'my_stream' with id 'stream_123'. + + Creation Summary: + General: + Name: my_stream + + HTTP Ingest: + Enabled: Yes + Authentication: Yes + Endpoint: https://pipelines.cloudflare.com/my_stream + CORS Origins: None + + Input Schema: + ┌─┬─┬─┬─┐ + │ Field Name │ Type │ Unit/Items │ Required │ + ├─┼─┼─┼─┤ + │ id │ string │ │ Yes │ + ├─┼─┼─┼─┤ + │ timestamp │ timestamp │ millisecond │ Yes │ + └─┴─┴─┴─┘" + `); }); + }); - it("should update a pipeline with source changes http auth", async () => { - const pipeline: Pipeline = samplePipeline; - mockGetRequest(pipeline.name, pipeline); + describe("pipelines streams list", () => { + function mockListStreamsRequest(streams: Stream[], pipelineId?: string) { + const requests = { count: 0 }; + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/v1/streams`, + ({ request }) => { + requests.count++; + const url = new URL(request.url); + if (pipelineId) { + expect(url.searchParams.get("pipeline_id")).toBe(pipelineId); + } + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: streams, + result_info: { + page: 1, + per_page: 20, + count: streams.length, + total_count: streams.length, + }, + }); + }, + { once: true } + ) + ); + return requests; + } - const update = JSON.parse(JSON.stringify(pipeline)); - update.source = [ + it("should list streams", async () => { + const mockStreams: Stream[] = [ { - type: "http", - format: "json", - authenticated: true, + id: "stream_1", + name: "stream_one", + version: 1, + endpoint: "https://pipelines.cloudflare.com/stream_1", + format: { type: "json", unstructured: true }, + schema: null, + http: { enabled: true, authentication: false }, + worker_binding: { enabled: true }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", }, ]; - const updateReq = mockUpdateRequest(update.name, update); - await runWrangler( - "pipelines update my-pipeline --source http --require-http-auth" - ); + const listRequest = mockListStreamsRequest(mockStreams); - expect(updateReq.count).toEqual(1); - expect(updateReq.body?.source.length).toEqual(1); - expect(updateReq.body?.source[0].type).toEqual("http"); - expect((updateReq.body?.source[0] as HttpSource).authentication).toEqual( - true - ); - }); + await runWrangler("pipelines streams list"); - it("should update a pipeline cors headers", async () => { - const pipeline: Pipeline = samplePipeline; - mockGetRequest(pipeline.name, pipeline); + expect(listRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "┌─┬─┬─┬─┐ + │ Name │ ID │ HTTP │ Created │ + ├─┼─┼─┼─┤ + │ stream_one │ stream_1 │ Yes (unauthenticated) │ 1/1/2024 │ + └─┴─┴─┴─┘" + `); + }); - const update = JSON.parse(JSON.stringify(pipeline)); - update.source = [ + it("should filter by pipeline ID", async () => { + const mockStreams: Stream[] = [ { - type: "http", - format: "json", - authenticated: true, + id: "stream_1", + name: "filtered_stream", + version: 1, + endpoint: "https://pipelines.cloudflare.com/stream_1", + format: { type: "json", unstructured: true }, + schema: null, + http: { enabled: true, authentication: false }, + worker_binding: { enabled: true }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", }, ]; - const updateReq = mockUpdateRequest(update.name, update); - await runWrangler( - "pipelines update my-pipeline --cors-origins http://localhost:8787" + const listRequest = mockListStreamsRequest(mockStreams, "pipeline_123"); + + await runWrangler("pipelines streams list --pipeline-id pipeline_123"); + + expect(listRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "┌─┬─┬─┬─┐ + │ Name │ ID │ HTTP │ Created │ + ├─┼─┼─┼─┤ + │ filtered_stream │ stream_1 │ Yes (unauthenticated) │ 1/1/2024 │ + └─┴─┴─┴─┘" + `); + }); + }); + + describe("pipelines streams get", () => { + it("should get stream details", async () => { + const mockStream: Stream = { + id: "stream_123", + name: "my_stream", + version: 1, + endpoint: "https://pipelines.cloudflare.com/stream_123", + format: { type: "json", unstructured: true }, + schema: null, + http: { enabled: true, authentication: true }, + worker_binding: { enabled: true }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }; + + const getRequest = mockGetStreamRequest("stream_123", mockStream); + + await runWrangler("pipelines streams get stream_123"); + + expect(getRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "Stream ID: stream_123 + + Configuration: + General: + Name: my_stream + Created At: 1/1/2024, 12:00:00 AM + Modified At: 1/1/2024, 12:00:00 AM + + HTTP Ingest: + Enabled: Yes + Authentication: Yes + Endpoint: https://pipelines.cloudflare.com/stream_123 + CORS Origins: None + + Input Schema: Unstructured JSON (single 'value' column)" + `); + }); + }); + + describe("pipelines streams delete", () => { + const { setIsTTY } = useMockIsTTY(); + function mockDeleteStreamRequest(streamId: string) { + const requests = { count: 0 }; + msw.use( + http.delete( + `*/accounts/${accountId}/pipelines/v1/streams/${streamId}`, + () => { + requests.count++; + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: null, + }); + }, + { once: true } + ) + ); + return requests; + } + + it("should prompt for confirmation", async () => { + const mockStream: Stream = { + id: "stream_123", + name: "my_stream", + version: 1, + endpoint: "https://pipelines.cloudflare.com/stream_123", + format: { type: "json", unstructured: true }, + schema: null, + http: { enabled: true, authentication: false }, + worker_binding: { enabled: true }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }; + + const getRequest = mockGetStreamRequest("stream_123", mockStream); + const deleteRequest = mockDeleteStreamRequest("stream_123"); + + setIsTTY(true); + mockConfirm({ + text: "Are you sure you want to delete the stream 'my_stream' (stream_123)?", + result: true, + }); + + await runWrangler("pipelines streams delete stream_123"); + + expect(getRequest.count).toBe(1); + expect(deleteRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "✨ Successfully deleted stream 'my_stream' with id 'stream_123'." + `); + }); + }); + + describe("pipelines sinks create", () => { + function mockCreateSinkRequest(expectedRequest: { + name: string; + type: string; + isDataCatalog?: boolean; + }) { + const requests = { count: 0 }; + msw.use( + http.post( + `*/accounts/${accountId}/pipelines/v1/sinks`, + async ({ request }) => { + requests.count++; + const body = (await request.json()) as { + name: string; + type: string; + config?: Record; + }; + expect(body.name).toBe(expectedRequest.name); + expect(body.type).toBe(expectedRequest.type); + + const config = expectedRequest.isDataCatalog + ? { + bucket: "catalog-bucket", + namespace: "default", + table_name: "my-table", + token: "token123", + } + : { + bucket: "my-bucket", + credentials: { + access_key_id: "key123", + secret_access_key: "secret123", + }, + }; + + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: { + id: "sink_123", + name: expectedRequest.name, + type: expectedRequest.type, + format: { type: "json" }, + schema: null, + config, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }, + }); + }, + { once: true } + ) ); + return requests; + } - expect(updateReq.count).toEqual(1); - expect(updateReq.body?.source.length).toEqual(2); - expect(updateReq.body?.source[1].type).toEqual("http"); - expect((updateReq.body?.source[1] as HttpSource).cors?.origins).toEqual([ - "http://localhost:8787", - ]); + it("should error when type is missing", async () => { + await expect( + runWrangler("pipelines sinks create my_sink --bucket my-bucket") + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Missing required argument: type]` + ); }); - it("should update remove cors headers", async () => { - const pipeline: Pipeline = samplePipeline; - mockGetRequest(pipeline.name, pipeline); + it("should error with invalid bucket name", async () => { + await expect( + runWrangler( + "pipelines sinks create my_sink --type r2 --bucket invalid_bucket_name" + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: The bucket name "invalid_bucket_name" is invalid. Bucket names must begin and end with an alphanumeric character, only contain lowercase letters, numbers, and hyphens, and be between 3 and 63 characters long.]` + ); + }); - const update = JSON.parse(JSON.stringify(pipeline)); - update.source = [ - { - type: "http", - format: "json", - authenticated: true, - }, - ]; - const updateReq = mockUpdateRequest(update.name, update); + it("should create R2 sink with explicit credentials", async () => { + const createRequest = mockCreateSinkRequest({ + name: "my_sink", + type: "r2", + }); await runWrangler( - "pipelines update my-pipeline --cors-origins http://localhost:8787" - ); - - expect(updateReq.count).toEqual(1); - expect(updateReq.body?.source.length).toEqual(2); - expect(updateReq.body?.source[1].type).toEqual("http"); - expect((updateReq.body?.source[1] as HttpSource).cors?.origins).toEqual([ - "http://localhost:8787", - ]); - - mockGetRequest(pipeline.name, pipeline); - const secondUpdateReq = mockUpdateRequest(update.name, update); - await runWrangler("pipelines update my-pipeline --cors-origins none"); - expect(secondUpdateReq.count).toEqual(1); - expect(secondUpdateReq.body?.source.length).toEqual(2); - expect(secondUpdateReq.body?.source[1].type).toEqual("http"); - expect( - (secondUpdateReq.body?.source[1] as HttpSource).cors?.origins - ).toEqual([]); - }); - - it("should fail a missing pipeline", async () => { - const requests = mockGetRequest("bad-pipeline", null, 404, { - code: 1000, - message: "Pipeline does not exist", + "pipelines sinks create my_sink --type r2 --bucket my-bucket --access-key-id mykey --secret-access-key mysecret" + ); + + expect(createRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "🌀 Creating sink 'my_sink'... + ✨ Successfully created sink 'my_sink' with id 'sink_123'. + + Creation Summary: + General: + Type: R2 + + Destination: + Bucket: my-bucket + Path: (root) + Partitioning: year=%Y/month=%m/day=%d + + Batching: + File Size: none + Time Interval: 30s + + Format: + Type: json" + `); + }); + + it("should create R2 Data Catalog sink", async () => { + const createRequest = mockCreateSinkRequest({ + name: "my_sink", + type: "r2_data_catalog", + isDataCatalog: true, }); + + await runWrangler( + "pipelines sinks create my_sink --type r2-data-catalog --bucket catalog-bucket --namespace default --table my-table --catalog-token token123" + ); + + expect(createRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "🌀 Creating sink 'my_sink'... + ✨ Successfully created sink 'my_sink' with id 'sink_123'. + + Creation Summary: + General: + Type: R2 Data Catalog + + Destination: + Bucket: catalog-bucket + Table: default.my-table + + Batching: + File Size: none + Time Interval: 30s + + Format: + Type: json" + `); + }); + + it("should error when r2-data-catalog missing required fields", async () => { await expect( runWrangler( - "pipelines update bad-pipeline --r2-bucket new-bucket --r2-access-key-id new-key --r2-secret-access-key new-secret" + "pipelines sinks create my_sink --type r2-data-catalog --bucket catalog-bucket" ) - ).rejects.toThrowError(); + ).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: --namespace is required for r2-data-catalog sinks]` + ); + }); + }); - await endEventLoop(); + describe("pipelines sinks list", () => { + function mockListSinksRequest(sinks: Sink[], pipelineId?: string) { + const requests = { count: 0 }; + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/v1/sinks`, + ({ request }) => { + requests.count++; + const url = new URL(request.url); + if (pipelineId) { + expect(url.searchParams.get("pipeline_id")).toBe(pipelineId); + } + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: sinks, + result_info: { + page: 1, + per_page: 20, + count: sinks.length, + total_count: sinks.length, + }, + }); + }, + { once: true } + ) + ); + return requests; + } + it("should list sinks", async () => { + const mockSinks: Sink[] = [ + { + id: "sink_1", + name: "sink_one", + type: "r2", + format: { type: "json" }, + schema: null, + config: { bucket: "bucket1" }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }, + ]; + + const listRequest = mockListSinksRequest(mockSinks); + + await runWrangler("pipelines sinks list"); + + expect(listRequest.count).toBe(1); expect(std.err).toMatchInlineSnapshot(`""`); - expect(normalizeOutput(std.out)).toMatchInlineSnapshot(` - "X [ERROR] A request to the Cloudflare API (/accounts/some-account-id/pipelines/bad-pipeline) failed. - Pipeline does not exist [code: 1000] - If you think this is a bug, please open an issue at: - https://github.com/cloudflare/workers-sdk/issues/new/choose" + expect(std.out).toMatchInlineSnapshot(` + "┌─┬─┬─┬─┬─┐ + │ Name │ ID │ Type │ Destination │ Created │ + ├─┼─┼─┼─┼─┤ + │ sink_one │ sink_1 │ R2 │ bucket1 │ 1/1/2024 │ + └─┴─┴─┴─┴─┘" `); - expect(requests.count).toEqual(1); }); - it("should remove transformations", async () => { - const pipeline: Pipeline = samplePipeline; - mockGetRequest(pipeline.name, pipeline); - - const update = JSON.parse(JSON.stringify(pipeline)); - update.transforms = [ + it("should filter by pipeline ID", async () => { + const mockSinks: Sink[] = [ { - script: "hello", - entrypoint: "MyTransform", + id: "sink_1", + name: "filtered_sink", + type: "r2", + format: { type: "json" }, + schema: null, + config: { bucket: "bucket1" }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", }, ]; - const updateReq = mockUpdateRequest(update.name, update); - await runWrangler("pipelines update my-pipeline --transform-worker none"); + const listRequest = mockListSinksRequest(mockSinks, "pipeline_123"); + + await runWrangler("pipelines sinks list --pipeline-id pipeline_123"); - expect(updateReq.count).toEqual(1); - expect(updateReq.body?.transforms.length).toEqual(0); + expect(listRequest.count).toBe(1); + expect(std.err).toMatchInlineSnapshot(`""`); + expect(std.out).toMatchInlineSnapshot(` + "┌─┬─┬─┬─┬─┐ + │ Name │ ID │ Type │ Destination │ Created │ + ├─┼─┼─┼─┼─┤ + │ filtered_sink │ sink_1 │ R2 │ bucket1 │ 1/1/2024 │ + └─┴─┴─┴─┴─┘" + `); }); }); - describe("delete", () => { - it("should delete pipeline", async () => { - const requests = mockDeleteRequest("foo"); - await runWrangler("pipelines delete foo"); + function mockGetSinkRequest(sinkId: string, sink: Sink) { + const requests = { count: 0 }; + msw.use( + http.get( + `*/accounts/${accountId}/pipelines/v1/sinks/${sinkId}`, + () => { + requests.count++; + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: sink, + }); + }, + { once: true } + ) + ); + return requests; + } + + describe("pipelines sinks get", () => { + it("should get sink details", async () => { + const mockSink: Sink = { + id: "sink_123", + name: "my_sink", + type: "r2", + format: { type: "json" }, + schema: null, + config: { + bucket: "my-bucket", + }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }; + + const getRequest = mockGetSinkRequest("sink_123", mockSink); + await runWrangler("pipelines sinks get sink_123"); + + expect(getRequest.count).toBe(1); expect(std.err).toMatchInlineSnapshot(`""`); expect(std.out).toMatchInlineSnapshot(` - "Deleting pipeline foo. - Deleted pipeline foo." + "ID: sink_123 + Name: my_sink + General: + Type: R2 + Created At: 1/1/2024, 12:00:00 AM + Modified At: 1/1/2024, 12:00:00 AM + + Destination: + Bucket: my-bucket + Path: (root) + Partitioning: year=%Y/month=%m/day=%d + + Batching: + File Size: none + Time Interval: 30s + + Format: + Type: json" `); - expect(requests.count).toEqual(1); }); + }); + + describe("pipelines sinks delete", () => { + const { setIsTTY } = useMockIsTTY(); + function mockDeleteSinkRequest(sinkId: string) { + const requests = { count: 0 }; + msw.use( + http.delete( + `*/accounts/${accountId}/pipelines/v1/sinks/${sinkId}`, + () => { + requests.count++; + return HttpResponse.json({ + success: true, + errors: [], + messages: [], + result: null, + }); + }, + { once: true } + ) + ); + return requests; + } + + it("should prompt for confirmation", async () => { + const mockSink: Sink = { + id: "sink_123", + name: "my_sink", + type: "r2", + format: { type: "json" }, + schema: null, + config: { bucket: "my-bucket" }, + created_at: "2024-01-01T00:00:00Z", + modified_at: "2024-01-01T00:00:00Z", + }; - it("should fail a missing pipeline", async () => { - const requests = mockDeleteRequest("bad-pipeline", 404, { - code: 1000, - message: "Pipeline does not exist", + const getSinkRequest = mockGetSinkRequest("sink_123", mockSink); + const deleteRequest = mockDeleteSinkRequest("sink_123"); + + setIsTTY(true); + mockConfirm({ + text: "Are you sure you want to delete the sink 'my_sink' (sink_123)?", + result: true, }); - await expect( - runWrangler("pipelines delete bad-pipeline") - ).rejects.toThrowError(); - await endEventLoop(); + await runWrangler("pipelines sinks delete sink_123"); + expect(getSinkRequest.count).toBe(1); + expect(deleteRequest.count).toBe(1); expect(std.err).toMatchInlineSnapshot(`""`); - expect(normalizeOutput(std.out)).toMatchInlineSnapshot(` - "Deleting pipeline bad-pipeline. - X [ERROR] A request to the Cloudflare API (/accounts/some-account-id/pipelines/bad-pipeline) failed. - Pipeline does not exist [code: 1000] - If you think this is a bug, please open an issue at: - https://github.com/cloudflare/workers-sdk/issues/new/choose" + expect(std.out).toMatchInlineSnapshot(` + "✨ Successfully deleted sink 'my_sink' with id 'sink_123'." `); - expect(requests.count).toEqual(1); }); }); }); diff --git a/packages/wrangler/src/index.ts b/packages/wrangler/src/index.ts index 8a40389c8242..6e13c4fc1e30 100644 --- a/packages/wrangler/src/index.ts +++ b/packages/wrangler/src/index.ts @@ -153,6 +153,17 @@ import { pipelinesCreateCommand } from "./pipelines/cli/create"; import { pipelinesDeleteCommand } from "./pipelines/cli/delete"; import { pipelinesGetCommand } from "./pipelines/cli/get"; import { pipelinesListCommand } from "./pipelines/cli/list"; +import { pipelinesSetupCommand } from "./pipelines/cli/setup"; +import { pipelinesSinksNamespace } from "./pipelines/cli/sinks"; +import { pipelinesSinksCreateCommand } from "./pipelines/cli/sinks/create"; +import { pipelinesSinksDeleteCommand } from "./pipelines/cli/sinks/delete"; +import { pipelinesSinksGetCommand } from "./pipelines/cli/sinks/get"; +import { pipelinesSinksListCommand } from "./pipelines/cli/sinks/list"; +import { pipelinesStreamsNamespace } from "./pipelines/cli/streams"; +import { pipelinesStreamsCreateCommand } from "./pipelines/cli/streams/create"; +import { pipelinesStreamsDeleteCommand } from "./pipelines/cli/streams/delete"; +import { pipelinesStreamsGetCommand } from "./pipelines/cli/streams/get"; +import { pipelinesStreamsListCommand } from "./pipelines/cli/streams/list"; import { pipelinesUpdateCommand } from "./pipelines/cli/update"; import { pubSubCommands } from "./pubsub/pubsub-commands"; import { queuesNamespace } from "./queues/cli/commands"; @@ -1383,6 +1394,10 @@ export function createCLIParser(argv: string[]) { command: "wrangler pipelines", definition: pipelinesNamespace, }, + { + command: "wrangler pipelines setup", + definition: pipelinesSetupCommand, + }, { command: "wrangler pipelines create", definition: pipelinesCreateCommand, @@ -1403,6 +1418,46 @@ export function createCLIParser(argv: string[]) { command: "wrangler pipelines delete", definition: pipelinesDeleteCommand, }, + { + command: "wrangler pipelines streams", + definition: pipelinesStreamsNamespace, + }, + { + command: "wrangler pipelines streams create", + definition: pipelinesStreamsCreateCommand, + }, + { + command: "wrangler pipelines streams list", + definition: pipelinesStreamsListCommand, + }, + { + command: "wrangler pipelines streams get", + definition: pipelinesStreamsGetCommand, + }, + { + command: "wrangler pipelines streams delete", + definition: pipelinesStreamsDeleteCommand, + }, + { + command: "wrangler pipelines sinks", + definition: pipelinesSinksNamespace, + }, + { + command: "wrangler pipelines sinks create", + definition: pipelinesSinksCreateCommand, + }, + { + command: "wrangler pipelines sinks list", + definition: pipelinesSinksListCommand, + }, + { + command: "wrangler pipelines sinks get", + definition: pipelinesSinksGetCommand, + }, + { + command: "wrangler pipelines sinks delete", + definition: pipelinesSinksDeleteCommand, + }, ]); registry.registerNamespace("pipelines"); diff --git a/packages/wrangler/src/pipelines/cli/create.ts b/packages/wrangler/src/pipelines/cli/create.ts index 5c7ee5925698..9e0bb4f64822 100644 --- a/packages/wrangler/src/pipelines/cli/create.ts +++ b/packages/wrangler/src/pipelines/cli/create.ts @@ -1,25 +1,12 @@ -import { updateConfigFile } from "../../config"; +import { readFileSync } from "node:fs"; import { createCommand } from "../../core/create-command"; -import { FatalError, UserError } from "../../errors"; +import { UserError } from "../../errors"; import { logger } from "../../logger"; -import { bucketFormatMessage, isValidR2BucketName } from "../../r2/helpers"; +import { APIError } from "../../parse"; import { requireAuth } from "../../user"; -import { getValidBindingName } from "../../utils/getValidBindingName"; -import { createPipeline } from "../client"; -import { - authorizeR2Bucket, - BYTES_PER_MB, - formatPipelinePretty, - getAccountR2Endpoint, - parseTransform, -} from "../index"; -import { validateCorsOrigins, validateInRange } from "../validate"; -import type { - BindingSource, - HttpSource, - PipelineUserConfig, - Source, -} from "../client"; +import { createPipeline, getPipeline, getStream, validateSql } from "../client"; +import { displayUsageExamples } from "./streams/utils"; +import type { CreatePipelineRequest } from "../types"; export const pipelinesCreateCommand = createCommand({ metadata: { @@ -29,274 +16,126 @@ export const pipelinesCreateCommand = createCommand({ }, args: { pipeline: { - describe: "The name of the new pipeline", + describe: "The name of the pipeline to create", type: "string", demandOption: true, }, - - source: { - type: "array", - describe: - "Space separated list of allowed sources. Options are 'http' or 'worker'", - default: ["http", "worker"], - demandOption: false, - group: "Source settings", - }, - "require-http-auth": { - type: "boolean", - describe: - "Require Cloudflare API Token for HTTPS endpoint authentication", - default: false, - demandOption: false, - group: "Source settings", - }, - "cors-origins": { - type: "array", - describe: - "CORS origin allowlist for HTTP endpoint (use * for any origin). Defaults to an empty array", - demandOption: false, - coerce: validateCorsOrigins, - group: "Source settings", - }, - - "batch-max-mb": { - type: "number", - describe: - "Maximum batch size in megabytes before flushing. Defaults to 100 MB if unset. Minimum: 1, Maximum: 100", - demandOption: false, - coerce: validateInRange("batch-max-mb", 1, 100), - group: "Batch hints", - }, - "batch-max-rows": { - type: "number", - describe: - "Maximum number of rows per batch before flushing. Defaults to 10,000,000 if unset. Minimum: 100, Maximum: 10,000,000", - demandOption: false, - coerce: validateInRange("batch-max-rows", 100, 10_000_000), - group: "Batch hints", - }, - "batch-max-seconds": { - type: "number", - describe: - "Maximum age of batch in seconds before flushing. Defaults to 300 if unset. Minimum: 1, Maximum: 300", - - demandOption: false, - coerce: validateInRange("batch-max-seconds", 1, 300), - group: "Batch hints", - }, - - // Transform options - "transform-worker": { - type: "string", - describe: - "Pipeline transform Worker and entrypoint (.)", - demandOption: false, - hidden: true, // TODO: Remove once transformations launch - group: "Transformations", - }, - - "r2-bucket": { + sql: { + describe: "Inline SQL query for the pipeline", type: "string", - describe: "Destination R2 bucket name", - demandOption: true, - group: "Destination settings", - }, - "r2-access-key-id": { - type: "string", - describe: - "R2 service Access Key ID for authentication. Leave empty for OAuth confirmation.", - demandOption: false, - group: "Destination settings", - implies: "r2-secret-access-key", + conflicts: "sql-file", }, - "r2-secret-access-key": { + "sql-file": { + describe: "Path to file containing SQL query for the pipeline", type: "string", - describe: - "R2 service Secret Access Key for authentication. Leave empty for OAuth confirmation.", - demandOption: false, - group: "Destination settings", - implies: "r2-access-key-id", - }, - - "r2-prefix": { - type: "string", - describe: - "Prefix for storing files in the destination bucket. Default is no prefix", - default: "", - demandOption: false, - group: "Destination settings", - }, - compression: { - type: "string", - describe: "Compression format for output files", - choices: ["none", "gzip", "deflate"], - default: "gzip", - demandOption: false, - group: "Destination settings", - }, - - // Pipeline settings - "shard-count": { - type: "number", - describe: - "Number of shards for the pipeline. More shards handle higher request volume; fewer shards produce larger output files. Defaults to 2 if unset. Minimum: 1, Maximum: 15", - demandOption: false, - group: "Pipeline settings", + conflicts: "sql", }, }, positionalArgs: ["pipeline"], - validateArgs(args) { - if ( - (args.r2AccessKeyId && !args.r2SecretAccessKey) || - (!args.r2AccessKeyId && args.r2SecretAccessKey) - ) { - throw new UserError( - "--r2-access-key-id and --r2-secret-access-key must be provided together" - ); - } - }, async handler(args, { config }) { - const bucket = args.r2Bucket; - if (!isValidR2BucketName(bucket)) { - throw new UserError( - `The bucket name "${bucket}" is invalid. ${bucketFormatMessage}` - ); - } - const name = args.pipeline; - const compression = args.compression; - - const batch = { - max_bytes: args.batchMaxMb - ? args.batchMaxMb * BYTES_PER_MB // convert to bytes for the API - : undefined, - max_duration_s: args.batchMaxSeconds, - max_rows: args.batchMaxRows, - }; - - const accountId = await requireAuth(config); - const pipelineConfig: PipelineUserConfig = { - name: name, - metadata: {}, - source: [], - transforms: [], - destination: { - type: "r2", - format: "json", - compression: { - type: compression, - }, - batch: batch, - path: { - bucket, - }, - credentials: { - endpoint: getAccountR2Endpoint(accountId), - access_key_id: args.r2AccessKeyId || "", - secret_access_key: args.r2SecretAccessKey || "", - }, - }, - }; - const destination = pipelineConfig.destination; - if ( - !destination.credentials.access_key_id && - !destination.credentials.secret_access_key - ) { - // auto-generate a service token - const auth = await authorizeR2Bucket( - config, - name, - accountId, - pipelineConfig.destination.path.bucket - ); - destination.credentials.access_key_id = auth.accessKeyId; - destination.credentials.secret_access_key = auth.secretAccessKey; - } - - if (!destination.credentials.access_key_id) { - throw new FatalError("Requires a r2 access key id"); + await requireAuth(config); + const pipelineName = args.pipeline; + + let sql: string; + if (args.sql) { + sql = args.sql; + } else if (args.sqlFile) { + try { + sql = readFileSync(args.sqlFile, "utf-8").trim(); + } catch (error) { + throw new UserError( + `Failed to read SQL file '${args.sqlFile}': ${error instanceof Error ? error.message : String(error)}` + ); + } + } else { + throw new UserError("Either --sql or --sql-file must be provided"); } - if (!destination.credentials.secret_access_key) { - throw new FatalError("Requires a r2 secret access key"); + if (!sql) { + throw new UserError("SQL query cannot be empty"); } - if (args.source.length > 0) { - const sourceHandlers: Record Source> = { - http: (): HttpSource => { - const http: HttpSource = { - type: "http", - format: "json", - authentication: args.requireHttpAuth, - }; - - if (args.corsOrigins && args.corsOrigins.length > 0) { - http.cors = { origins: args.corsOrigins }; - } - - return http; - }, - worker: (): BindingSource => ({ - type: "binding", - format: "json", - }), - }; - - for (const source of args.source) { - const handler = sourceHandlers[source]; - if (handler) { - pipelineConfig.source.push(handler()); + // Validate SQL before creating pipeline + logger.log("🌀 Validating SQL..."); + try { + const validationResult = await validateSql(config, { sql }); + + if ( + validationResult.tables && + Object.keys(validationResult.tables).length > 0 + ) { + const tableNames = Object.keys(validationResult.tables); + logger.log( + `✅ SQL validated successfully. References tables: ${tableNames.join(", ")}` + ); + } else { + logger.log("✅ SQL validated successfully."); + } + } catch (error) { + let errorMessage = "Unknown validation error"; + if (error && typeof error === "object") { + const errorObj = error as { + notes?: Array<{ text?: string }>; + message?: string; + }; + if ( + errorObj.notes && + Array.isArray(errorObj.notes) && + errorObj.notes[0]?.text + ) { + errorMessage = errorObj.notes[0].text; + } else if (error instanceof Error) { + errorMessage = error.message; } } + throw new UserError(`SQL validation failed: ${errorMessage}`); } - if (pipelineConfig.source.length === 0) { - throw new UserError( - "No sources have been enabled. At least one source (HTTP or Worker Binding) should be enabled" - ); - } - - if (args.transformWorker) { - pipelineConfig.transforms.push(parseTransform(args.transformWorker)); - } + const pipelineConfig: CreatePipelineRequest = { + name: pipelineName, + sql, + }; - if (args.r2Prefix) { - pipelineConfig.destination.path.prefix = args.r2Prefix; - } - if (args.shardCount) { - pipelineConfig.metadata.shards = args.shardCount; + logger.log(`🌀 Creating pipeline '${pipelineName}'...`); + + let pipeline; + try { + pipeline = await createPipeline(config, pipelineConfig); + } catch (error) { + if (error instanceof APIError && error.code === 10000) { + // Show error when no access to v1 Pipelines API + throw new UserError( + `Your account does not have access to the new Pipelines API. To use the legacy Pipelines API, please run:\n\nnpx wrangler@4.36.0 pipelines create ${pipelineName}\n\nThis will use an older version of Wrangler that supports the legacy API.` + ); + } + throw error; } - logger.log(`🌀 Creating pipeline named "${name}"`); - const pipeline = await createPipeline(config, accountId, pipelineConfig); - logger.log( - `✅ Successfully created pipeline "${pipeline.name}" with ID ${pipeline.id}\n` + `✨ Successfully created pipeline '${pipeline.name}' with id '${pipeline.id}'.` ); - logger.log(formatPipelinePretty(pipeline)); - logger.log("🎉 You can now send data to your pipeline!"); - if (args.source.includes("worker")) { - await updateConfigFile( - (bindingName) => ({ - pipelines: [ - { - pipeline: pipeline.name, - binding: getValidBindingName( - bindingName ?? "PIPELINE", - "PIPELINE" - ), - }, - ], - }), - config.configPath, - args.env + try { + const fullPipeline = await getPipeline(config, pipeline.id); + + const streamTable = fullPipeline.tables?.find( + (table) => table.type === "stream" ); - } - if (args.source.includes("http")) { - logger.log(`\nSend data to your pipeline's HTTP endpoint:\n`); - logger.log(`curl "${pipeline.endpoint}" -d '[{"foo": "bar"}]'\n`); + if (streamTable) { + const stream = await getStream(config, streamTable.id); + await displayUsageExamples(stream, config, args); + } else { + logger.log( + `\nRun 'wrangler pipelines get ${pipeline.id}' to view full details.` + ); + } + } catch { + logger.warn("Could not fetch pipeline details for usage examples."); + logger.log( + `\nRun 'wrangler pipelines get ${pipeline.id}' to view full details.` + ); } }, }); diff --git a/packages/wrangler/src/pipelines/cli/delete.ts b/packages/wrangler/src/pipelines/cli/delete.ts index 4f4894e95361..0f2fb11031b8 100644 --- a/packages/wrangler/src/pipelines/cli/delete.ts +++ b/packages/wrangler/src/pipelines/cli/delete.ts @@ -1,8 +1,10 @@ import { createCommand } from "../../core/create-command"; +import { confirm } from "../../dialogs"; import { logger } from "../../logger"; +import { APIError } from "../../parse"; import { requireAuth } from "../../user"; -import { deletePipeline } from "../client"; -import { validateName } from "../validate"; +import { deletePipeline, getPipeline } from "../client"; +import { tryDeleteLegacyPipeline } from "./legacy-helpers"; export const pipelinesDeleteCommand = createCommand({ metadata: { @@ -13,20 +15,57 @@ export const pipelinesDeleteCommand = createCommand({ args: { pipeline: { type: "string", - describe: "The name of the pipeline to delete", + describe: "The ID or name of the pipeline to delete", demandOption: true, }, + force: { + describe: "Skip confirmation", + type: "boolean", + alias: "y", + default: false, + }, }, positionalArgs: ["pipeline"], async handler(args, { config }) { const accountId = await requireAuth(config); - const name = args.pipeline; + const pipelineId = args.pipeline; + + try { + const pipeline = await getPipeline(config, pipelineId); + + if (!args.force) { + const confirmedDelete = await confirm( + `Are you sure you want to delete the pipeline '${pipeline.name}' (${pipelineId})?`, + { fallbackValue: false } + ); + if (!confirmedDelete) { + logger.log("Delete cancelled."); + return; + } + } - validateName("pipeline name", name); + await deletePipeline(config, pipelineId); - logger.log(`Deleting pipeline ${name}.`); - await deletePipeline(config, accountId, name); + logger.log( + `✨ Successfully deleted pipeline '${pipeline.name}' with id '${pipeline.id}'.` + ); + } catch (error) { + if ( + error instanceof APIError && + (error.code === 1000 || error.code === 2) + ) { + const deletedFromLegacy = await tryDeleteLegacyPipeline( + config, + accountId, + pipelineId, + args.force + ); - logger.log(`Deleted pipeline ${name}.`); + if (deletedFromLegacy) { + return; + } + } + throw error; + } }, }); diff --git a/packages/wrangler/src/pipelines/cli/get.ts b/packages/wrangler/src/pipelines/cli/get.ts index 89b8d18a5559..af266e3d4ea5 100644 --- a/packages/wrangler/src/pipelines/cli/get.ts +++ b/packages/wrangler/src/pipelines/cli/get.ts @@ -1,44 +1,103 @@ import { createCommand } from "../../core/create-command"; import { logger } from "../../logger"; +import { APIError } from "../../parse"; import { requireAuth } from "../../user"; +import formatLabelledValues from "../../utils/render-labelled-values"; import { getPipeline } from "../client"; -import { formatPipelinePretty } from "../index"; -import { validateName } from "../validate"; +import { tryGetLegacyPipeline } from "./legacy-helpers"; export const pipelinesGetCommand = createCommand({ metadata: { - description: "Get a pipeline's configuration", + description: "Get details about a specific pipeline", owner: "Product: Pipelines", status: "open-beta", }, args: { pipeline: { type: "string", - describe: "The name of the pipeline to inspect", + describe: "The ID of the pipeline to retrieve", demandOption: true, }, - format: { - choices: ["pretty", "json"], - describe: "The output format for pipeline", - default: "pretty", + json: { + describe: "Output in JSON format", + type: "boolean", + default: false, }, }, positionalArgs: ["pipeline"], async handler(args, { config }) { const accountId = await requireAuth(config); - const name = args.pipeline; + const pipelineId = args.pipeline; - validateName("pipeline name", name); + let pipeline; - const pipeline = await getPipeline(config, accountId, name); + try { + pipeline = await getPipeline(config, pipelineId); + } catch (error) { + if ( + error instanceof APIError && + (error.code === 1000 || error.code === 2) + ) { + const foundInLegacy = await tryGetLegacyPipeline( + config, + accountId, + pipelineId, + args.json ? "json" : "pretty" + ); - switch (args.format) { - case "json": - logger.log(JSON.stringify(pipeline, null, 2)); - break; - case "pretty": - logger.log(formatPipelinePretty(pipeline)); - break; + if (foundInLegacy) { + return; + } + } + throw error; + } + + if (args.json) { + logger.json(pipeline); + return; + } + + const general: Record = { + ID: pipeline.id, + Name: pipeline.name, + "Created At": new Date(pipeline.created_at).toLocaleString(), + "Modified At": new Date(pipeline.modified_at).toLocaleString(), + }; + + logger.log("General:"); + logger.log(formatLabelledValues(general, { indentationCount: 2 })); + + logger.log("\nPipeline SQL:"); + logger.log(pipeline.sql); + + if (pipeline.tables && pipeline.tables.length > 0) { + const streams = pipeline.tables.filter( + (table) => table.type === "stream" + ); + const sinks = pipeline.tables.filter((table) => table.type === "sink"); + + if (streams.length > 0) { + logger.log("\nConnected Streams:"); + logger.table( + streams.map((stream) => ({ + Name: stream.name, + ID: stream.id, + })) + ); + } + + if (sinks.length > 0) { + logger.log("\nConnected Sinks:"); + logger.table( + sinks.map((sink) => ({ + Name: sink.name, + ID: sink.id, + })) + ); + } + } else { + logger.log("\nConnected Streams: None"); + logger.log("Connected Sinks: None"); } }, }); diff --git a/packages/wrangler/src/pipelines/cli/legacy-helpers.ts b/packages/wrangler/src/pipelines/cli/legacy-helpers.ts new file mode 100644 index 000000000000..8eab7e01756c --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/legacy-helpers.ts @@ -0,0 +1,275 @@ +import { confirm } from "../../dialogs"; +import { FatalError, UserError } from "../../errors"; +import { logger } from "../../logger"; +import { APIError } from "../../parse"; +import { + authorizeR2Bucket, + BYTES_PER_MB, + getAccountR2Endpoint, +} from "../index"; +import { + deletePipeline, + formatPipelinePretty, + getPipeline, + listPipelines, + updatePipeline, +} from "../legacy-client"; +import { validateName } from "../validate"; +import type { Config } from "../../config"; +import type { BindingSource, HttpSource, Source } from "../legacy-client"; + +export async function listLegacyPipelines( + config: Config, + accountId: string +): Promise { + const list = await listPipelines(config, accountId); + + logger.table( + list.map((pipeline) => ({ + name: pipeline.name, + id: pipeline.id, + endpoint: pipeline.endpoint, + })) + ); +} + +export async function getLegacyPipeline( + config: Config, + accountId: string, + name: string, + format: "pretty" | "json" +): Promise { + validateName("pipeline name", name); + + const pipeline = await getPipeline(config, accountId, name); + + switch (format) { + case "json": + logger.log(JSON.stringify(pipeline, null, 2)); + break; + case "pretty": + logger.warn( + "⚠️ This is a legacy pipeline. Consider creating a new pipeline by running 'wrangler pipelines setup'." + ); + logger.log(formatPipelinePretty(pipeline)); + break; + } +} + +interface LegacyUpdateArgs { + pipeline: string; + compression?: string; + batchMaxMb?: number; + batchMaxSeconds?: number; + batchMaxRows?: number; + r2Bucket?: string; + r2Prefix?: string; + r2AccessKeyId?: string; + r2SecretAccessKey?: string; + source?: (string | number)[] | undefined; + requireHttpAuth?: boolean; + shardCount?: number; + corsOrigins?: string[]; +} + +export async function tryGetLegacyPipeline( + config: Config, + accountId: string, + name: string, + format: "pretty" | "json" +): Promise { + try { + await getLegacyPipeline(config, accountId, name, format); + return true; + } catch (error) { + if (error instanceof APIError && error.code === 1000) { + return false; + } + throw error; + } +} + +export async function tryListLegacyPipelines( + config: Config, + accountId: string +): Promise | null> { + try { + const pipelines = await listPipelines(config, accountId); + return pipelines; + } catch { + return []; + } +} + +export async function tryDeleteLegacyPipeline( + config: Config, + accountId: string, + name: string, + force: boolean = false +): Promise { + try { + await getPipeline(config, accountId, name); + + if (!force) { + const confirmedDelete = await confirm( + `Are you sure you want to delete the legacy pipeline '${name}'?` + ); + if (!confirmedDelete) { + logger.log("Delete cancelled."); + return true; + } + } + + await deletePipeline(config, accountId, name); + logger.log(`✨ Successfully deleted legacy pipeline '${name}'.`); + return true; + } catch (error) { + if ( + error instanceof APIError && + (error.code === 1000 || error.code === 2) + ) { + // Not found in legacy API + return false; + } + throw error; + } +} + +export async function updateLegacyPipeline( + config: Config, + accountId: string, + args: LegacyUpdateArgs +): Promise { + const name = args.pipeline; + + const pipelineConfig = await getPipeline(config, accountId, name); + + logger.warn( + "⚠️ Updating legacy pipeline. Consider recreating with 'wrangler pipelines setup'." + ); + + if (args.compression) { + pipelineConfig.destination.compression.type = args.compression; + } + if (args.batchMaxMb) { + pipelineConfig.destination.batch.max_bytes = args.batchMaxMb * BYTES_PER_MB; // convert to bytes for the API + } + if (args.batchMaxSeconds) { + pipelineConfig.destination.batch.max_duration_s = args.batchMaxSeconds; + } + if (args.batchMaxRows) { + pipelineConfig.destination.batch.max_rows = args.batchMaxRows; + } + + const bucket = args.r2Bucket; + const accessKeyId = args.r2AccessKeyId; + const secretAccessKey = args.r2SecretAccessKey; + if (bucket || accessKeyId || secretAccessKey) { + const destination = pipelineConfig.destination; + if (bucket) { + pipelineConfig.destination.path.bucket = bucket; + } + destination.credentials = { + endpoint: getAccountR2Endpoint(accountId), + access_key_id: accessKeyId || "", + secret_access_key: secretAccessKey || "", + }; + if (!accessKeyId && !secretAccessKey) { + const auth = await authorizeR2Bucket( + config, + name, + accountId, + destination.path.bucket + ); + destination.credentials.access_key_id = auth.accessKeyId; + destination.credentials.secret_access_key = auth.secretAccessKey; + } + if (!destination.credentials.access_key_id) { + throw new FatalError("Requires a r2 access key id"); + } + + if (!destination.credentials.secret_access_key) { + throw new FatalError("Requires a r2 secret access key"); + } + } + + if (args.source && args.source.length > 0) { + const existingSources = pipelineConfig.source; + pipelineConfig.source = []; // Reset the list + + const sourceHandlers: Record Source> = { + http: (): HttpSource => { + const existing = existingSources.find((s: Source) => s.type === "http"); + + return { + ...existing, // Copy over existing properties for forwards compatibility + type: "http", + format: "json", + ...(args.requireHttpAuth && { + authentication: args.requireHttpAuth, + }), // Include only if defined + }; + }, + worker: (): BindingSource => { + const existing = existingSources.find( + (s: Source) => s.type === "binding" + ); + + return { + ...existing, // Copy over existing properties for forwards compatibility + type: "binding", + format: "json", + }; + }, + }; + + for (const source of args.source) { + const handler = sourceHandlers[source]; + if (handler) { + pipelineConfig.source.push(handler()); + } + } + } + + if (pipelineConfig.source.length === 0) { + throw new UserError( + "No sources have been enabled. At least one source (HTTP or Worker Binding) should be enabled" + ); + } + + pipelineConfig.transforms = []; + + if (args.r2Prefix) { + pipelineConfig.destination.path.prefix = args.r2Prefix; + } + + if (args.shardCount) { + pipelineConfig.metadata.shards = args.shardCount; + } + + // This covers the case where `--source` wasn't passed but `--cors-origins` or + // `--require-http-auth` was. + const httpSource = pipelineConfig.source.find( + (s: Source) => s.type === "http" + ); + if (httpSource) { + if (args.requireHttpAuth) { + httpSource.authentication = args.requireHttpAuth; + } + if (args.corsOrigins) { + httpSource.cors = { origins: args.corsOrigins }; + } + } + + logger.log(`🌀 Updating pipeline "${name}"`); + const pipeline = await updatePipeline( + config, + accountId, + name, + pipelineConfig + ); + + logger.log( + `✨ Successfully updated pipeline "${pipeline.name}" with ID ${pipeline.id}\n` + ); +} diff --git a/packages/wrangler/src/pipelines/cli/list.ts b/packages/wrangler/src/pipelines/cli/list.ts index 9446d4821535..3eade0d7f6b6 100644 --- a/packages/wrangler/src/pipelines/cli/list.ts +++ b/packages/wrangler/src/pipelines/cli/list.ts @@ -2,6 +2,7 @@ import { createCommand } from "../../core/create-command"; import { logger } from "../../logger"; import { requireAuth } from "../../user"; import { listPipelines } from "../client"; +import { tryListLegacyPipelines } from "./legacy-helpers"; export const pipelinesListCommand = createCommand({ metadata: { @@ -9,19 +10,86 @@ export const pipelinesListCommand = createCommand({ owner: "Product: Pipelines", status: "open-beta", }, - - async handler(_, { config }) { + args: { + page: { + describe: "Page number for pagination", + type: "number", + default: 1, + }, + "per-page": { + describe: "Number of pipelines per page", + type: "number", + default: 20, + }, + json: { + describe: "Output in JSON format", + type: "boolean", + default: false, + }, + }, + async handler(args, { config }) { const accountId = await requireAuth(config); - // TODO: we should show bindings & transforms if they exist for given ids - const list = await listPipelines(config, accountId); + const [newPipelines, legacyPipelines] = await Promise.all([ + listPipelines(config, { + page: args.page, + per_page: args.perPage, + }), + tryListLegacyPipelines(config, accountId), + ]); + + if (args.json) { + const hasLegacyPipelines = legacyPipelines && legacyPipelines.length > 0; + const result = hasLegacyPipelines + ? { + pipelines: newPipelines || [], + legacyPipelines: legacyPipelines, + } + : newPipelines || []; + logger.json(result); + return; + } + + const hasLegacyPipelines = legacyPipelines && legacyPipelines.length > 0; + const hasNewPipelines = newPipelines && newPipelines.length > 0; + + if (!hasNewPipelines && !hasLegacyPipelines) { + logger.log("No pipelines found."); + return; + } + + if (hasLegacyPipelines) { + const tableData = [ + ...(newPipelines || []).map((pipeline) => ({ + Name: pipeline.name, + ID: pipeline.id, + Created: new Date(pipeline.created_at).toLocaleDateString(), + Modified: new Date(pipeline.modified_at).toLocaleDateString(), + Type: "", + })), + ...legacyPipelines.map((pipeline) => ({ + Name: pipeline.name, + ID: pipeline.id, + Created: "N/A", + Modified: "N/A", + Type: "Legacy", + })), + ]; + logger.table(tableData); + } else { + const tableData = (newPipelines || []).map((pipeline) => ({ + Name: pipeline.name, + ID: pipeline.id, + Created: new Date(pipeline.created_at).toLocaleDateString(), + Modified: new Date(pipeline.modified_at).toLocaleDateString(), + })); + logger.table(tableData); + } - logger.table( - list.map((pipeline) => ({ - name: pipeline.name, - id: pipeline.id, - endpoint: pipeline.endpoint, - })) - ); + if (hasLegacyPipelines) { + logger.warn( + "⚠️ You have legacy pipelines. Consider creating new pipelines by running 'wrangler pipelines setup'." + ); + } }, }); diff --git a/packages/wrangler/src/pipelines/cli/setup.ts b/packages/wrangler/src/pipelines/cli/setup.ts new file mode 100644 index 000000000000..ce04bf5bb1cd --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/setup.ts @@ -0,0 +1,810 @@ +import { readFileSync } from "node:fs"; +import { createCommand } from "../../core/create-command"; +import { confirm, prompt, select } from "../../dialogs"; +import { UserError } from "../../errors"; +import { logger } from "../../logger"; +import { parseJSON } from "../../parse"; +import { bucketFormatMessage, isValidR2BucketName } from "../../r2/helpers"; +import { requireAuth } from "../../user"; +import { + createPipeline, + createSink, + createStream, + deleteSink, + deleteStream, + validateSql, +} from "../client"; +import { authorizeR2Bucket } from "../index"; +import { + displayUsageExamples, + formatSchemaFieldsForTable, +} from "./streams/utils"; +import type { Config } from "../../config"; +import type { + CreatePipelineRequest, + CreateSinkRequest, + CreateStreamRequest, + ParquetFormat, + SchemaField, + Sink, + SinkFormat, + Stream, +} from "../types"; + +interface SetupConfig { + pipelineName: string; + streamName: string; + sinkName: string; + streamConfig: CreateStreamRequest; + sinkConfig: CreateSinkRequest; + pipelineConfig?: CreatePipelineRequest; +} + +export const pipelinesSetupCommand = createCommand({ + metadata: { + description: "Interactive setup for a complete pipeline", + owner: "Product: Pipelines", + status: "open-beta", + }, + args: { + name: { + describe: "Pipeline name", + type: "string", + }, + }, + async handler(args, { config }) { + await requireAuth(config); + + logger.log("🚀 Welcome to Cloudflare Pipelines Setup!"); + logger.log( + "This will guide you through creating a complete pipeline: stream → pipeline → sink\n" + ); + + try { + const setupConfig = await setupPipelineNaming(args.name); + + await setupStreamConfiguration(setupConfig); + await setupSinkConfiguration(config, setupConfig); + const created = await reviewAndCreateStreamSink(config, setupConfig); + await setupSQLTransformationWithValidation(config, setupConfig, created); + await createPipelineIfNeeded(config, setupConfig, created, args); + } catch (error) { + if (error instanceof UserError) { + throw error; + } + throw new UserError( + `Setup failed: ${error instanceof Error ? error.message : String(error)}` + ); + } + }, +}); + +async function setupPipelineNaming( + providedName?: string +): Promise { + const pipelineName = + providedName || + (await prompt("What would you like to name your pipeline?")); + + if (!pipelineName) { + throw new UserError("Pipeline name is required"); + } + + if (!/^[a-zA-Z0-9_-]+$/.test(pipelineName)) { + throw new UserError( + "Pipeline name must contain only letters, numbers, hyphens, and underscores" + ); + } + + const streamName = `${pipelineName}_stream`; + const sinkName = `${pipelineName}_sink`; + + return { + pipelineName, + streamName, + sinkName, + streamConfig: { + name: streamName, + http: { enabled: true, authentication: false }, + worker_binding: { enabled: true }, + }, + sinkConfig: { + name: sinkName, + type: "r2", + config: { bucket: "" }, + }, + }; +} + +async function setupStreamConfiguration( + setupConfig: SetupConfig +): Promise { + logger.log("\n▶ Let's configure your data source (stream):"); + + const httpEnabled = await confirm("Enable HTTP endpoint for sending data?", { + defaultValue: true, + }); + + let httpAuth = false; + if (httpEnabled) { + httpAuth = await confirm("Require authentication for HTTP endpoint?", { + defaultValue: false, + }); + } + + let corsOrigins: string[] | undefined; + if (httpEnabled) { + const customCors = await confirm("Configure custom CORS origins?", { + defaultValue: false, + }); + if (customCors) { + const origins = await prompt( + "CORS origins (comma-separated, or * for all):" + ); + corsOrigins = + origins === "*" ? ["*"] : origins.split(",").map((o) => o.trim()); + } + } + + const schema = await setupSchemaConfiguration(); + + setupConfig.streamConfig = { + name: setupConfig.streamName, + format: { type: "json" as const, ...(!schema && { unstructured: true }) }, + http: { + enabled: httpEnabled, + authentication: httpAuth, + ...(httpEnabled && corsOrigins && { cors: { origins: corsOrigins } }), + }, + worker_binding: { enabled: true }, + ...(schema && { schema: { fields: schema } }), + }; + + logger.log("✨ Stream configuration complete\n"); +} + +async function setupSchemaConfiguration(): Promise { + const schemaMethod = await select( + "How would you like to define the schema?", + { + choices: [ + { title: "Build interactively", value: "interactive" }, + { title: "Load from file", value: "file" }, + { title: "Skip (unstructured)", value: "skip" }, + ], + defaultOption: 0, + fallbackOption: 2, + } + ); + + switch (schemaMethod) { + case "interactive": + return buildSchemaInteractively(); + case "file": + return loadSchemaFromFile(); + case "skip": + return undefined; + default: + return undefined; + } +} + +async function buildSchemaInteractively(): Promise { + const fields: SchemaField[] = []; + let fieldNumber = 1; + + logger.log("\n▶ Building schema interactively:"); + + let continueAdding = true; + while (continueAdding) { + const field = await buildField(fieldNumber); + fields.push(field); + + const addAnother = await confirm(`Add field #${fieldNumber + 1}?`, { + defaultValue: fieldNumber < 3, + }); + + if (!addAnother) { + continueAdding = false; + } else { + fieldNumber++; + } + } + + return fields; +} + +async function buildField( + fieldNumber: number, + depth = 0 +): Promise { + const indent = " ".repeat(depth); + logger.log(`${indent}Field #${fieldNumber}:`); + + const name = await prompt(`${indent} Name:`); + + if (!name) { + throw new UserError("Field name is required"); + } + + const typeChoices = [ + { title: "string", value: "string" }, + { title: "int32", value: "int32" }, + { title: "int64", value: "int64" }, + { title: "u_int32", value: "u_int32" }, + { title: "u_int64", value: "u_int64" }, + { title: "f32", value: "f32" }, + { title: "f64", value: "f64" }, + { title: "bool", value: "bool" }, + { title: "timestamp", value: "timestamp" }, + { title: "json", value: "json" }, + { title: "bytes", value: "bytes" }, + ]; + + // Only show complex types if not nested too deep + if (depth < 2) { + typeChoices.push( + { title: "struct (nested object)", value: "struct" }, + { title: "list (array)", value: "list" } + ); + } + + const type = await select(`${indent} Type:`, { + choices: typeChoices, + defaultOption: 0, + fallbackOption: 0, + }); + + const required = await confirm(`${indent} Required?`, { + defaultValue: true, + }); + + const field: SchemaField = { + name, + type: type as SchemaField["type"], + required, + }; + + // Handle type-specific configuration + if (type === "timestamp") { + const unit = await select(`${indent} Unit:`, { + choices: [ + { title: "millisecond", value: "millisecond" }, + { title: "second", value: "second" }, + { title: "microsecond", value: "microsecond" }, + { title: "nanosecond", value: "nanosecond" }, + ], + defaultOption: 0, + fallbackOption: 0, + }); + field.unit = unit; + } else if (type === "struct" && depth < 2) { + logger.log(`\nDefine nested fields for struct '${name}':`); + field.fields = []; + let structFieldNumber = 1; + + let continueAdding = true; + while (continueAdding) { + const structField = await buildField(structFieldNumber, depth + 1); + field.fields.push(structField); + + const addAnother = await confirm( + `${indent}Add another field to struct '${name}'?`, + { defaultValue: false } + ); + + if (!addAnother) { + continueAdding = false; + } else { + structFieldNumber++; + } + } + } else if (type === "list" && depth < 2) { + logger.log(`\nDefine item type for list '${name}':`); + field.items = await buildField(1, depth + 1); + } + + return field; +} + +async function loadSchemaFromFile(): Promise { + const filePath = await prompt("Schema file path:"); + + try { + const schemaContent = readFileSync(filePath, "utf-8"); + const parsedSchema = parseJSON(schemaContent, filePath) as { + fields: SchemaField[]; + }; + + if (!parsedSchema || !Array.isArray(parsedSchema.fields)) { + throw new UserError("Schema file must contain a 'fields' array"); + } + + return parsedSchema.fields; + } catch (error) { + logger.error( + `Failed to read schema file: ${error instanceof Error ? error.message : String(error)}` + ); + + const retry = await confirm("Would you like to try again?", { + defaultValue: true, + }); + + if (retry) { + return loadSchemaFromFile(); + } else { + throw new UserError("Schema file loading cancelled"); + } + } +} + +async function setupSinkConfiguration( + config: Config, + setupConfig: SetupConfig +): Promise { + logger.log("▶ Let's configure your destination (sink):"); + + const sinkType = await select("Select destination type:", { + choices: [ + { title: "R2 Bucket", value: "r2" }, + { title: "Data Catalog Table", value: "r2_data_catalog" }, + ], + defaultOption: 0, + fallbackOption: 0, + }); + + const accountId = await requireAuth(config); + + if (sinkType === "r2") { + await setupR2Sink(config, accountId, setupConfig); + } else { + await setupDataCatalogSink(setupConfig); + } + + logger.log("✨ Sink configuration complete\n"); +} + +async function setupR2Sink( + config: Config, + accountId: string, + setupConfig: SetupConfig +): Promise { + const bucket = await prompt("R2 bucket name:"); + + if (!bucket) { + throw new UserError("Bucket name is required"); + } + + if (!isValidR2BucketName(bucket)) { + throw new UserError( + `The bucket name "${bucket}" is invalid. ${bucketFormatMessage}` + ); + } + + const path = await prompt( + "The base prefix in your bucket where data will be written (optional):", + { + defaultValue: "", + } + ); + + const timePartitionPattern = await prompt( + "Time partition pattern (optional):", + { + defaultValue: "year=%Y/month=%m/day=%d", + } + ); + + const format = await select("Output format:", { + choices: [ + { title: "Parquet (recommended for analytics)", value: "parquet" }, + { title: "JSON", value: "json" }, + ], + defaultOption: 0, + fallbackOption: 0, + }); + + let compression; + if (format === "parquet") { + compression = await select("Compression:", { + choices: [ + { title: "uncompressed", value: "uncompressed" }, + { title: "snappy", value: "snappy" }, + { title: "gzip", value: "gzip" }, + { title: "zstd", value: "zstd" }, + { title: "lz4", value: "lz4" }, + ], + defaultOption: 3, + fallbackOption: 3, + }); + } + + const fileSizeMB = await prompt("Roll file when size reaches (MB):", { + defaultValue: "100", + }); + const intervalSeconds = await prompt( + "Roll file when time reaches (seconds):", + { + defaultValue: "300", + } + ); + + const useOAuth = await confirm( + "Automatically generate credentials needed to write to your R2 bucket?", + { + defaultValue: true, + } + ); + + let credentials; + if (useOAuth) { + logger.log("🔐 Generating R2 credentials..."); + // Clean up sink name for service token generation (remove underscores) + const cleanedSinkName = setupConfig.sinkName.replace(/_/g, "-"); + const auth = await authorizeR2Bucket( + config, + cleanedSinkName, + accountId, + bucket + ); + credentials = { + access_key_id: auth.accessKeyId, + secret_access_key: auth.secretAccessKey, + }; + } else { + credentials = { + access_key_id: await prompt("R2 Access Key ID:"), + secret_access_key: await prompt("R2 Secret Access Key:", { + isSecret: true, + }), + }; + } + + let formatConfig: SinkFormat; + if (format === "json") { + formatConfig = { type: "json" }; + } else { + formatConfig = { + type: "parquet", + ...(compression && { + compression: compression as ParquetFormat["compression"], + }), + }; + } + + setupConfig.sinkConfig = { + name: setupConfig.sinkName, + type: "r2", + format: formatConfig, + config: { + bucket, + ...(path && { path }), + ...(timePartitionPattern && { + partitioning: { + time_pattern: timePartitionPattern, + }, + }), + credentials, + rolling_policy: { + file_size_bytes: parseInt(fileSizeMB) * 1024 * 1024, // Convert MB to bytes + interval_seconds: parseInt(intervalSeconds), + }, + }, + }; +} + +async function setupDataCatalogSink(setupConfig: SetupConfig): Promise { + const bucket = await prompt("R2 bucket name (for catalog storage):"); + const namespace = await prompt("Namespace:", { defaultValue: "default" }); + const tableName = await prompt("Table name:"); + const token = await prompt("Catalog API token:", { isSecret: true }); + + if (!bucket || !namespace || !tableName || !token) { + throw new UserError("All Data Catalog fields are required"); + } + + const compression = await select("Compression:", { + choices: [ + { title: "uncompressed", value: "uncompressed" }, + { title: "snappy", value: "snappy" }, + { title: "gzip", value: "gzip" }, + { title: "zstd", value: "zstd" }, + { title: "lz4", value: "lz4" }, + ], + defaultOption: 0, + fallbackOption: 0, + }); + + const fileSizeMB = await prompt("Roll file when size reaches (MB):", { + defaultValue: "100", + }); + const intervalSeconds = await prompt( + "Roll file when time reaches (seconds):", + { + defaultValue: "300", + } + ); + + setupConfig.sinkConfig = { + name: setupConfig.sinkName, + type: "r2_data_catalog", + format: { + type: "parquet", + compression: compression as ParquetFormat["compression"], + }, + config: { + bucket, + namespace, + table_name: tableName, + token, + rolling_policy: { + file_size_bytes: parseInt(fileSizeMB) * 1024 * 1024, + interval_seconds: parseInt(intervalSeconds), + }, + }, + }; +} + +async function setupSQLTransformationWithValidation( + config: Config, + setupConfig: SetupConfig, + created: { stream?: Stream; sink?: Sink } +): Promise { + logger.log("\n▶ Pipeline SQL:"); + + logger.log("\nAvailable tables:"); + logger.log(` • ${setupConfig.streamName} (source stream)`); + logger.log(` • ${setupConfig.sinkName} (destination sink)`); + + if (setupConfig.streamConfig.schema?.fields) { + logger.log("\nStream input schema:"); + const schemaRows = formatSchemaFieldsForTable( + setupConfig.streamConfig.schema.fields + ); + logger.table(schemaRows); + } + + const sqlMethod = await select( + "How would you like to provide SQL that will define how your pipeline will transform and route data?", + { + choices: [ + { + title: + "Use simple ingestion query (copy all data from stream to sink)", + value: "simple", + }, + { title: "Write interactively", value: "interactive" }, + { title: "Load from file", value: "file" }, + ], + defaultOption: 0, + fallbackOption: 0, + } + ); + + let sql: string; + + if (sqlMethod === "simple") { + sql = `INSERT INTO ${setupConfig.sinkName} SELECT * FROM ${setupConfig.streamName};`; + logger.log(`\nUsing query: ${sql}`); + } else if (sqlMethod === "interactive") { + logger.log( + `\n💡 Example: INSERT INTO ${setupConfig.sinkName} SELECT * FROM ${setupConfig.streamName};` + ); + + sql = await promptMultiline("Enter SQL query:", "SQL"); + } else { + const filePath = await prompt("SQL file path:"); + try { + sql = readFileSync(filePath, "utf-8").trim(); + } catch (error) { + throw new UserError( + `Failed to read SQL file: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + if (!sql) { + throw new UserError("SQL query cannot be empty"); + } + + logger.log("🌀 Validating SQL..."); + try { + const validationResult = await validateSql(config, { sql }); + + if ( + !validationResult.tables || + Object.keys(validationResult.tables).length === 0 + ) { + logger.warn( + "SQL validation returned no tables - this might indicate an issue with the query" + ); + } else { + const tableNames = Object.keys(validationResult.tables); + logger.log( + `✨ SQL validation passed. References tables: ${tableNames.join(", ")}` + ); + } + + setupConfig.pipelineConfig = { + name: setupConfig.pipelineName, + sql, + }; + } catch (error) { + let errorMessage = "SQL validation failed"; + + if (error && typeof error === "object") { + const errorObj = error as { + notes?: Array<{ text?: string }>; + message?: string; + }; + + if ( + errorObj.notes && + Array.isArray(errorObj.notes) && + errorObj.notes[0]?.text + ) { + errorMessage = errorObj.notes[0].text; + } + } + + const retry = await confirm( + `SQL validation failed: ${errorMessage}\n\nRetry with different SQL?`, + { defaultValue: true } + ); + + if (retry) { + return setupSQLTransformationWithValidation(config, setupConfig, created); + } else { + throw new UserError( + "SQL validation failed and setup cannot continue without valid pipeline SQL" + ); + } + } + + logger.log("✨ SQL configuration complete\n"); +} + +async function promptMultiline( + message: string, + promptPrefix: string = "INPUT" +): Promise { + logger.log(message); + logger.log("(Press Enter to finish each line, empty line to complete)\n"); + + const lines: string[] = []; + let line: string; + + do { + line = await prompt(lines.length === 0 ? `${promptPrefix}> ` : `...> `); + if (line.trim()) { + lines.push(line); + } + } while (line.trim() !== ""); + + return lines.join(" "); +} + +async function reviewAndCreateStreamSink( + config: Config, + setupConfig: SetupConfig +): Promise<{ stream?: Stream; sink?: Sink }> { + // Display summary + logger.log("▶ Configuration Summary:"); + logger.log(`\nStream: ${setupConfig.streamName}`); + logger.log( + ` • HTTP: ${setupConfig.streamConfig.http.enabled ? "Enabled" : "Disabled"}` + ); + if (setupConfig.streamConfig.http.enabled) { + logger.log( + ` • Authentication: ${setupConfig.streamConfig.http.authentication ? "Required" : "None"}` + ); + } + logger.log( + ` • Schema: ${setupConfig.streamConfig.schema?.fields ? `${setupConfig.streamConfig.schema.fields.length} fields` : "Unstructured"}` + ); + + logger.log(`\nSink: ${setupConfig.sinkName}`); + logger.log( + ` • Type: ${setupConfig.sinkConfig.type === "r2" ? "R2 Bucket" : "Data Catalog"}` + ); + if (setupConfig.sinkConfig.type === "r2") { + logger.log(` • Bucket: ${setupConfig.sinkConfig.config.bucket}`); + logger.log( + ` • Format: ${setupConfig.sinkConfig.format?.type || "parquet"}` + ); + } else { + logger.log( + ` • Table: ${setupConfig.sinkConfig.config.namespace}/${setupConfig.sinkConfig.config.table_name}` + ); + } + + const proceed = await confirm("Create stream and sink?", { + defaultValue: true, + }); + + if (!proceed) { + throw new UserError("Setup cancelled"); + } + + // Create resources with rollback on failure + const created: { stream?: Stream; sink?: Sink } = {}; + + try { + // Create stream + logger.log("\n🌀 Creating stream..."); + created.stream = await createStream(config, setupConfig.streamConfig); + logger.log(`✨ Created stream: ${created.stream.name}`); + + // Create sink + logger.log("🌀 Creating sink..."); + created.sink = await createSink(config, setupConfig.sinkConfig); + logger.log(`✨ Created sink: ${created.sink.name}`); + + logger.log("\n✨ Stream and sink created successfully!"); + return created; + } catch (error) { + logger.error( + `❌ Setup failed: ${error instanceof Error ? error.message : String(error)}` + ); + + logger.log("🌀 Rolling back created resources..."); + + if (created.stream) { + try { + await deleteStream(config, created.stream.id); + logger.log(`✨ Cleaned up stream: ${created.stream.name}`); + } catch (cleanupError) { + logger.warn( + `Failed to cleanup stream: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}` + ); + } + } + + if (created.sink) { + try { + await deleteSink(config, created.sink.id); + logger.log(`✨ Cleaned up sink: ${created.sink.name}`); + } catch (cleanupError) { + logger.warn( + `Failed to cleanup sink: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}` + ); + } + } + + throw error; + } +} + +async function createPipelineIfNeeded( + config: Config, + setupConfig: SetupConfig, + created: { stream?: Stream; sink?: Sink }, + args: { env?: string } +): Promise { + if (!setupConfig.pipelineConfig) { + throw new UserError("Pipeline configuration is missing"); + } + + try { + logger.log("🌀 Creating pipeline..."); + const pipeline = await createPipeline(config, setupConfig.pipelineConfig); + logger.log(`✨ Created pipeline: ${pipeline.name}`); + + logger.log("\n✨ Setup complete!"); + + if (created.stream) { + await displayUsageExamples(created.stream, config, args); + } + } catch (error) { + logger.error( + `❌ Pipeline creation failed: ${error instanceof Error ? error.message : String(error)}` + ); + logger.log( + "\n⚠️ Stream and sink were created successfully, but pipeline creation failed." + ); + logger.log( + "You can try creating the pipeline manually with: wrangler pipelines create" + ); + throw error; + } +} diff --git a/packages/wrangler/src/pipelines/cli/sinks/create.ts b/packages/wrangler/src/pipelines/cli/sinks/create.ts new file mode 100644 index 000000000000..a445cf21a509 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/sinks/create.ts @@ -0,0 +1,271 @@ +import { createCommand } from "../../../core/create-command"; +import { CommandLineArgsError, UserError } from "../../../errors"; +import { logger } from "../../../logger"; +import { bucketFormatMessage, isValidR2BucketName } from "../../../r2/helpers"; +import { requireAuth } from "../../../user"; +import { createSink } from "../../client"; +import { applyDefaultsToSink, SINK_DEFAULTS } from "../../defaults"; +import { authorizeR2Bucket } from "../../index"; +import { displaySinkConfiguration } from "./utils"; +import type { CreateSinkRequest, SinkFormat } from "../../types"; + +function parseSinkType(type: string): "r2" | "r2_data_catalog" { + if (type === "r2" || type === "r2-data-catalog") { + return type === "r2-data-catalog" ? "r2_data_catalog" : "r2"; + } + throw new UserError( + `Invalid sink type: ${type}. Must be 'r2' or 'r2-data-catalog'` + ); +} + +export const pipelinesSinksCreateCommand = createCommand({ + metadata: { + description: "Create a new sink", + owner: "Product: Pipelines", + status: "open-beta", + }, + positionalArgs: ["sink"], + args: { + sink: { + describe: "The name of the sink to create", + type: "string", + demandOption: true, + }, + type: { + describe: "The type of sink to create", + type: "string", + choices: ["r2", "r2-data-catalog"], + demandOption: true, + }, + bucket: { + describe: "R2 bucket name", + type: "string", + demandOption: true, + }, + format: { + describe: "Output format", + type: "string", + choices: ["json", "parquet"], + default: SINK_DEFAULTS.format.type, + }, + compression: { + describe: "Compression method (parquet only)", + type: "string", + choices: ["uncompressed", "snappy", "gzip", "zstd", "lz4"], + default: SINK_DEFAULTS.format.compression, + }, + "target-row-group-size": { + describe: "Target row group size for parquet format", + type: "string", + }, + path: { + describe: "The base prefix in your bucket where data will be written", + type: "string", + }, + partitioning: { + describe: "Time partition pattern (r2 sinks only)", + type: "string", + }, + "roll-size": { + describe: "Roll file size in MB", + type: "number", + default: SINK_DEFAULTS.rolling_policy.file_size_bytes / (1024 * 1024), + }, + "roll-interval": { + describe: "Roll file interval in seconds", + type: "number", + default: SINK_DEFAULTS.rolling_policy.interval_seconds, + }, + "access-key-id": { + describe: + "R2 access key ID (leave empty for R2 credentials to be automatically created)", + type: "string", + implies: "secret-access-key", + }, + "secret-access-key": { + describe: + "R2 secret access key (leave empty for R2 credentials to be automatically created)", + type: "string", + implies: "access-key-id", + }, + namespace: { + describe: "Data catalog namespace (required for r2-data-catalog)", + type: "string", + }, + table: { + describe: "Table name within namespace (required for r2-data-catalog)", + type: "string", + }, + "catalog-token": { + describe: + "Authentication token for data catalog (required for r2-data-catalog)", + type: "string", + }, + }, + validateArgs: (args) => { + const sinkType = parseSinkType(args.type); + + if (!isValidR2BucketName(args.bucket)) { + throw new CommandLineArgsError( + `The bucket name "${args.bucket}" is invalid. ${bucketFormatMessage}` + ); + } + + if (sinkType === "r2_data_catalog") { + if (!args.namespace) { + throw new CommandLineArgsError( + "--namespace is required for r2-data-catalog sinks" + ); + } + if (!args.table) { + throw new CommandLineArgsError( + "--table is required for r2-data-catalog sinks" + ); + } + if (!args.catalogToken) { + throw new CommandLineArgsError( + "--catalog-token is required for r2-data-catalog sinks" + ); + } + if (args.format === "json") { + throw new CommandLineArgsError( + "r2-data-catalog sinks only support parquet format, not JSON" + ); + } + } + }, + async handler(args, { config }) { + const accountId = await requireAuth(config); + const sinkName = args.sink; + const sinkType = parseSinkType(args.type); + + const sinkConfig: CreateSinkRequest = { + name: sinkName, + type: sinkType, + config: { + bucket: args.bucket, + }, + }; + + if (args.format || args.compression || args.targetRowGroupSize) { + let formatConfig: SinkFormat; + if (args.format === "json") { + formatConfig = { type: "json" }; + } else { + formatConfig = { + type: "parquet", + ...(args.compression && { + compression: args.compression as + | "uncompressed" + | "snappy" + | "gzip" + | "zstd" + | "lz4", + }), + ...(args.targetRowGroupSize && { + row_group_bytes: + parseInt(args.targetRowGroupSize.replace(/MB$/i, "")) * + 1024 * + 1024, + }), + }; + } + sinkConfig.format = formatConfig; + } + + if (args.path) { + sinkConfig.config.path = args.path; + } + if (args.partitioning && sinkType === "r2") { + sinkConfig.config.partitioning = { + time_pattern: args.partitioning, + }; + } + + if (args.rollSize || args.rollInterval) { + let file_size_bytes: number = + SINK_DEFAULTS.rolling_policy.file_size_bytes; + let interval_seconds: number = + SINK_DEFAULTS.rolling_policy.interval_seconds; + + if (args.rollSize) { + file_size_bytes = args.rollSize * 1024 * 1024; + } + if (args.rollInterval) { + interval_seconds = args.rollInterval; + } + + sinkConfig.config.rolling_policy = { + file_size_bytes, + interval_seconds, + }; + } + + if (sinkType === "r2") { + if (args.accessKeyId && args.secretAccessKey) { + // Use provided credentials + sinkConfig.config.credentials = { + access_key_id: args.accessKeyId, + secret_access_key: args.secretAccessKey, + }; + } else { + // Use OAuth flow to generate credentials + const auth = await authorizeR2Bucket( + config, + sinkName, + accountId, + args.bucket + ); + sinkConfig.config.credentials = { + access_key_id: auth.accessKeyId, + secret_access_key: auth.secretAccessKey, + }; + } + } + + if (sinkType === "r2_data_catalog") { + sinkConfig.config.namespace = args.namespace; + sinkConfig.config.table_name = args.table; + sinkConfig.config.token = args.catalogToken; + } + + logger.log(`🌀 Creating sink '${sinkName}'...`); + + let sink; + try { + sink = await createSink(config, sinkConfig); + } catch (error) { + // Extract user-friendly error message + let errorMessage = "Unknown error occurred"; + if (error && typeof error === "object") { + const errorObj = error as { + notes?: Array<{ text?: string }>; + message?: string; + }; + if ( + errorObj.notes && + Array.isArray(errorObj.notes) && + errorObj.notes[0]?.text + ) { + errorMessage = errorObj.notes[0].text; + } else if (error instanceof Error) { + errorMessage = error.message; + } + } + + throw new UserError( + `Failed to create sink '${sinkName}': ${errorMessage}` + ); + } + + logger.log( + `✨ Successfully created sink '${sink.name}' with id '${sink.id}'.` + ); + + const sinkWithDefaults = applyDefaultsToSink(sink); + + displaySinkConfiguration(sinkWithDefaults, "Creation Summary", { + includeTimestamps: false, + }); + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/sinks/delete.ts b/packages/wrangler/src/pipelines/cli/sinks/delete.ts new file mode 100644 index 000000000000..f635f83a72c1 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/sinks/delete.ts @@ -0,0 +1,49 @@ +import { createCommand } from "../../../core/create-command"; +import { confirm } from "../../../dialogs"; +import { logger } from "../../../logger"; +import { requireAuth } from "../../../user"; +import { deleteSink, getSink } from "../../client"; + +export const pipelinesSinksDeleteCommand = createCommand({ + metadata: { + description: "Delete a sink", + owner: "Product: Pipelines", + status: "open-beta", + }, + positionalArgs: ["sink"], + args: { + sink: { + describe: "The ID of the sink to delete", + type: "string", + demandOption: true, + }, + force: { + describe: "Skip confirmation", + type: "boolean", + alias: "y", + default: false, + }, + }, + async handler(args, { config }) { + await requireAuth(config); + + const sink = await getSink(config, args.sink); + + if (!args.force) { + const confirmedDelete = await confirm( + `Are you sure you want to delete the sink '${sink.name}' (${args.sink})?`, + { fallbackValue: false } + ); + if (!confirmedDelete) { + logger.log("Delete cancelled."); + return; + } + } + + await deleteSink(config, args.sink); + + logger.log( + `✨ Successfully deleted sink '${sink.name}' with id '${sink.id}'.` + ); + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/sinks/get.ts b/packages/wrangler/src/pipelines/cli/sinks/get.ts new file mode 100644 index 000000000000..af6512e06d21 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/sinks/get.ts @@ -0,0 +1,49 @@ +import { createCommand } from "../../../core/create-command"; +import { logger } from "../../../logger"; +import { requireAuth } from "../../../user"; +import formatLabelledValues from "../../../utils/render-labelled-values"; +import { getSink } from "../../client"; +import { applyDefaultsToSink } from "../../defaults"; +import { displaySinkConfiguration } from "./utils"; + +export const pipelinesSinksGetCommand = createCommand({ + metadata: { + description: "Get details about a specific sink", + owner: "Product: Pipelines", + status: "open-beta", + }, + positionalArgs: ["sink"], + args: { + sink: { + describe: "The ID of the sink to retrieve", + type: "string", + demandOption: true, + }, + json: { + describe: "Output in JSON format", + type: "boolean", + default: false, + }, + }, + async handler(args, { config }) { + await requireAuth(config); + + const rawSink = await getSink(config, args.sink); + + if (args.json) { + logger.json(rawSink); + return; + } + + const sink = applyDefaultsToSink(rawSink); + + logger.log( + formatLabelledValues({ + ID: sink.id, + Name: sink.name, + }) + ); + + displaySinkConfiguration(sink, "", { includeTimestamps: true }); + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/sinks/index.ts b/packages/wrangler/src/pipelines/cli/sinks/index.ts new file mode 100644 index 000000000000..856e86911044 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/sinks/index.ts @@ -0,0 +1,9 @@ +import { createNamespace } from "../../../core/create-command"; + +export const pipelinesSinksNamespace = createNamespace({ + metadata: { + description: "Manage sinks for pipelines", + owner: "Product: Pipelines", + status: "open-beta", + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/sinks/list.ts b/packages/wrangler/src/pipelines/cli/sinks/list.ts new file mode 100644 index 000000000000..59a446c7576f --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/sinks/list.ts @@ -0,0 +1,67 @@ +import { createCommand } from "../../../core/create-command"; +import { logger } from "../../../logger"; +import { requireAuth } from "../../../user"; +import { listSinks } from "../../client"; + +export const pipelinesSinksListCommand = createCommand({ + metadata: { + description: "List all sinks", + owner: "Product: Pipelines", + status: "open-beta", + }, + args: { + page: { + describe: "Page number for pagination", + type: "number", + default: 1, + }, + "per-page": { + describe: "Number of sinks per page", + type: "number", + default: 20, + }, + "pipeline-id": { + describe: "Filter sinks by pipeline ID", + type: "string", + }, + json: { + describe: "Output in JSON format", + type: "boolean", + default: false, + }, + }, + async handler(args, { config }) { + await requireAuth(config); + + const sinks = await listSinks(config, { + page: args.page, + per_page: args.perPage, + pipeline_id: args.pipelineId, + }); + + if (args.json) { + logger.log(sinks); + return; + } + + if (!sinks || sinks.length === 0) { + logger.log("No sinks found."); + return; + } + + logger.table( + sinks.map((sink) => ({ + Name: sink.name, + ID: sink.id, + Type: sink.type === "r2" ? "R2" : "R2 Data Catalog", + Destination: + sink.type === "r2" + ? sink.config.bucket + : `${sink.config.namespace}.${sink.config.table_name}`, + Created: sink.created_at + ? new Date(sink.created_at).toLocaleDateString() + : "N/A", + })) + ); + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/sinks/utils.ts b/packages/wrangler/src/pipelines/cli/sinks/utils.ts new file mode 100644 index 000000000000..5a2690d7b778 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/sinks/utils.ts @@ -0,0 +1,105 @@ +import { logger } from "../../../logger"; +import formatLabelledValues from "../../../utils/render-labelled-values"; +import { SINK_DEFAULTS } from "../../defaults"; +import type { Sink } from "../../types"; + +export function displaySinkConfiguration( + sink: Sink, + title: string = "Configuration", + options: { includeTimestamps?: boolean } = {} +) { + const { includeTimestamps = true } = options; + + if (title) { + logger.log(`\n${title}:`); + } + + const general: Record = { + Type: sink.type === "r2" ? "R2" : "R2 Data Catalog", + }; + + if (includeTimestamps) { + if (sink.created_at) { + general["Created At"] = new Date(sink.created_at).toLocaleString(); + } + if (sink.modified_at) { + general["Modified At"] = new Date(sink.modified_at).toLocaleString(); + } + } + + const destination: Record = { + Bucket: sink.config.bucket, + }; + + if (sink.type === "r2") { + destination.Path = sink.config.path || "(root)"; + destination.Partitioning = + sink.config.partitioning?.time_pattern || + SINK_DEFAULTS.r2.partitioning.time_pattern; + } + + if ( + sink.type === "r2_data_catalog" && + sink.config.namespace && + sink.config.table_name + ) { + destination.Table = `${sink.config.namespace}.${sink.config.table_name}`; + } + + const fileSizeBytes = + sink.config.rolling_policy?.file_size_bytes ?? + SINK_DEFAULTS.rolling_policy.file_size_bytes; + const intervalSeconds = + sink.config.rolling_policy?.interval_seconds ?? + SINK_DEFAULTS.rolling_policy.interval_seconds; + + const batching: Record = { + "File Size": + fileSizeBytes === 0 + ? "none" + : `${Math.round(fileSizeBytes / (1024 * 1024))}MB`, + "Time Interval": `${intervalSeconds}s`, + }; + + const format: Record = { + Type: sink.format.type, + }; + + // Only show compression and row group size for parquet (JSON doesn't support these) + if (sink.format.type === "parquet") { + const defaultParquet = + SINK_DEFAULTS.format.type === "parquet" ? SINK_DEFAULTS.format : null; + if (defaultParquet) { + const compression = sink.format.compression || defaultParquet.compression; + if (compression) { + format.Compression = compression; + } + const rowGroupBytes = + sink.format.row_group_bytes ?? defaultParquet.row_group_bytes; + if (rowGroupBytes !== undefined) { + format["Target Row Group Size"] = + `${Math.round(rowGroupBytes / (1024 * 1024))}MB`; + } + } else { + if (sink.format.compression) { + format.Compression = sink.format.compression; + } + if (sink.format.row_group_bytes !== undefined) { + format["Target Row Group Size"] = + `${Math.round(sink.format.row_group_bytes / (1024 * 1024))}MB`; + } + } + } + + logger.log("General:"); + logger.log(formatLabelledValues(general, { indentationCount: 2 })); + + logger.log("\nDestination:"); + logger.log(formatLabelledValues(destination, { indentationCount: 2 })); + + logger.log("\nBatching:"); + logger.log(formatLabelledValues(batching, { indentationCount: 2 })); + + logger.log("\nFormat:"); + logger.log(formatLabelledValues(format, { indentationCount: 2 })); +} diff --git a/packages/wrangler/src/pipelines/cli/streams/create.ts b/packages/wrangler/src/pipelines/cli/streams/create.ts new file mode 100644 index 000000000000..c62bd5c39fe3 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/streams/create.ts @@ -0,0 +1,121 @@ +import { readFileSync } from "node:fs"; +import { createCommand } from "../../../core/create-command"; +import { confirm } from "../../../dialogs"; +import { UserError } from "../../../errors"; +import { logger } from "../../../logger"; +import { parseJSON } from "../../../parse"; +import { requireAuth } from "../../../user"; +import { createStream } from "../../client"; +import { displayStreamConfiguration } from "./utils"; +import type { CreateStreamRequest, SchemaField } from "../../types"; + +export const pipelinesStreamsCreateCommand = createCommand({ + metadata: { + description: "Create a new stream", + owner: "Product: Pipelines", + status: "open-beta", + }, + positionalArgs: ["stream"], + args: { + stream: { + describe: "The name of the stream to create", + type: "string", + demandOption: true, + }, + "schema-file": { + describe: "Path to JSON file containing stream schema", + type: "string", + }, + "http-enabled": { + describe: "Enable HTTP endpoint", + type: "boolean", + default: true, + }, + "http-auth": { + describe: "Require authentication for HTTP endpoint", + type: "boolean", + default: true, + }, + "cors-origin": { + describe: "CORS origin", + type: "string", + array: true, + }, + }, + async handler(args, { config }) { + await requireAuth(config); + const streamName = args.stream; + + const corsOrigins = args.corsOrigin; + + let schema: { fields: SchemaField[] } | undefined; + if (args.schemaFile) { + let schemaContent: string; + let parsedSchema: { fields: SchemaField[] }; + + try { + schemaContent = readFileSync(args.schemaFile, "utf-8"); + } catch (error) { + throw new UserError( + `Failed to read schema file '${args.schemaFile}': ${error instanceof Error ? error.message : String(error)}` + ); + } + + try { + parsedSchema = parseJSON(schemaContent, args.schemaFile) as { + fields: SchemaField[]; + }; + } catch (error) { + throw new UserError( + `Failed to parse schema file '${args.schemaFile}': ${error instanceof Error ? error.message : String(error)}` + ); + } + + if (!parsedSchema || !Array.isArray(parsedSchema.fields)) { + throw new UserError("Schema file must contain a 'fields' array"); + } + + schema = parsedSchema; + } else { + // No schema file provided - confirm with user + const confirmNoSchema = await confirm( + "No schema file provided. Do you want to create stream without a schema (unstructured JSON)?", + { defaultValue: false } + ); + + if (!confirmNoSchema) { + throw new UserError( + "Stream creation cancelled. Please provide a schema file using --schema-file" + ); + } + } + + const streamConfig: CreateStreamRequest = { + name: streamName, + format: { type: "json" as const, ...(!schema && { unstructured: true }) }, + http: { + enabled: args.httpEnabled, + authentication: args.httpAuth, + ...(args.httpEnabled && + corsOrigins && + corsOrigins.length > 0 && { cors: { origins: corsOrigins } }), + }, + worker_binding: { + enabled: true, + }, + ...(schema && { schema }), + }; + + logger.log(`🌀 Creating stream '${streamName}'...`); + + const stream = await createStream(config, streamConfig); + + logger.log( + `✨ Successfully created stream '${stream.name}' with id '${stream.id}'.` + ); + + displayStreamConfiguration(stream, "Creation Summary", { + includeTimestamps: false, + }); + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/streams/delete.ts b/packages/wrangler/src/pipelines/cli/streams/delete.ts new file mode 100644 index 000000000000..f2f09bd16e27 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/streams/delete.ts @@ -0,0 +1,49 @@ +import { createCommand } from "../../../core/create-command"; +import { confirm } from "../../../dialogs"; +import { logger } from "../../../logger"; +import { requireAuth } from "../../../user"; +import { deleteStream, getStream } from "../../client"; + +export const pipelinesStreamsDeleteCommand = createCommand({ + metadata: { + description: "Delete a stream", + owner: "Product: Pipelines", + status: "open-beta", + }, + positionalArgs: ["stream"], + args: { + stream: { + describe: "The ID of the stream to delete", + type: "string", + demandOption: true, + }, + force: { + describe: "Skip confirmation", + type: "boolean", + alias: "y", + default: false, + }, + }, + async handler(args, { config }) { + await requireAuth(config); + + const stream = await getStream(config, args.stream); + + if (!args.force) { + const confirmedDelete = await confirm( + `Are you sure you want to delete the stream '${stream.name}' (${args.stream})?`, + { fallbackValue: false } + ); + if (!confirmedDelete) { + logger.log("Delete cancelled."); + return; + } + } + + await deleteStream(config, args.stream); + + logger.log( + `✨ Successfully deleted stream '${stream.name}' with id '${stream.id}'.` + ); + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/streams/get.ts b/packages/wrangler/src/pipelines/cli/streams/get.ts new file mode 100644 index 000000000000..fa0bd3d425b2 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/streams/get.ts @@ -0,0 +1,42 @@ +import { createCommand } from "../../../core/create-command"; +import { logger } from "../../../logger"; +import { requireAuth } from "../../../user"; +import { getStream } from "../../client"; +import { displayStreamConfiguration } from "./utils"; + +export const pipelinesStreamsGetCommand = createCommand({ + metadata: { + description: "Get details about a specific stream", + owner: "Product: Pipelines", + status: "open-beta", + }, + positionalArgs: ["stream"], + args: { + stream: { + describe: "The ID of the stream to retrieve", + type: "string", + demandOption: true, + }, + json: { + describe: "Output in JSON format", + type: "boolean", + default: false, + }, + }, + async handler(args, { config }) { + await requireAuth(config); + + const stream = await getStream(config, args.stream); + + if (args.json) { + logger.json(stream); + return; + } + + logger.log(`Stream ID: ${stream.id}`); + + displayStreamConfiguration(stream, undefined, { + includeTimestamps: true, + }); + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/streams/index.ts b/packages/wrangler/src/pipelines/cli/streams/index.ts new file mode 100644 index 000000000000..0eabfd1614a3 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/streams/index.ts @@ -0,0 +1,9 @@ +import { createNamespace } from "../../../core/create-command"; + +export const pipelinesStreamsNamespace = createNamespace({ + metadata: { + description: "Manage streams for pipelines", + owner: "Product: Pipelines", + status: "open-beta", + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/streams/list.ts b/packages/wrangler/src/pipelines/cli/streams/list.ts new file mode 100644 index 000000000000..2b86aa351ba0 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/streams/list.ts @@ -0,0 +1,65 @@ +import { createCommand } from "../../../core/create-command"; +import { logger } from "../../../logger"; +import { requireAuth } from "../../../user"; +import { listStreams } from "../../client"; + +export const pipelinesStreamsListCommand = createCommand({ + metadata: { + description: "List all streams", + owner: "Product: Pipelines", + status: "open-beta", + }, + args: { + page: { + describe: "Page number for pagination", + type: "number", + default: 1, + }, + "per-page": { + describe: "Number of streams per page", + type: "number", + default: 20, + }, + "pipeline-id": { + describe: "Filter streams by pipeline ID", + type: "string", + }, + json: { + describe: "Output in JSON format", + type: "boolean", + default: false, + }, + }, + async handler(args, { config }) { + await requireAuth(config); + + const streams = await listStreams(config, { + page: args.page, + per_page: args.perPage, + pipeline_id: args.pipelineId, + }); + + if (args.json) { + logger.json(streams); + return; + } + + if (!streams || streams.length === 0) { + logger.log("No streams found."); + return; + } + + logger.table( + streams.map((stream) => ({ + Name: stream.name, + ID: stream.id, + HTTP: stream.http.enabled + ? stream.http.authentication + ? "Yes (authenticated)" + : "Yes (unauthenticated)" + : "No", + Created: new Date(stream.created_at).toLocaleDateString(), + })) + ); + }, +}); diff --git a/packages/wrangler/src/pipelines/cli/streams/utils.ts b/packages/wrangler/src/pipelines/cli/streams/utils.ts new file mode 100644 index 000000000000..966a9c170a69 --- /dev/null +++ b/packages/wrangler/src/pipelines/cli/streams/utils.ts @@ -0,0 +1,267 @@ +import { updateConfigFile } from "../../../config"; +import { logger } from "../../../logger"; +import formatLabelledValues from "../../../utils/render-labelled-values"; +import type { Config } from "../../../config"; +import type { SchemaField, Stream } from "../../types"; + +export function formatSchemaFieldsForTable( + fields: SchemaField[], + indent = 0 +): Array<{ + "Field Name": string; + Type: string; + "Unit/Items": string; + Required: string; +}> { + const result: Array<{ + "Field Name": string; + Type: string; + "Unit/Items": string; + Required: string; + }> = []; + const indentStr = " ".repeat(indent); + + for (const field of fields) { + let unitItems = ""; + + if (field.unit) { + unitItems = field.unit; + } + + if (field.type === "list" && field.items) { + unitItems = field.items.type; + } + + const row = { + "Field Name": `${indentStr}${field.name}`, + Type: field.type, + "Unit/Items": unitItems, + Required: field.required ? "Yes" : "No", + }; + + result.push(row); + + if (field.type === "struct" && field.fields) { + result.push(...formatSchemaFieldsForTable(field.fields, indent + 1)); + } + + if ( + field.type === "list" && + field.items && + field.items.type === "struct" && + field.items.fields + ) { + result.push( + ...formatSchemaFieldsForTable(field.items.fields, indent + 1) + ); + } + } + + return result; +} + +export function displayStreamConfiguration( + stream: Stream, + title: string = "Configuration", + options: { includeTimestamps?: boolean } = {} +) { + const { includeTimestamps = true } = options; + + if (title) { + logger.log(`\n${title}:`); + } + + const general: Record = { + Name: stream.name, + }; + + if (includeTimestamps) { + general["Created At"] = new Date(stream.created_at).toLocaleString(); + general["Modified At"] = new Date(stream.modified_at).toLocaleString(); + } + + const httpIngest: Record = { + Enabled: stream.http.enabled ? "Yes" : "No", + }; + + if (stream.http.enabled) { + httpIngest.Authentication = stream.http.authentication ? "Yes" : "No"; + httpIngest.Endpoint = stream.endpoint; + + if ( + stream.http.cors && + stream.http.cors.origins && + stream.http.cors.origins.length > 0 + ) { + httpIngest["CORS Origins"] = stream.http.cors.origins.join(", "); + } else { + httpIngest["CORS Origins"] = "None"; + } + } + + logger.log("General:"); + logger.log(formatLabelledValues(general, { indentationCount: 2 })); + + logger.log("\nHTTP Ingest:"); + logger.log(formatLabelledValues(httpIngest, { indentationCount: 2 })); + + if (stream.schema && stream.schema.fields.length > 0) { + logger.log("\nInput Schema:"); + const schemaRows = formatSchemaFieldsForTable(stream.schema.fields); + logger.table(schemaRows); + } else { + logger.log("\nInput Schema: Unstructured JSON (single 'value' column)"); + } +} + +function generateStreamBindingName(streamName: string): string { + const upperName = streamName.toUpperCase().replace(/[^A-Z0-9]/g, "_"); + return upperName.endsWith("_STREAM") ? upperName : upperName + "_STREAM"; +} + +type SampleValue = + | string + | number + | boolean + | null + | Record + | SampleValue[]; + +function generateSampleValue(field: SchemaField): SampleValue { + switch (field.type) { + case "bool": + return true; + case "int32": + return 42; + case "int64": + return "9223372036854775807"; // Large numbers as strings to avoid JS precision issues + case "f32": + case "f64": + return 3.14; + case "json": + return { example: "json_value" }; + case "bytes": + return "base64_encoded_bytes"; + case "string": + return `sample_${field.name}`; + case "timestamp": + // Return timestamp based on unit + if (field.unit === "second") { + return Math.floor(Date.now() / 1000); + } else if (field.unit === "millisecond") { + return Date.now(); + } else if (field.unit === "microsecond") { + return Date.now() * 1000; + } else if (field.unit === "nanosecond") { + return Date.now() * 1000000; + } + return Date.now(); // Default to milliseconds + case "list": + if (field.items) { + return [ + generateSampleValue(field.items), + generateSampleValue(field.items), + ]; + } + return []; + case "struct": + if (field.fields) { + const structObj: Record = {}; + field.fields.forEach((subField) => { + structObj[subField.name] = generateSampleValue(subField); + }); + return structObj; + } + return {}; + default: + return null; + } +} + +function generateExampleData(stream: Stream): string { + if ( + stream.schema && + stream.schema.fields && + stream.schema.fields.length > 0 + ) { + const exampleObject: Record = {}; + + stream.schema.fields.forEach((field) => { + exampleObject[field.name] = generateSampleValue(field); + }); + + return JSON.stringify(exampleObject, null, 2); + } else { + // Default unstructured example + return JSON.stringify( + { + user_id: "sample_user_id", + event_name: "sample_event_name", + timestamp: Date.now(), + }, + null, + 2 + ); + } +} + +export async function displayUsageExamples( + stream: Stream, + config: Config, + args: { env?: string } +) { + const bindingName = generateStreamBindingName(stream.name); + const exampleData = generateExampleData(stream); + + logger.log(`\nSend your first event to stream '${stream.name}':`); + + // Worker binding example (always shown since worker_binding is always enabled) + logger.log("\nWorker Integration:"); + + await updateConfigFile( + (customBindingName) => ({ + pipelines: [ + { + pipeline: stream.id, + binding: customBindingName ?? bindingName, + }, + ], + }), + config.configPath, + args.env, + false // Don't offer to update automatically + ); + + logger.log("\nIn your Worker:"); + logger.log(`await env.${bindingName}.send([${exampleData}]);`); + + // HTTP endpoint example (only if HTTP is enabled) + if (stream.http.enabled) { + logger.log("\nHTTP Endpoint:"); + + const curlCommand = [ + `curl -X POST ${stream.endpoint}`, + stream.http.authentication + ? `-H "Authorization: Bearer YOUR_API_TOKEN"` + : null, + `-H "Content-Type: application/json"`, + `-d '[${exampleData}]'`, + ] + .filter(Boolean) + .join(" \\\n "); + + logger.log(curlCommand); + + if (stream.http.authentication) { + logger.log("(Replace YOUR_API_TOKEN with your Cloudflare API token)"); + } + + if ( + stream.http.cors && + stream.http.cors.origins && + !stream.http.cors.origins.includes("*") + ) { + logger.log(`CORS origins: ${stream.http.cors.origins.join(", ")}`); + } + } +} diff --git a/packages/wrangler/src/pipelines/cli/update.ts b/packages/wrangler/src/pipelines/cli/update.ts index 6327eababa3f..2d306378a0c3 100644 --- a/packages/wrangler/src/pipelines/cli/update.ts +++ b/packages/wrangler/src/pipelines/cli/update.ts @@ -1,27 +1,21 @@ import { createCommand } from "../../core/create-command"; -import { FatalError, UserError } from "../../errors"; -import { logger } from "../../logger"; +import { UserError } from "../../errors"; +import { APIError } from "../../parse"; import { requireAuth } from "../../user"; -import { getPipeline, updatePipeline } from "../client"; -import { - authorizeR2Bucket, - BYTES_PER_MB, - getAccountR2Endpoint, - parseTransform, -} from "../index"; +import { getPipeline } from "../client"; import { validateCorsOrigins, validateInRange } from "../validate"; -import type { BindingSource, HttpSource, Source } from "../client"; +import { updateLegacyPipeline } from "./legacy-helpers"; export const pipelinesUpdateCommand = createCommand({ metadata: { - description: "Update a pipeline", + description: "Update a pipeline configuration (legacy pipelines only)", owner: "Product: Pipelines", status: "open-beta", }, positionalArgs: ["pipeline"], args: { pipeline: { - describe: "The name of the new pipeline", + describe: "The name of the legacy pipeline to update", type: "string", demandOption: true, }, @@ -73,16 +67,6 @@ export const pipelinesUpdateCommand = createCommand({ group: "Batch hints", }, - // Transform options - "transform-worker": { - type: "string", - describe: - "Pipeline transform Worker and entrypoint (.)", - demandOption: false, - hidden: true, // TODO: Remove once transformations launch - group: "Transformations", - }, - "r2-bucket": { type: "string", describe: "Destination R2 bucket name", @@ -130,145 +114,34 @@ export const pipelinesUpdateCommand = createCommand({ }, }, async handler(args, { config }) { - const name = args.pipeline; - // only the fields set will be updated - other fields will use the existing config const accountId = await requireAuth(config); + const pipelineId = args.pipeline; - const pipelineConfig = await getPipeline(config, accountId, name); - - if (args.compression) { - pipelineConfig.destination.compression.type = args.compression; - } - if (args.batchMaxMb) { - pipelineConfig.destination.batch.max_bytes = - args.batchMaxMb * BYTES_PER_MB; // convert to bytes for the API - } - if (args.batchMaxSeconds) { - pipelineConfig.destination.batch.max_duration_s = args.batchMaxSeconds; - } - if (args.batchMaxRows) { - pipelineConfig.destination.batch.max_rows = args.batchMaxRows; - } - - const bucket = args.r2Bucket; - const accessKeyId = args.r2AccessKeyId; - const secretAccessKey = args.r2SecretAccessKey; - if (bucket || accessKeyId || secretAccessKey) { - const destination = pipelineConfig.destination; - if (bucket) { - pipelineConfig.destination.path.bucket = bucket; - } - destination.credentials = { - endpoint: getAccountR2Endpoint(accountId), - access_key_id: accessKeyId || "", - secret_access_key: secretAccessKey || "", - }; - if (!accessKeyId && !secretAccessKey) { - const auth = await authorizeR2Bucket( - config, - name, - accountId, - destination.path.bucket - ); - destination.credentials.access_key_id = auth.accessKeyId; - destination.credentials.secret_access_key = auth.secretAccessKey; - } - if (!destination.credentials.access_key_id) { - throw new FatalError("Requires a r2 access key id"); - } - - if (!destination.credentials.secret_access_key) { - throw new FatalError("Requires a r2 secret access key"); - } - } - - if (args.source && args.source.length > 0) { - const existingSources = pipelineConfig.source; - pipelineConfig.source = []; // Reset the list - - const sourceHandlers: Record Source> = { - http: (): HttpSource => { - const existing = existingSources.find( - (s: Source) => s.type === "http" - ); - - return { - ...existing, // Copy over existing properties for forwards compatibility - type: "http", - format: "json", - ...(args.requireHttpAuth && { - authentication: args.requireHttpAuth, - }), // Include only if defined - }; - }, - worker: (): BindingSource => { - const existing = existingSources.find( - (s: Source) => s.type === "binding" - ); - - return { - ...existing, // Copy over existing properties for forwards compatibility - type: "binding", - format: "json", - }; - }, - }; - - for (const source of args.source) { - const handler = sourceHandlers[source]; - if (handler) { - pipelineConfig.source.push(handler()); - } - } - } - - if (pipelineConfig.source.length === 0) { + try { + await getPipeline(config, pipelineId); throw new UserError( - "No sources have been enabled. At least one source (HTTP or Worker Binding) should be enabled" + "Pipelines created with the V1 API cannot be updated. To modify your pipeline, delete and recreate it with your new SQL." ); - } - - if (args.transformWorker) { - if (args.transformWorker === "none") { - // Unset transformations - pipelineConfig.transforms = []; - } else { - pipelineConfig.transforms.push(parseTransform(args.transformWorker)); - } - } - - if (args.r2Prefix) { - pipelineConfig.destination.path.prefix = args.r2Prefix; - } - - if (args.shardCount) { - pipelineConfig.metadata.shards = args.shardCount; - } - - // This covers the case where `--source` wasn't passed but `--cors-origins` or - // `--require-http-auth` was. - const httpSource = pipelineConfig.source.find( - (s: Source) => s.type === "http" - ); - if (httpSource) { - if (args.requireHttpAuth) { - httpSource.authentication = args.requireHttpAuth; + } catch (error) { + if ( + error instanceof APIError && + (error.code === 1000 || error.code === 2) + ) { + try { + return await updateLegacyPipeline(config, accountId, args); + } catch (legacyError) { + if (legacyError instanceof APIError && legacyError.code === 1000) { + throw new UserError( + `Pipeline "${pipelineId}" not found. The update command only works with legacy pipelines. Pipelines created with the V1 API cannot be updated and must be recreated to modify.` + ); + } + throw legacyError; + } } - if (args.corsOrigins) { - httpSource.cors = { origins: args.corsOrigins }; + if (!(error instanceof UserError)) { + throw error; } + throw error; } - - logger.log(`🌀 Updating pipeline "${name}"`); - const pipeline = await updatePipeline( - config, - accountId, - name, - pipelineConfig - ); - - logger.log( - `✅ Successfully updated pipeline "${pipeline.name}" with ID ${pipeline.id}\n` - ); }, }); diff --git a/packages/wrangler/src/pipelines/client.ts b/packages/wrangler/src/pipelines/client.ts index 2e3131a54dd4..483508328644 100644 --- a/packages/wrangler/src/pipelines/client.ts +++ b/packages/wrangler/src/pipelines/client.ts @@ -1,115 +1,289 @@ import assert from "node:assert"; -import { createHash } from "node:crypto"; import http from "node:http"; import { setTimeout as setTimeoutPromise } from "node:timers/promises"; +import { URLSearchParams } from "node:url"; import { fetchResult } from "../cfetch"; import { getCloudflareApiEnvironmentFromEnv } from "../environment-variables/misc-variables"; import { UserError } from "../errors"; import { logger } from "../logger"; import openInBrowser from "../open-in-browser"; +import { requireAuth } from "../user"; +import type { Config } from "../config"; import type { ComplianceConfig } from "../environment-variables/misc-variables"; import type { R2BucketInfo } from "../r2/helpers"; +import type { + CreatePipelineRequest, + CreateSinkRequest, + CreateStreamRequest, + ListPipelinesParams, + ListSinksParams, + ListStreamsParams, + Pipeline, + Sink, + Stream, + ValidateSqlRequest, + ValidateSqlResponse, +} from "./types"; -// ensure this is in sync with: -// https://bitbucket.cfdata.org/projects/PIPE/repos/superpipe/browse/src/coordinator/types.ts#6 -type RecursivePartial = T extends object - ? { - [P in keyof T]?: RecursivePartial; +export async function listPipelines( + config: Config, + params?: ListPipelinesParams +): Promise { + const accountId = await requireAuth(config); + const searchParams = new URLSearchParams(); + + if (params?.page) { + searchParams.set("page", params.page.toString()); + } + if (params?.per_page) { + searchParams.set("per_page", params.per_page.toString()); + } + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/pipelines`, + { + method: "GET", + }, + searchParams + ); + + return response; +} + +export async function listStreams( + config: Config, + params?: ListStreamsParams +): Promise { + const accountId = await requireAuth(config); + const searchParams = new URLSearchParams(); + + if (params?.page) { + searchParams.set("page", params.page.toString()); + } + if (params?.per_page) { + searchParams.set("per_page", params.per_page.toString()); + } + if (params?.pipeline_id) { + searchParams.set("pipeline_id", params.pipeline_id); + } + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/streams`, + { + method: "GET", + }, + searchParams + ); + + return response; +} + +export async function listSinks( + config: Config, + params?: ListSinksParams +): Promise { + const accountId = await requireAuth(config); + const searchParams = new URLSearchParams(); + + if (params?.page) { + searchParams.set("page", params.page.toString()); + } + if (params?.per_page) { + searchParams.set("per_page", params.per_page.toString()); + } + if (params?.pipeline_id) { + searchParams.set("pipeline_id", params.pipeline_id); + } + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/sinks`, + { + method: "GET", + }, + searchParams + ); + + return response; +} + +export async function createStream( + config: Config, + streamConfig: CreateStreamRequest +): Promise { + const accountId = await requireAuth(config); + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/streams`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(streamConfig), + } + ); + + return response; +} + +export async function getStream( + config: Config, + streamId: string +): Promise { + const accountId = await requireAuth(config); + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/streams/${streamId}`, + { + method: "GET", + } + ); + + return response; +} + +export async function deleteStream( + config: Config, + streamId: string +): Promise { + const accountId = await requireAuth(config); + + await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/streams/${streamId}`, + { + method: "DELETE", + } + ); +} + +export async function getSink(config: Config, sinkId: string): Promise { + const accountId = await requireAuth(config); + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/sinks/${sinkId}`, + { + method: "GET", + } + ); + + return response; +} + +export async function deleteSink( + config: Config, + sinkId: string +): Promise { + const accountId = await requireAuth(config); + + await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/sinks/${sinkId}`, + { + method: "DELETE", + } + ); +} + +export async function createSink( + config: Config, + sinkConfig: CreateSinkRequest +): Promise { + const accountId = await requireAuth(config); + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/sinks`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(sinkConfig), + } + ); + + return response; +} + +export async function getPipeline( + config: Config, + pipelineId: string +): Promise { + const accountId = await requireAuth(config); + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/pipelines/${pipelineId}`, + { + method: "GET", } - : T; - -export type PartialExcept = RecursivePartial & - Pick; - -export type TransformConfig = { - script: string; - entrypoint: string; -}; - -export type HttpSource = { - type: "http"; - format: string; - schema?: string; - authentication?: boolean; - cors?: { - origins: ["*"] | string[]; - }; -}; - -export type BindingSource = { - type: "binding"; - format: string; - schema?: string; -}; - -export type Metadata = { - shards?: number; - [x: string]: unknown; -}; - -export type Source = HttpSource | BindingSource; - -export type PipelineUserConfig = { - name: string; - metadata: Metadata; - source: Source[]; - transforms: TransformConfig[]; - destination: { - type: string; - format: string; - compression: { - type: string; - }; - batch: { - max_duration_s?: number; - max_bytes?: number; - max_rows?: number; - }; - path: { - bucket: string; - prefix?: string; - filepath?: string; - filename?: string; - }; - credentials: { - endpoint: string; - secret_access_key: string; - access_key_id: string; - }; - }; -}; - -// Pipeline from v4 API -export type Pipeline = Omit & { - id: string; - version: number; - endpoint: string; - destination: Omit & { - credentials?: PipelineUserConfig["destination"]["credentials"]; - }; -}; - -// abbreviated Pipeline from Pipeline list call -export type PipelineEntry = { - id: string; - name: string; - endpoint: string; -}; - -// Payload for Service Tokens -export type ServiceToken = { - id: string; - name: string; - value: string; -}; - -// standard headers for update calls to v4 API -const API_HEADERS = { - "Content-Type": "application/json", -}; - -export function sha256(s: string): string { - return createHash("sha256").update(s).digest("hex"); + ); + + return response; +} + +export async function deletePipeline( + config: Config, + pipelineId: string +): Promise { + const accountId = await requireAuth(config); + + await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/pipelines/${pipelineId}`, + { + method: "DELETE", + } + ); +} + +export async function createPipeline( + config: Config, + pipelineConfig: CreatePipelineRequest +): Promise { + const accountId = await requireAuth(config); + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/pipelines`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(pipelineConfig), + } + ); + + return response; +} + +export async function validateSql( + config: Config, + sqlRequest: ValidateSqlRequest +): Promise { + const accountId = await requireAuth(config); + + const response = await fetchResult( + config, + `/accounts/${accountId}/pipelines/v1/validate_sql`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(sqlRequest), + } + ); + + return response; } export interface S3AccessKey { @@ -221,83 +395,3 @@ export async function getR2Bucket( `/accounts/${accountId}/r2/buckets/${name}` ); } - -// v4 API to Create new Pipeline -export async function createPipeline( - complianceConfig: ComplianceConfig, - accountId: string, - pipelineConfig: PipelineUserConfig -): Promise { - return await fetchResult( - complianceConfig, - `/accounts/${accountId}/pipelines`, - { - method: "POST", - headers: API_HEADERS, - body: JSON.stringify(pipelineConfig), - } - ); -} - -// v4 API to Get Pipeline Details -export async function getPipeline( - complianceConfig: ComplianceConfig, - accountId: string, - name: string -): Promise { - return await fetchResult( - complianceConfig, - `/accounts/${accountId}/pipelines/${name}`, - { - method: "GET", - } - ); -} - -// v4 API to Update Pipeline Configuration -export async function updatePipeline( - complianceConfig: ComplianceConfig, - accountId: string, - name: string, - pipelineConfig: PartialExcept -): Promise { - return await fetchResult( - complianceConfig, - `/accounts/${accountId}/pipelines/${name}`, - { - method: "PUT", - headers: API_HEADERS, - body: JSON.stringify(pipelineConfig), - } - ); -} - -// v4 API to List Available Pipelines -export async function listPipelines( - complianceConfig: ComplianceConfig, - accountId: string -): Promise { - return await fetchResult( - complianceConfig, - `/accounts/${accountId}/pipelines`, - { - method: "GET", - } - ); -} - -// v4 API to Delete Pipeline -export async function deletePipeline( - complianceConfig: ComplianceConfig, - accountId: string, - name: string -): Promise { - return await fetchResult( - complianceConfig, - `/accounts/${accountId}/pipelines/${name}`, - { - method: "DELETE", - headers: API_HEADERS, - } - ); -} diff --git a/packages/wrangler/src/pipelines/defaults.ts b/packages/wrangler/src/pipelines/defaults.ts new file mode 100644 index 000000000000..634e4af5e45f --- /dev/null +++ b/packages/wrangler/src/pipelines/defaults.ts @@ -0,0 +1,65 @@ +import type { ParquetFormat, Sink } from "./types"; + +export const SINK_DEFAULTS = { + format: { + type: "parquet", + compression: "zstd", + row_group_bytes: 1024 * 1024 * 1024, + } as ParquetFormat, + rolling_policy: { + file_size_bytes: 0, + interval_seconds: 30, + }, + r2: { + path: "", + partitioning: { + time_pattern: "year=%Y/month=%m/day=%d", + }, + }, + r2_data_catalog: {}, +} as const; + +export function applyDefaultsToSink(sink: Sink): Sink { + const withDefaults: Sink = { + ...sink, + format: { ...sink.format }, + config: { ...sink.config }, + }; + + if (withDefaults.format.type === "parquet") { + if (!withDefaults.format.compression) { + withDefaults.format.compression = SINK_DEFAULTS.format.compression; + } + if (!withDefaults.format.row_group_bytes) { + withDefaults.format.row_group_bytes = + SINK_DEFAULTS.format.row_group_bytes; + } + } + + if (!withDefaults.config.rolling_policy) { + withDefaults.config.rolling_policy = { + file_size_bytes: SINK_DEFAULTS.rolling_policy.file_size_bytes, + interval_seconds: SINK_DEFAULTS.rolling_policy.interval_seconds, + }; + } else { + if (!withDefaults.config.rolling_policy.file_size_bytes) { + withDefaults.config.rolling_policy.file_size_bytes = + SINK_DEFAULTS.rolling_policy.file_size_bytes; + } + if (!withDefaults.config.rolling_policy.interval_seconds) { + withDefaults.config.rolling_policy.interval_seconds = + SINK_DEFAULTS.rolling_policy.interval_seconds; + } + } + + if (withDefaults.type === "r2") { + if (!withDefaults.config.path) { + withDefaults.config.path = SINK_DEFAULTS.r2.path; + } + if (!withDefaults.config.partitioning) { + withDefaults.config.partitioning = SINK_DEFAULTS.r2.partitioning; + } + } + + return withDefaults; +} diff --git a/packages/wrangler/src/pipelines/index.ts b/packages/wrangler/src/pipelines/index.ts index 3e3bd515a992..bc6615ec40e5 100644 --- a/packages/wrangler/src/pipelines/index.ts +++ b/packages/wrangler/src/pipelines/index.ts @@ -1,15 +1,12 @@ import { setTimeout } from "node:timers/promises"; import { HeadBucketCommand, S3Client } from "@aws-sdk/client-s3"; -import prettyBytes from "pretty-bytes"; import { createNamespace } from "../core/create-command"; import { getCloudflareApiEnvironmentFromEnv } from "../environment-variables/misc-variables"; import { FatalError } from "../errors"; import { logger } from "../logger"; import { APIError } from "../parse"; -import formatLabelledValues from "../utils/render-labelled-values"; import { generateR2ServiceToken, getR2Bucket } from "./client"; import type { ComplianceConfig } from "../environment-variables/misc-variables"; -import type { Pipeline } from "./client"; export const BYTES_PER_MB = 1000 * 1000; @@ -130,76 +127,3 @@ export const pipelinesNamespace = createNamespace({ export function __testSkipDelays() { __testSkipDelaysFlag = true; } - -/* - - */ -export function formatPipelinePretty(pipeline: Pipeline) { - let buffer = ""; - - const formatTypeLabels: Record = { - json: "JSON", - }; - - buffer += `${formatLabelledValues({ - Id: pipeline.id, - Name: pipeline.name, - })}\n`; - - buffer += "Sources:\n"; - const httpSource = pipeline.source.find((s) => s.type === "http"); - if (httpSource) { - const httpInfo = { - Endpoint: pipeline.endpoint, - Authentication: httpSource.authentication === true ? "on" : "off", - ...(httpSource?.cors?.origins && { - "CORS Origins": httpSource.cors.origins.join(", "), - }), - Format: formatTypeLabels[httpSource.format], - }; - buffer += " HTTP:\n"; - buffer += `${formatLabelledValues(httpInfo, { indentationCount: 4 })}\n`; - } - - const bindingSource = pipeline.source.find((s) => s.type === "binding"); - if (bindingSource) { - const bindingInfo = { - Format: formatTypeLabels[bindingSource.format], - }; - buffer += " Worker:\n"; - buffer += `${formatLabelledValues(bindingInfo, { indentationCount: 4 })}\n`; - } - - const destinationInfo = { - Type: pipeline.destination.type.toUpperCase(), - Bucket: pipeline.destination.path.bucket, - Format: "newline-delimited JSON", // TODO: Make dynamic once we support more output formats - ...(pipeline.destination.path.prefix && { - Prefix: pipeline.destination.path.prefix, - }), - ...(pipeline.destination.compression.type && { - Compression: pipeline.destination.compression.type.toUpperCase(), - }), - }; - buffer += "Destination:\n"; - buffer += `${formatLabelledValues(destinationInfo, { indentationCount: 2 })}\n`; - - const batchHints = { - ...(pipeline.destination.batch.max_bytes && { - "Max bytes": prettyBytes(pipeline.destination.batch.max_bytes), - }), - ...(pipeline.destination.batch.max_duration_s && { - "Max duration": `${pipeline.destination.batch.max_duration_s?.toLocaleString()} seconds`, - }), - ...(pipeline.destination.batch.max_rows && { - "Max records": pipeline.destination.batch.max_rows?.toLocaleString(), - }), - }; - - if (Object.keys(batchHints).length > 0) { - buffer += " Batch hints:\n"; - buffer += `${formatLabelledValues(batchHints, { indentationCount: 4 })}\n`; - } - - return buffer; -} diff --git a/packages/wrangler/src/pipelines/legacy-client.ts b/packages/wrangler/src/pipelines/legacy-client.ts new file mode 100644 index 000000000000..e5e7e9a36c57 --- /dev/null +++ b/packages/wrangler/src/pipelines/legacy-client.ts @@ -0,0 +1,260 @@ +import { createHash } from "node:crypto"; +import prettyBytes from "pretty-bytes"; +import { fetchResult } from "../cfetch"; +import formatLabelledValues from "../utils/render-labelled-values"; +import type { ComplianceConfig } from "../environment-variables/misc-variables"; + +// ensure this is in sync with: +// https://bitbucket.cfdata.org/projects/PIPE/repos/superpipe/browse/src/coordinator/types.ts#6 +type RecursivePartial = T extends object + ? { + [P in keyof T]?: RecursivePartial; + } + : T; + +export type PartialExcept = RecursivePartial & + Pick; + +export type TransformConfig = { + script: string; + entrypoint: string; +}; + +export type HttpSource = { + type: "http"; + format: string; + schema?: string; + authentication?: boolean; + cors?: { + origins: ["*"] | string[]; + }; +}; + +export type BindingSource = { + type: "binding"; + format: string; + schema?: string; +}; + +export type Metadata = { + shards?: number; + [x: string]: unknown; +}; + +export type Source = HttpSource | BindingSource; + +export type PipelineUserConfig = { + name: string; + metadata: Metadata; + source: Source[]; + transforms: TransformConfig[]; + destination: { + type: string; + format: string; + compression: { + type: string; + }; + batch: { + max_duration_s?: number; + max_bytes?: number; + max_rows?: number; + }; + path: { + bucket: string; + prefix?: string; + filepath?: string; + filename?: string; + }; + credentials: { + endpoint: string; + secret_access_key: string; + access_key_id: string; + }; + }; +}; + +// Pipeline from v4 API +export type Pipeline = Omit & { + id: string; + version: number; + endpoint: string; + destination: Omit & { + credentials?: PipelineUserConfig["destination"]["credentials"]; + }; +}; + +// abbreviated Pipeline from Pipeline list call +export type PipelineEntry = { + id: string; + name: string; + endpoint: string; +}; + +// Payload for Service Tokens +export type ServiceToken = { + id: string; + name: string; + value: string; +}; + +// standard headers for update calls to v4 API +const API_HEADERS = { + "Content-Type": "application/json", +}; + +export function sha256(s: string): string { + return createHash("sha256").update(s).digest("hex"); +} + +// v4 API to Create new Pipeline +export async function createPipeline( + complianceConfig: ComplianceConfig, + accountId: string, + pipelineConfig: PipelineUserConfig +): Promise { + return await fetchResult( + complianceConfig, + `/accounts/${accountId}/pipelines`, + { + method: "POST", + headers: API_HEADERS, + body: JSON.stringify(pipelineConfig), + } + ); +} + +// v4 API to Get Pipeline Details +export async function getPipeline( + complianceConfig: ComplianceConfig, + accountId: string, + name: string +): Promise { + return await fetchResult( + complianceConfig, + `/accounts/${accountId}/pipelines/${name}`, + { + method: "GET", + } + ); +} + +// v4 API to Update Pipeline Configuration +export async function updatePipeline( + complianceConfig: ComplianceConfig, + accountId: string, + name: string, + pipelineConfig: PartialExcept +): Promise { + return await fetchResult( + complianceConfig, + `/accounts/${accountId}/pipelines/${name}`, + { + method: "PUT", + headers: API_HEADERS, + body: JSON.stringify(pipelineConfig), + } + ); +} + +// v4 API to List Available Pipelines +export async function listPipelines( + complianceConfig: ComplianceConfig, + accountId: string +): Promise { + return await fetchResult( + complianceConfig, + `/accounts/${accountId}/pipelines`, + { + method: "GET", + } + ); +} + +// v4 API to Delete Pipeline +export async function deletePipeline( + complianceConfig: ComplianceConfig, + accountId: string, + name: string +): Promise { + return await fetchResult( + complianceConfig, + `/accounts/${accountId}/pipelines/${name}`, + { + method: "DELETE", + headers: API_HEADERS, + } + ); +} + +/* + + */ +export function formatPipelinePretty(pipeline: Pipeline) { + let buffer = ""; + + const formatTypeLabels: Record = { + json: "JSON", + }; + + buffer += `${formatLabelledValues({ + Id: pipeline.id, + Name: pipeline.name, + })}\n`; + + buffer += "Sources:\n"; + const httpSource = pipeline.source.find((s) => s.type === "http"); + if (httpSource) { + const httpInfo = { + Endpoint: pipeline.endpoint, + Authentication: httpSource.authentication === true ? "on" : "off", + ...(httpSource?.cors?.origins && { + "CORS Origins": httpSource.cors.origins.join(", "), + }), + Format: formatTypeLabels[httpSource.format], + }; + buffer += " HTTP:\n"; + buffer += `${formatLabelledValues(httpInfo, { indentationCount: 4 })}\n`; + } + + const bindingSource = pipeline.source.find((s) => s.type === "binding"); + if (bindingSource) { + const bindingInfo = { + Format: formatTypeLabels[bindingSource.format], + }; + buffer += " Worker:\n"; + buffer += `${formatLabelledValues(bindingInfo, { indentationCount: 4 })}\n`; + } + + const destinationInfo = { + Type: pipeline.destination.type.toUpperCase(), + Bucket: pipeline.destination.path.bucket, + Format: "newline-delimited JSON", // TODO: Make dynamic once we support more output formats + ...(pipeline.destination.path.prefix && { + Prefix: pipeline.destination.path.prefix, + }), + ...(pipeline.destination.compression.type && { + Compression: pipeline.destination.compression.type.toUpperCase(), + }), + }; + buffer += "Destination:\n"; + buffer += `${formatLabelledValues(destinationInfo, { indentationCount: 2 })}\n`; + + const batchHints = { + ...(pipeline.destination.batch.max_bytes && { + "Max bytes": prettyBytes(pipeline.destination.batch.max_bytes), + }), + ...(pipeline.destination.batch.max_duration_s && { + "Max duration": `${pipeline.destination.batch.max_duration_s?.toLocaleString()} seconds`, + }), + ...(pipeline.destination.batch.max_rows && { + "Max records": pipeline.destination.batch.max_rows?.toLocaleString(), + }), + }; + + if (Object.keys(batchHints).length > 0) { + buffer += " Batch hints:\n"; + buffer += `${formatLabelledValues(batchHints, { indentationCount: 4 })}\n`; + } + + return buffer; +} diff --git a/packages/wrangler/src/pipelines/types.ts b/packages/wrangler/src/pipelines/types.ts new file mode 100644 index 000000000000..6148ac8d6f1f --- /dev/null +++ b/packages/wrangler/src/pipelines/types.ts @@ -0,0 +1,241 @@ +export type PipelineTable = { + id: string; + version: number; + latest: number; + type: "stream" | "sink"; + name: string; + href: string; +}; + +export interface Pipeline { + id: string; + name: string; + created_at: string; + modified_at: string; + sql: string; + status: string; + tables?: PipelineTable[]; +} + +export interface PaginationInfo { + count: number; + page: number; + per_page: number; + total_count: number; +} + +export interface CloudflareAPIResponse { + success: boolean; + errors: string[]; + messages: string[]; + result: T; +} + +export interface PipelineListResponse + extends CloudflareAPIResponse { + result_info: PaginationInfo; +} + +export interface CreatePipelineRequest { + name: string; + sql: string; +} + +export interface ValidateSqlRequest { + sql: string; +} + +export type ValidateSqlResponse = CloudflareAPIResponse<{ + graph?: { + nodes: Array<{ + node_id: number; + operator: string; + description: string; + parallelism: number; + }>; + edges: Array<{ + src_id: number; + dest_id: number; + key_type: string; + value_type: string; + edge_type: string; + }>; + }; + tables: Record< + string, + { + id: string; + version: number; + type: string; + name: string; + } + >; +}>; + +export interface ListPipelinesParams { + page?: number; + per_page?: number; +} + +export interface Stream { + id: string; + name: string; + version: number; + created_at: string; + modified_at: string; + endpoint: string; + format: StreamJsonFormat; + schema: { + fields: SchemaField[]; + } | null; + http: { + enabled: boolean; + authentication: boolean; + cors?: { + origins: string[]; + }; + }; + worker_binding: { + enabled: boolean; + }; +} + +export interface StreamListResponse extends CloudflareAPIResponse { + result_info: PaginationInfo; +} + +export interface ListStreamsParams { + page?: number; + per_page?: number; + pipeline_id?: string; +} + +// Stream Format type (only JSON supported) +export type StreamJsonFormat = { + type: "json"; + timestamp_format?: "rfc3339" | "unix_millis"; + unstructured?: boolean; +}; + +// Format types +export type JsonFormat = { + type: "json"; +}; + +export type ParquetFormat = { + type: "parquet"; + compression?: "uncompressed" | "snappy" | "gzip" | "zstd" | "lz4"; + row_group_bytes?: number; +}; + +export type SinkFormat = JsonFormat | ParquetFormat; + +// Schema types +export type SchemaField = { + name: string; + type: + | "bool" + | "int32" + | "int64" + | "f32" + | "f64" + | "string" + | "timestamp" + | "json" + | "bytes" + | "list" + | "struct"; + required: boolean; + fields?: SchemaField[]; + items?: SchemaField; + unit?: "second" | "millisecond" | "microsecond" | "nanosecond"; // For timestamp type +}; + +export type Sink = { + id: string; + name: string; + created_at?: string; + modified_at?: string; + version?: number; + type: "r2" | "r2_data_catalog"; + format: SinkFormat; + schema: { + fields: SchemaField[]; + } | null; + config: { + bucket: string; + path?: string; + partitioning?: { + time_pattern: string; + }; + rolling_policy?: { + file_size_bytes: number; + interval_seconds: number; + }; + // r2_data_catalog specific fields + account_id?: string; + token?: string; + namespace?: string; + table_name?: string; + }; + used_by?: Array<{ + href: string; + }>; +}; + +export interface SinkListResponse extends CloudflareAPIResponse { + result_info: PaginationInfo; +} + +export interface ListSinksParams { + page?: number; + per_page?: number; + pipeline_id?: string; +} + +export interface CreateSinkRequest { + name: string; + type: "r2" | "r2_data_catalog"; + format?: SinkFormat; + schema?: { + fields: SchemaField[]; + }; + config: { + bucket: string; + path?: string; + partitioning?: { + time_pattern: string; + }; + rolling_policy?: { + file_size_bytes: number; + interval_seconds: number; + }; + // R2 credentials (for r2 type) + credentials?: { + access_key_id: string; + secret_access_key: string; + }; + // R2 Data Catalog specific fields + namespace?: string; + table_name?: string; + token?: string; + }; +} + +export interface CreateStreamRequest { + name: string; + format?: StreamJsonFormat; + schema?: { + fields: SchemaField[]; + }; + http: { + enabled: boolean; + authentication: boolean; + cors?: { + origins: string[]; + }; + }; + worker_binding: { + enabled: boolean; + }; +}