Add Datasets and Uploads APIs, deprecate Tables API (#86)

Author: va3093

.github/workflows/pull-request.yaml

@@ -38,6 +38,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }}
+      DUNE_API_KEY_OWNER_HANDLE: ${{ secrets.DUNE_API_KEY_OWNER_HANDLE }}
 
     steps:
       - name: Checkout code

src/api/client.ts

@@ -26,6 +26,8 @@ import { TableAPI } from "./table";
 import { CustomAPI } from "./custom";
 import { UsageAPI } from "./usage";
 import { PipelineAPI } from "./pipeline";
+import { DatasetAPI } from "./dataset";
+import { UploadsAPI } from "./uploads";
 import { deprecationWarning } from "../deprecation";
 
 /// Various states of query execution that are "terminal".
@@ -45,14 +47,18 @@ export class DuneClient {
   exec: ExecutionAPI;
   /// Query Management Interface.
   query: QueryAPI;
-  /// Table Management Interface
+  /// Table Management Interface (deprecated, use uploads instead)
   table: TableAPI;
   /// Custom Endpoint Interface
   custom: CustomAPI;
   /// Usage Interface
   usage: UsageAPI;
   /// Pipeline Interface
   pipeline: PipelineAPI;
+  /// Dataset Interface
+  dataset: DatasetAPI;
+  /// Uploads Interface
+  uploads: UploadsAPI;
 
   constructor(apiKey: string) {
     this.exec = new ExecutionAPI(apiKey);
@@ -61,6 +67,8 @@ export class DuneClient {
     this.custom = new CustomAPI(apiKey);
     this.usage = new UsageAPI(apiKey);
     this.pipeline = new PipelineAPI(apiKey);
+    this.dataset = new DatasetAPI(apiKey);
+    this.uploads = new UploadsAPI(apiKey);
   }
 
   /**

src/api/dataset.ts

@@ -0,0 +1,12 @@
+import { Router } from "./router";
+import { DatasetResponse, ListDatasetsArgs, ListDatasetsResponse } from "../types";
+
+export class DatasetAPI extends Router {
+  async list(args?: ListDatasetsArgs): Promise<ListDatasetsResponse> {
+    return this._get<ListDatasetsResponse>("datasets", args);
+  }
+
+  async getBySlug(slug: string): Promise<DatasetResponse> {
+    return this._get<DatasetResponse>(`datasets/${slug}`);
+  }
+}

src/api/index.ts

@@ -1,8 +1,10 @@
 export * from "./client";
 export * from "./custom";
+export * from "./dataset";
 export * from "./execution";
 export * from "./pipeline";
 export * from "./query";
 export * from "./router";
 export * from "./table";
+export * from "./uploads";
 export * from "./usage";

src/api/table.ts

@@ -11,6 +11,7 @@ import {
   DeleteTableResult,
 } from "../types";
 import { withDefaults } from "../utils";
+import { deprecationWarning } from "../deprecation";
 
 /**
  * Table Management Interface (includes uploadCSV)
@@ -24,8 +25,10 @@ export class TableAPI extends Router {
    *
    * @param args UploadCSVParams relevant fields related to dataset upload.
    * @returns boolean representing if upload was successful.
+   * @deprecated Use uploads.uploadCsv() instead. The /v1/table endpoints are deprecated in favor of /v1/uploads.
    */
   async uploadCsv(args: UploadCSVArgs): Promise<boolean> {
+    deprecationWarning("table.uploadCsv", "uploads.uploadCsv");
     const response = await this.post<SuccessResponse>("table/upload/csv", args);
     try {
       return Boolean(response.success);
@@ -44,10 +47,12 @@ export class TableAPI extends Router {
    *
    * The only limitations are:
    * - If a table already exists with the same name, the request will fail.
-   * - Column names in the table can’t start with a special character or a digit.
+   * - Column names in the table can't start with a special character or a digit.
    * @param args
+   * @deprecated Use uploads.create() instead. The /v1/table endpoints are deprecated in favor of /v1/uploads.
    */
   async create(args: CreateTableArgs): Promise<CreateTableResult> {
+    deprecationWarning("table.create", "uploads.create");
     return this.post<CreateTableResult>(
       "table/create",
       withDefaults<CreateTableArgs>(args, { description: "", is_private: false }),
@@ -59,8 +64,10 @@ export class TableAPI extends Router {
    * Delete a Dune table with the specified name and namespace.
    *
    * To be able to delete a table, it must have been created with the /create endpoint.
+   * @deprecated Use uploads.delete() instead. The /v1/table endpoints are deprecated in favor of /v1/uploads.
    */
   async delete(args: DeleteTableArgs): Promise<DeleteTableResult> {
+    deprecationWarning("table.delete", "uploads.delete");
     const route = `table/${args.namespace}/${args.table_name}`;
     return this._delete<DeleteTableResult>(route);
   }
@@ -73,8 +80,10 @@ export class TableAPI extends Router {
    * - The file has to have the same schema as the table
    * @param args
    * @returns
+   * @deprecated Use uploads.insert() instead. The /v1/table endpoints are deprecated in favor of /v1/uploads.
    */
   async insert(args: InsertTableArgs): Promise<InsertTableResult> {
+    deprecationWarning("table.insert", "uploads.insert");
     return this.post<InsertTableResult>(
       `table/${args.namespace}/${args.table_name}/insert`,
       args.data,

src/api/uploads.ts

@@ -0,0 +1,60 @@
+import { Router } from "./router";
+import {
+  CreateTableResult,
+  SuccessResponse,
+  UploadCSVArgs,
+  CreateTableArgs,
+  InsertTableArgs,
+  InsertTableResult,
+  DeleteTableArgs,
+  DeleteTableResult,
+  TableListResponse,
+  ListUploadsArgs,
+  ClearTableArgs,
+  TableClearResponse,
+  DuneError,
+} from "../types";
+import { withDefaults } from "../utils";
+
+export class UploadsAPI extends Router {
+  async list(args?: ListUploadsArgs): Promise<TableListResponse> {
+    return this._get<TableListResponse>("uploads", args);
+  }
+
+  async create(args: CreateTableArgs): Promise<CreateTableResult> {
+    return this.post<CreateTableResult>(
+      "uploads",
+      withDefaults<CreateTableArgs>(args, { description: "", is_private: false }),
+    );
+  }
+
+  async uploadCsv(args: UploadCSVArgs): Promise<boolean> {
+    const response = await this.post<SuccessResponse>("uploads/csv", args);
+    try {
+      return Boolean(response.success);
+    } catch (error: unknown) {
+      console.error(
+        `Upload CSV Error ${error instanceof Error ? error.message : String(error)}`,
+      );
+      throw new DuneError(`UploadCsvResponse ${JSON.stringify(response)}`);
+    }
+  }
+
+  async delete(args: DeleteTableArgs): Promise<DeleteTableResult> {
+    const route = `uploads/${args.namespace}/${args.table_name}`;
+    return this._delete<DeleteTableResult>(route);
+  }
+
+  async clear(args: ClearTableArgs): Promise<TableClearResponse> {
+    const route = `uploads/${args.namespace}/${args.table_name}/clear`;
+    return this.post<TableClearResponse>(route);
+  }
+
+  async insert(args: InsertTableArgs): Promise<InsertTableResult> {
+    return this.post<InsertTableResult>(
+      `uploads/${args.namespace}/${args.table_name}/insert`,
+      args.data,
+      args.content_type,
+    );
+  }
+}

src/index.ts

@@ -6,6 +6,8 @@ export {
   TableAPI,
   UsageAPI,
   PipelineAPI,
+  DatasetAPI,
+  UploadsAPI,
 } from "./api";
 export * from "./types";
 export { Paginator } from "./paginator";

src/types/requestArgs.ts

@@ -304,6 +304,31 @@ export interface InsertTableArgs {
   content_type: ContentType;
 }
 
+export interface ListDatasetsArgs {
+  /// Number of results to return (default 50, max 250)
+  limit?: number;
+  /// Offset for pagination
+  offset?: number;
+  /// Filter by owner handle
+  owner_handle?: string;
+  /// Filter by dataset types (comma-separated: transformation_view, transformation_table, uploaded_table, decoded_table, spell, dune_table)
+  type?: string;
+}
+
+export interface ListUploadsArgs {
+  /// Number of tables to return on a page. Default: 50, max: 10000
+  limit?: number;
+  /// Offset used for pagination. Negative values are treated as 0
+  offset?: number;
+}
+
+export interface ClearTableArgs {
+  /// The namespace of the table to clear (e.g. my_user).
+  namespace: string;
+  /// The name of the table to clear (e.g. interest_rates).
+  table_name: string;
+}
+
 export interface Options {
   /// The page size when retriving results.
   batchSize?: number;

src/types/response.ts

@@ -247,3 +247,68 @@ export interface UsageResponse {
   bytes_allowed: number;
   billing_periods: BillingPeriod[];
 }
+
+export interface DatasetColumnMetadata {
+  description?: string;
+  filtering_column?: boolean;
+}
+
+export interface DatasetColumn {
+  name: string;
+  type: string;
+  nullable?: boolean;
+  metadata?: DatasetColumnMetadata;
+}
+
+export interface DatasetOwner {
+  handle: string;
+  type: string;
+}
+
+export interface DatasetResponse {
+  full_name: string;
+  type: string;
+  columns: DatasetColumn[];
+  owner: DatasetOwner;
+  is_private: boolean;
+  created_at: string;
+  updated_at: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface ListDatasetsResponse {
+  datasets: DatasetResponse[];
+  total: number;
+}
+
+export interface TableOwner {
+  handle: string;
+  type: string;
+}
+
+export interface TableColumnInfo {
+  name: string;
+  type: string;
+  nullable?: boolean;
+  metadata?: Record<string, unknown>;
+}
+
+export interface TableListElement {
+  full_name: string;
+  is_private: boolean;
+  owner: TableOwner;
+  columns: TableColumnInfo[];
+  table_size_bytes?: string;
+  created_at: string;
+  updated_at: string;
+  purged_at?: string;
+}
+
+export interface TableListResponse {
+  tables: TableListElement[];
+  next_offset?: number;
+}
+
+export interface TableClearResponse {
+  message: string;
+}

tests/e2e/dataset.spec.ts

@@ -0,0 +1,60 @@
+import log from "loglevel";
+import { DatasetAPI } from "../../src/api";
+
+log.setLevel("silent", true);
+
+const API_KEY = process.env.DUNE_API_KEY!;
+
+describe("Dataset API", () => {
+  let datasetClient: DatasetAPI;
+
+  beforeAll(() => {
+    datasetClient = new DatasetAPI(API_KEY);
+  });
+
+  it("lists datasets with owner filter", async () => {
+    const response = await datasetClient.list({
+      limit: 5,
+      owner_handle: "dune",
+    });
+
+    expect(response.datasets).toBeDefined();
+    expect(Array.isArray(response.datasets)).toBe(true);
+
+    if (response.datasets.length > 0) {
+      expect(response.datasets[0].owner.handle).toBe("dune");
+    }
+  });
+
+  it("lists datasets with owner filter", async () => {
+    const response = await datasetClient.list({
+      limit: 1,
+      type: "materialized_view",
+    });
+
+    expect(response.datasets).toBeDefined();
+    expect(Array.isArray(response.datasets)).toBe(true);
+
+    expect(response.datasets[0].type).toBe("materialized_view");
+  });
+
+  it("gets dataset by slug", async () => {
+    const dataset = await datasetClient.getBySlug("dex.trades");
+
+    expect(dataset).toHaveProperty("full_name");
+    expect(dataset.full_name).toContain("dex.trades");
+    expect(dataset).toHaveProperty("type");
+    expect(dataset).toHaveProperty("columns");
+    expect(dataset).toHaveProperty("owner");
+    expect(dataset).toHaveProperty("is_private");
+    expect(dataset).toHaveProperty("created_at");
+    expect(dataset).toHaveProperty("updated_at");
+
+    expect(Array.isArray(dataset.columns)).toBe(true);
+    expect(dataset.columns.length).toBeGreaterThan(0);
+
+    const firstColumn = dataset.columns[0];
+    expect(firstColumn).toHaveProperty("name");
+    expect(firstColumn).toHaveProperty("type");
+  });
+});