Merge pull request #417 from dolthub/liuliu/add-pagination

liuliu-dev · web-flow · commit 5533cec981e9 · 2025-04-03T18:02:51.000-04:00
Web, Graphql: add pagination to sql select
diff --git a/graphql-server/schema.gql b/graphql-server/schema.gql
@@ -254,7 +254,7 @@ type SqlSelect {
   refName: String!
   queryString: String!
   columns: [Column!]!
-  rows: [Row!]!
+  rows: RowList!
   queryExecutionStatus: QueryExecutionStatus!
   queryExecutionMessage: String!
   warnings: [String!]
@@ -350,8 +350,8 @@ type Query {
   doltSchemas(refName: String!, databaseName: String!, schemaName: String): [SchemaItem!]!
   views(refName: String!, databaseName: String!, schemaName: String): [SchemaItem!]!
   doltProcedures(databaseName: String!, refName: String!): [SchemaItem!]!
-  sqlSelect(schemaName: String, refName: String!, databaseName: String!, queryString: String!): SqlSelect!
-  sqlSelectForCsvDownload(schemaName: String, refName: String!, databaseName: String!, queryString: String!): String!
+  sqlSelect(schemaName: String, refName: String!, databaseName: String!, queryString: String!, offset: Int): SqlSelect!
+  sqlSelectForCsvDownload(schemaName: String, refName: String!, databaseName: String!, queryString: String!, offset: Int): String!
   status(databaseName: String!, refName: String!): [Status!]!
   table(tableName: String!, refName: String!, databaseName: String!, schemaName: String): Table!
   tableNames(schemaName: String, refName: String!, databaseName: String!, filterSystemTables: Boolean): TableNames!
diff --git a/graphql-server/src/sqlSelects/sqlSelect.model.ts b/graphql-server/src/sqlSelects/sqlSelect.model.ts
@@ -3,6 +3,8 @@ import * as column from "../columns/column.model";
 import { RawRow } from "../queryFactory/types";
 import * as row from "../rows/row.model";
 import { QueryExecutionStatus } from "./sqlSelect.enums";
+import { ROW_LIMIT, getNextOffset } from "../utils";
+import { RowList } from "../rows/row.model";
 
 @ObjectType()
 export class SqlSelect {
@@ -21,8 +23,8 @@ export class SqlSelect {
   @Field(_type => [column.Column])
   columns: column.Column[];
 
-  @Field(_type => [row.Row])
-  rows: row.Row[];
+  @Field(_type => RowList)
+  rows: RowList;
 
   @Field(_type => QueryExecutionStatus)
   queryExecutionStatus: QueryExecutionStatus;
@@ -39,14 +41,15 @@ export function fromSqlSelectRow(
   refName: string,
   doltRows: RawRow | RawRow[] | undefined,
   queryString: string,
+  offset: number,
   warnings?: string[],
 ): SqlSelect {
   const res = {
     _id: `/databases/${databaseName}/refs/${refName}/queries/${queryString}`,
     databaseName,
     refName,
     queryString,
-    rows: [],
+    rows: { list: [] },
     columns: [],
     queryExecutionStatus: QueryExecutionStatus.Success,
     queryExecutionMessage: "",
@@ -71,11 +74,19 @@ export function fromSqlSelectRow(
   if (!doltRows.length) {
     return res;
   }
-
-  const rows: row.Row[] = doltRows.map(row.fromDoltRowRes);
+  const rows: row.Row[] = doltRows
+    .slice(offset, offset + ROW_LIMIT)
+    .map(row.fromDoltRowRes);
   const columns: column.Column[] = Object.keys(doltRows[0]).map(c => {
     return { name: c, isPrimaryKey: false, type: "unknown" };
   });
 
-  return { ...res, columns, rows };
+  return {
+    ...res,
+    columns,
+    rows: {
+      list: rows,
+      nextOffset: getNextOffset(doltRows.length, offset),
+    },
+  };
 }
diff --git a/graphql-server/src/sqlSelects/sqlSelect.resolver.ts b/graphql-server/src/sqlSelects/sqlSelect.resolver.ts
@@ -1,4 +1,4 @@
-import { Args, ArgsType, Field, Query, Resolver } from "@nestjs/graphql";
+import { Args, ArgsType, Field, Int, Query, Resolver } from "@nestjs/graphql";
 import { ConnectionProvider } from "../connections/connection.provider";
 import { RawRows } from "../queryFactory/types";
 import { getCellValue } from "../rows/row.model";
@@ -9,6 +9,9 @@ import { SqlSelect, fromSqlSelectRow } from "./sqlSelect.model";
 export class SqlSelectArgs extends RefMaybeSchemaArgs {
   @Field()
   queryString: string;
+
+  @Field(_type => Int, { nullable: true })
+  offset?: number;
 }
 
 @Resolver(_of => SqlSelect)
@@ -18,12 +21,15 @@ export class SqlSelectResolver {
   @Query(_returns => SqlSelect)
   async sqlSelect(@Args() args: SqlSelectArgs): Promise<SqlSelect> {
     const conn = this.conn.connection();
+    const offset = args.offset ?? 0;
     const res = await conn.getSqlSelect(args);
+
     return fromSqlSelectRow(
       args.databaseName,
       args.refName,
       res.rows,
       args.queryString,
+      offset,
       res.warnings,
     );
   }
diff --git a/web/renderer/components/FormSelectForRefs/tests/utils.test.ts b/web/renderer/components/FormSelectForRefs/tests/utils.test.ts
@@ -71,9 +71,9 @@ export async function waitForVisibleValueToClick(
     ).not.toBeInTheDocument(),
   );
 
-  expect(
-    await screen.findByLabelText(`single-value-${selected}`),
-  ).toBeVisible();
+  await waitFor(() => {
+    expect(screen.getByLabelText(`single-value-${selected}`)).toBeVisible();
+  });
 
   await user.click(screen.getByText(selected));
 }
diff --git a/web/renderer/components/SchemaFragment/index.tsx b/web/renderer/components/SchemaFragment/index.tsx
@@ -81,7 +81,7 @@ export default function SchemaFragment(props: Props) {
   return (
     <QueryHandler
       result={res}
-      render={data => <Inner {...props} rows={data.sqlSelect.rows} />}
+      render={data => <Inner {...props} rows={data.sqlSelect.rows.list} />}
     />
   );
 }
diff --git a/web/renderer/components/SqlDataTable/index.tsx b/web/renderer/components/SqlDataTable/index.tsx
@@ -1,86 +1,75 @@
-import {
-  ApolloClient,
-  ApolloError,
-  NormalizedCacheObject,
-} from "@apollo/client";
+import { ApolloClient } from "@apollo/client";
 import { Inner as InnerDataTable } from "@components/DataTable";
 import DataTableLayout from "@components/layouts/DataTableLayout";
 import { Button, Loader } from "@dolthub/react-components";
 import { useSessionQueryHistory } from "@dolthub/react-hooks";
-import {
-  ColumnForSqlDataTableFragment,
-  QueryExecutionStatus,
-  RowForDataTableFragment,
-  useSqlSelectForSqlDataTableQuery,
-} from "@gen/graphql-types";
 import { SqlQueryParams } from "@lib/params";
 import { useState } from "react";
 import { Maybe } from "@dolthub/web-utils";
+import { ApolloErrorType } from "@lib/errors/types";
 import SqlMessage from "./SqlMessage";
 import { isReadOnlyDatabaseRevisionError } from "./SqlMessage/utils";
 import WorkingDiff from "./WorkingDiff";
 import css from "./index.module.css";
 import useSqlQuery from "./useSqlQuery";
+import useSqlSelectRows, { RowsState } from "./useSqlSelectRows";
 
 type Props = {
   params: SqlQueryParams;
 };
 
 type InnerProps = Props & {
-  gqlError?: ApolloError;
-  executionStatus?: QueryExecutionStatus;
-  executionMessage?: string;
-  rows?: RowForDataTableFragment[];
-  columns?: ColumnForSqlDataTableFragment[];
-  client: ApolloClient<NormalizedCacheObject>;
+  fetchMore: () => Promise<void>;
+  state: RowsState;
+  hasMore: boolean;
+  client: ApolloClient<any>;
+  error?: ApolloErrorType;
   warnings?: Maybe<string[]>;
 };
 
 function Inner(props: InnerProps) {
-  const isMut = useSqlQuery(props.params, props.client, props.gqlError);
-  const msg = <SqlMessage {...props} rowsLen={props.rows?.length ?? 0} />;
+  const isMut = useSqlQuery(props.params, props.client, props.error);
+  const msg = (
+    <SqlMessage
+      params={props.params}
+      {...props.state}
+      rowsLen={props.state.rows.length}
+    />
+  );
   return (
     <>
       <DataTableLayout params={props.params}>
         <InnerDataTable
           params={props.params}
-          rows={props.rows}
-          columns={props.columns}
-          loadMore={async () => {}}
+          rows={props.state.rows}
+          columns={props.state.cols}
+          loadMore={props.fetchMore}
           message={msg}
           warnings={props.warnings}
+          hasMore={props.hasMore}
         />
       </DataTableLayout>
-      {isMut && !isReadOnlyDatabaseRevisionError(props.gqlError) && (
+      {isMut && !isReadOnlyDatabaseRevisionError(props.error) && (
         <WorkingDiff {...props} />
       )}
     </>
   );
 }
 
 function Query(props: Props) {
-  const { data, loading, error, client } = useSqlSelectForSqlDataTableQuery({
-    variables: {
-      databaseName: props.params.databaseName,
-      refName: props.params.refName,
-      queryString: props.params.q,
-      schemaName: props.params.schemaName,
-    },
-    fetchPolicy: "cache-and-network",
-  });
+  const { state, fetchMore, hasMore, loading, client } = useSqlSelectRows(
+    props.params,
+  );
 
   if (loading) return <Loader loaded={false} />;
 
   return (
     <Inner
-      gqlError={error}
-      executionStatus={data?.sqlSelect.queryExecutionStatus}
-      executionMessage={data?.sqlSelect.queryExecutionMessage}
-      rows={data?.sqlSelect.rows}
-      columns={data?.sqlSelect.columns}
-      params={props.params}
+      {...props}
+      state={state}
+      fetchMore={fetchMore}
+      hasMore={hasMore}
       client={client}
-      warnings={data?.sqlSelect.warnings}
     />
   );
 }
diff --git a/web/renderer/components/SqlDataTable/queries.ts b/web/renderer/components/SqlDataTable/queries.ts
@@ -12,25 +12,33 @@ export const SQL_SELECT_QUERY = gql`
     type
     sourceTable
   }
+  fragment RowListForSqlDataTableRows on RowList {
+    nextOffset
+    list {
+      ...RowForSqlDataTable
+    }
+  }
   query SqlSelectForSqlDataTable(
     $databaseName: String!
     $refName: String!
     $queryString: String!
     $schemaName: String
+    $offset: Int
   ) {
     sqlSelect(
       databaseName: $databaseName
       refName: $refName
       queryString: $queryString
       schemaName: $schemaName
+      offset: $offset
     ) {
       queryExecutionStatus
       queryExecutionMessage
       columns {
         ...ColumnForSqlDataTable
       }
       rows {
-        ...RowForSqlDataTable
+        ...RowListForSqlDataTableRows
       }
       warnings
     }
diff --git a/web/renderer/components/SqlDataTable/useSqlQuery.ts b/web/renderer/components/SqlDataTable/useSqlQuery.ts
@@ -1,10 +1,7 @@
-import {
-  ApolloClient,
-  ApolloError,
-  NormalizedCacheObject,
-} from "@apollo/client";
+import { ApolloClient, NormalizedCacheObject } from "@apollo/client";
 import { useSessionQueryHistory } from "@dolthub/react-hooks";
 import useSqlParser from "@hooks/useSqlParser";
+import { ApolloErrorType } from "@lib/errors/types";
 import { SqlQueryParams } from "@lib/params";
 import { refetchUpdateDatabaseQueriesCacheEvict } from "@lib/refetchQueries";
 import { databases } from "@lib/urls";
@@ -14,7 +11,7 @@ import { useEffect } from "react";
 export default function useSqlQuery(
   params: SqlQueryParams,
   client: ApolloClient<NormalizedCacheObject>,
-  gqlError?: ApolloError,
+  gqlError?: ApolloErrorType,
 ): boolean {
   const { isMutation } = useSqlParser();
   const router = useRouter();
diff --git a/web/renderer/components/SqlDataTable/useSqlSelectRows.ts b/web/renderer/components/SqlDataTable/useSqlSelectRows.ts
diff --git a/web/renderer/gen/graphql-types.tsx b/web/renderer/gen/graphql-types.tsx

Original file line number	Diff line number	Diff line change
`@@ -81,7 +81,7 @@ export default function SchemaFragment(props: Props) {`
`81`	`81`	`return (`
`82`	`82`	`<QueryHandler`
`83`	`83`	`result={res}`
`84`		`- render={data => <Inner {...props} rows={data.sqlSelect.rows} />}`
	`84`	`+ render={data => <Inner {...props} rows={data.sqlSelect.rows.list} />}`
`85`	`85`	`/>`
`86`	`86`	`);`
`87`	`87`	`}`