Dataset graphql resolver (#5558)

* feat(datasets): change catalog graphql to dataset

* chore(graphql): run codegen

Author: IcaroG

apps/frontend/app/api/v1/osograph/schema.graphql

@@ -81,18 +81,21 @@ type Invitation @key(fields: "id") {
 }
 
 """
-Catalog entity - represents a data catalog accessible to the user
+Dataset entity - represents a dataset within an organization
 """
-type Catalog @key(fields: "name") {
-  name: String!
-  schemas: [Schema!]!
-}
-
-"""
-Schema entity - represents a schema within a catalog
-"""
-type Schema {
+type Dataset @key(fields: "id") {
+  id: ID!
+  orgId: ID!
+  createdAt: DateTime!
+  updatedAt: DateTime!
+  deletedAt: DateTime
   name: String!
+  displayName: String!
+  description: String
+  catalog: String!
+  schema: String!
+  createdBy: ID!
+  isPublic: Boolean!
   tables: [Table!]!
 }
 
@@ -134,11 +137,11 @@ type Query {
   osoApp_invitation(id: ID!): Invitation
 
   """
-  Get list of catalogs for the current user
+  Get list of datasets for a given organization
   """
-  osoApp_myCatalogs(orgName: String!): [Catalog!]!
+  osoApp_orgDatasets(orgName: String!): [Dataset!]!
 
-  osoApp_tableColumns(
+  osoApp_datasetTableMetadata(
     orgName: String!
     catalogName: String!
     schemaName: String!

apps/frontend/app/api/v1/osograph/schema/resolvers/catalog.ts renamed to apps/frontend/app/api/v1/osograph/schema/resolvers/dataset.ts

@@ -5,7 +5,10 @@ import {
   requireOrgMembership,
   type GraphQLContext,
 } from "@/app/api/v1/osograph/utils/auth";
-import { ServerErrors } from "@/app/api/v1/osograph/utils/errors";
+import {
+  ResourceErrors,
+  ServerErrors,
+} from "@/app/api/v1/osograph/utils/errors";
 import { signTrinoJWT } from "@/lib/auth/auth";
 import { getTrinoClient } from "@/lib/clients/trino";
 import {
@@ -17,10 +20,12 @@ import {
 } from "@/lib/types/catalog";
 import { logger } from "@/lib/logger";
 import { assert } from "@opensource-observer/utils";
+import { createAdminClient } from "@/lib/supabase/admin";
+import { DatasetsRow } from "@/lib/types/schema-types";
 
-export const catalogResolvers: GraphQLResolverModule<GraphQLContext> = {
+export const datasetResolver: GraphQLResolverModule<GraphQLContext> = {
   Query: {
-    osoApp_myCatalogs: async (
+    osoApp_orgDatasets: async (
       _: unknown,
       { orgName }: { orgName: string },
       context: GraphQLContext,
@@ -104,11 +109,36 @@ export const catalogResolvers: GraphQLResolverModule<GraphQLContext> = {
             },
           ),
       );
-      const filteredResults = results.filter((result) => result !== null);
-      return filteredResults;
+      const filteredResults = results.filter(
+        (result): result is Catalog => result !== null,
+      );
+
+      const datasets = await getOrganizationDatasets(organization.id);
+
+      const tablesByCatalogAndSchema = filteredResults.reduce(
+        (acc, catalog) => {
+          for (const schema of catalog.schemas) {
+            const key = `${catalog.name}.${schema.name}`;
+            acc[key] = schema.tables;
+          }
+          return acc;
+        },
+        {} as Record<string, { name: string }[]>,
+      );
+
+      const response = datasets.map((dataset) => {
+        const key = `${dataset.catalog}.${dataset.schema}`;
+        return {
+          ...dataset,
+          tables: tablesByCatalogAndSchema[key] ?? [],
+        };
+      });
+      console.log("orgDatasets response:", response);
+
+      return response;
     },
 
-    osoApp_tableColumns: async (
+    osoApp_datasetTableMetadata: async (
       _: unknown,
       {
         orgName,
@@ -174,4 +204,41 @@ export const catalogResolvers: GraphQLResolverModule<GraphQLContext> = {
       return filteredResults;
     },
   },
+  Dataset: {
+    id: (parent: DatasetsRow) => parent.id,
+    orgId: (parent: DatasetsRow) => parent.org_id,
+    createdAt: (parent: DatasetsRow) => parent.created_at,
+    updatedAt: (parent: DatasetsRow) => parent.updated_at,
+    deletedAt: (parent: DatasetsRow) => parent.deleted_at,
+    name: (parent: DatasetsRow) => parent.name,
+    displayName: (parent: DatasetsRow) => parent.display_name,
+    description: (parent: DatasetsRow) => parent.description,
+    catalog: (parent: DatasetsRow) => parent.catalog,
+    schema: (parent: DatasetsRow) => parent.schema,
+    createdBy: (parent: DatasetsRow) => parent.created_by,
+    isPublic: (parent: DatasetsRow) => parent.is_public,
+  },
 };
+
+async function getOrganizationDatasets(orgId: string): Promise<DatasetsRow[]> {
+  const supabase = createAdminClient();
+
+  const { data, error } = await supabase
+    .from("datasets_by_organization")
+    .select("datasets(*)")
+    .eq("org_id", orgId);
+
+  if (error) {
+    throw ResourceErrors.notFound("Datasets", `org_id: ${orgId}`);
+  }
+
+  const { data: ownedDatasets, error: ownedError } = await supabase
+    .from("datasets")
+    .select("*")
+    .eq("org_id", orgId);
+  if (ownedError) {
+    throw ResourceErrors.notFound("Datasets", `org_id: ${orgId}`);
+  }
+
+  return data.map((d) => d.datasets).concat(ownedDatasets);
+}

apps/frontend/app/api/v1/osograph/schema/resolvers/index.ts

@@ -5,7 +5,7 @@ import { organizationResolvers } from "@/app/api/v1/osograph/schema/resolvers/or
 import { userResolvers } from "@/app/api/v1/osograph/schema/resolvers/user";
 import { memberResolvers } from "@/app/api/v1/osograph/schema/resolvers/member";
 import { invitationResolvers } from "@/app/api/v1/osograph/schema/resolvers/invitation";
-import { catalogResolvers } from "@/app/api/v1/osograph/schema/resolvers/catalog";
+import { datasetResolver } from "@/app/api/v1/osograph/schema/resolvers/dataset";
 
 const dateTimeScalar = new GraphQLScalarType({
   name: "DateTime",
@@ -39,7 +39,7 @@ export const resolvers: GraphQLResolverMap<GraphQLContext> = {
     ...userResolvers.Query,
     ...organizationResolvers.Query,
     ...invitationResolvers.Query,
-    ...catalogResolvers.Query,
+    ...datasetResolver.Query,
   },
   Mutation: {
     ...userResolvers.Mutation,
@@ -50,4 +50,5 @@ export const resolvers: GraphQLResolverMap<GraphQLContext> = {
   Organization: organizationResolvers.Organization,
   OrganizationMember: memberResolvers.OrganizationMember,
   Invitation: invitationResolvers.Invitation,
+  Dataset: datasetResolver.Dataset,
 };

apps/frontend/lib/graphql/generated/graphql.ts

@@ -992,13 +992,6 @@ export type CapturedLogsMetadata = {
   stdoutLocation?: Maybe<Scalars["String"]["output"]>;
 };
 
-/** Catalog entity - represents a data catalog accessible to the user */
-export type Catalog = {
-  __typename?: "Catalog";
-  name: Scalars["String"]["output"];
-  schemas: Array<Schema>;
-};
-
 /**
  * What change an asset has undergone between two deployments. Used
  *     in distinguishing asset definition changes in branch deployment and
@@ -1363,6 +1356,24 @@ export type DagsterTypeOrError =
   | PythonError
   | RegularDagsterType;
 
+/** Dataset entity - represents a dataset within an organization */
+export type Dataset = {
+  __typename?: "Dataset";
+  catalog: Scalars["String"]["output"];
+  createdAt: Scalars["DateTime"]["output"];
+  createdBy: Scalars["ID"]["output"];
+  deletedAt?: Maybe<Scalars["DateTime"]["output"]>;
+  description?: Maybe<Scalars["String"]["output"]>;
+  displayName: Scalars["String"]["output"];
+  id: Scalars["ID"]["output"];
+  isPublic: Scalars["Boolean"]["output"];
+  name: Scalars["String"]["output"];
+  orgId: Scalars["ID"]["output"];
+  schema: Scalars["String"]["output"];
+  tables: Array<Table>;
+  updatedAt: Scalars["DateTime"]["output"];
+};
+
 export type DefaultPartitionStatuses = {
   __typename?: "DefaultPartitionStatuses";
   failedPartitions: Array<Scalars["String"]["output"]>;
@@ -5256,17 +5267,17 @@ export type Query = {
   locationStatusesOrError: WorkspaceLocationStatusEntriesOrError;
   /** Retrieve event logs after applying a run id filter, cursor, and limit. */
   logsForRun: EventConnectionOrError;
+  osoApp_datasetTableMetadata: Array<Column>;
   /** Get a specific invitation by ID */
   osoApp_invitation?: Maybe<Invitation>;
   /** Get the current authenticated user's profile */
   osoApp_me: User;
-  /** Get list of catalogs for the current user */
-  osoApp_myCatalogs: Array<Catalog>;
   /** Get invitations sent to the current user's email */
   osoApp_myInvitations: Array<Invitation>;
+  /** Get list of datasets for a given organization */
+  osoApp_orgDatasets: Array<Dataset>;
   /** Get a specific organization by name */
   osoApp_organization?: Maybe<Organization>;
-  osoApp_tableColumns: Array<Column>;
   oso_artifactsByCollectionV1?: Maybe<Array<Oso_ArtifactsByCollectionV1>>;
   oso_artifactsByProjectV1?: Maybe<Array<Oso_ArtifactsByProjectV1>>;
   oso_artifactsByUserV1?: Maybe<Array<Oso_ArtifactsByUserV1>>;
@@ -5541,6 +5552,14 @@ export type QueryLogsForRunArgs = {
   runId: Scalars["ID"]["input"];
 };
 
+/** The root for all queries to retrieve data from the Dagster instance. */
+export type QueryOsoApp_DatasetTableMetadataArgs = {
+  catalogName: Scalars["String"]["input"];
+  orgName: Scalars["String"]["input"];
+  schemaName: Scalars["String"]["input"];
+  tableName: Scalars["String"]["input"];
+};
+
 /** The root for all queries to retrieve data from the Dagster instance. */
 export type QueryOsoApp_InvitationArgs = {
   id: Scalars["ID"]["input"];
@@ -5552,15 +5571,13 @@ export type QueryOsoApp_MyInvitationsArgs = {
 };
 
 /** The root for all queries to retrieve data from the Dagster instance. */
-export type QueryOsoApp_OrganizationArgs = {
+export type QueryOsoApp_OrgDatasetsArgs = {
   orgName: Scalars["String"]["input"];
 };
 
 /** The root for all queries to retrieve data from the Dagster instance. */
-export type QueryOsoApp_TableColumnsArgs = {
-  catalogName: Scalars["String"]["input"];
-  schemaName: Scalars["String"]["input"];
-  tableName: Scalars["String"]["input"];
+export type QueryOsoApp_OrganizationArgs = {
+  orgName: Scalars["String"]["input"];
 };
 
 /** The root for all queries to retrieve data from the Dagster instance. */
@@ -6965,13 +6982,6 @@ export type SchedulesOrError =
   | RepositoryNotFoundError
   | Schedules;
 
-/** Schema entity - represents a schema within a catalog */
-export type Schema = {
-  __typename?: "Schema";
-  name: Scalars["String"]["output"];
-  tables: Array<Table>;
-};
-
 export type SelectorTypeConfigError = PipelineConfigValidationError & {
   __typename?: "SelectorTypeConfigError";
   incomingFields: Array<Scalars["String"]["output"]>;
@@ -7680,7 +7690,7 @@ export type WrappingDagsterType = {
 };
 
 export type _Entity =
-  | Catalog
+  | Dataset
   | Invitation
   | Organization
   | OrganizationMember

apps/frontend/supabase/seed.sql

@@ -50,3 +50,7 @@ VALUES
     ((SELECT id FROM organizations WHERE org_name = 'oso' LIMIT 1), (SELECT id FROM auth.users WHERE email = '[email protected]' LIMIT 1), 'template-value-chain', '{"cells":[{"id":"1","type":"python","content":"print(\"Hello from value-chain notebook!\")"}]}');
 
 INSERT INTO resource_permissions (permission_level, notebook_id) SELECT 'read', id FROM notebooks;
+
+INSERT INTO datasets (org_id, created_by, name, display_name, description, catalog, "schema")
+VALUES
+    ((SELECT id FROM organizations WHERE org_name = 'oso' LIMIT 1), (SELECT id FROM auth.users WHERE email = '[email protected]' LIMIT 1), 'oso-dataset', 'oso-dataset', 'OSO dataset', 'iceberg', 'oso');