Datacube: Improvements in lakehouse consumer flow

.changeset/large-sails-juggle.md

@@ -0,0 +1,5 @@
+---
+'@finos/legend-server-lakehouse': patch
+---
+
+datacube: adding helper functions for consumer flow backward compatibility

.changeset/shy-donkeys-lay.md

@@ -0,0 +1,6 @@
+---
+'@finos/legend-application-data-cube': patch
+'@finos/legend-graph': patch
+---
+
+datacube: improving lakehouse consumer flow

packages/legend-application-data-cube/src/components/__test-utils__/LegendDataCubeStoreTestUtils.tsx

@@ -191,6 +191,19 @@ export const TEST__provideMockedLegendDataCubeEngine = async (customization?: {
     lakehouseIngestServerClient,
     'getIngestDefinitionGrammar',
   ).mockResolvedValue('test-ingest-definition-grammar');
+  createSpy(
+    lakehouseContractServerClient,
+    'getUserEntitlementEnvs',
+  ).mockResolvedValue({
+    total: 1,
+    users: [
+      {
+        name: 'TESTER',
+        userType: 'TEST_USER',
+        lakehouseEnvironment: 'testEnv1',
+      },
+    ],
+  });
 
   const graphManager =
     customization?.graphManager ??
@@ -217,6 +230,8 @@ export const TEST__provideMockedLegendDataCubeEngine = async (customization?: {
 export const mockLakehouseConsumerAdHocDataProduct = {
   _type: 'lakehouseConsumer',
   warehouse: 'TEST_WAREHOUSE',
+  // this environment gets overridden as we have runtime calls for fetching user entitlement envs
+  // this makes saving view user agnostic
   environment: 'dev-testEnv',
   paths: ['dataProduct::test_DataProduct', 'test_dataset'],
   origin: {

packages/legend-application-data-cube/src/components/__tests__/LegendDataCubeBuilder.data-cube-test.tsx

@@ -1516,7 +1516,7 @@ test(
     expect(runtimeElement?.runtimeValue).toBeInstanceOf(V1_LakehouseRuntime);
     const runtimeValue = runtimeElement?.runtimeValue as V1_LakehouseRuntime;
     expect(runtimeValue.warehouse).toBe('TEST_WAREHOUSE');
-    expect(runtimeValue.environment).toBe('dev-testEnv');
+    expect(runtimeValue.environment).toBe('dev-testEnv1');
   },
 );
 

packages/legend-application-data-cube/src/components/builder/LegendDataCubeSourceViewer.tsx

@@ -529,16 +529,16 @@ const LakehouseConsumerSourceViewer = observer(
                 </button>
               </div>
             )}
-            {source.environment && (
+            {source.userEnvironment && (
               <div className="mt-2 flex h-6 w-full">
                 <div className="flex h-full w-[calc(100%_-_20px)] items-center border border-r-0 border-neutral-400 px-1.5">
-                  {source.environment}
+                  {source.userEnvironment}
                 </div>
                 <button
                   className="flex aspect-square h-full w-6 items-center justify-center border border-neutral-400 bg-neutral-300 hover:brightness-95"
                   onClick={() => {
                     store.application.clipboardService
-                      .copyTextToClipboard(source.environment)
+                      .copyTextToClipboard(source.userEnvironment)
                       .catch((error) =>
                         store.alertService.alertUnhandledError(error),
                       );

packages/legend-application-data-cube/src/components/builder/source/LakehouseConsumerDataCubeSourceBuilder.tsx

@@ -18,52 +18,58 @@ import { DataCubeCodeEditor, FormTextInput } from '@finos/legend-data-cube';
 import { CustomSelectorInput } from '@finos/legend-art';
 import { useAuth } from 'react-oidc-context';
 import { assertErrorThrown, guaranteeNonNullable } from '@finos/legend-shared';
-import { useEffect } from 'react';
+import { useEffect, type ReactNode } from 'react';
 import type { LakehouseConsumerDataCubeSourceBuilderState } from '../../../stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.js';
 import { useLegendDataCubeBuilderStore } from '../LegendDataCubeBuilderStoreProvider.js';
 import {
-  buildIngestDeploymentServerConfigOption,
-  type IngestDeploymentServerConfigOption,
-} from '@finos/legend-server-lakehouse';
-import { V1_IngestEnvironmentClassification } from '@finos/legend-graph';
+  V1_EntitlementsLakehouseEnvironmentType,
+  type V1_EntitlementsDataProductLite,
+} from '@finos/legend-graph';
 
 export const LakehouseConsumerDataCubeSourceBuilder: React.FC<{
   sourceBuilder: LakehouseConsumerDataCubeSourceBuilderState;
 }> = observer(({ sourceBuilder }) => {
   const auth = useAuth();
   const store = useLegendDataCubeBuilderStore();
-  const envOptions = sourceBuilder.environments
-    .map(buildIngestDeploymentServerConfigOption)
-    // not include dev
-    .filter(
-      (config) =>
-        config.value.environmentClassification !==
-        V1_IngestEnvironmentClassification.DEV,
-    )
-    .sort(
-      (a, b) =>
-        a.value.environmentName.localeCompare(b.value.environmentName) ||
-        a.value.environmentClassification.localeCompare(
-          b.value.environmentClassification,
-        ),
-    );
 
-  const selectedEnvOption = sourceBuilder.selectedEnvironment
-    ? buildIngestDeploymentServerConfigOption(sourceBuilder.selectedEnvironment)
-    : null;
-  const onEnvChange = (newValue: IngestDeploymentServerConfigOption | null) => {
-    sourceBuilder.setSelectedEnvironment(newValue?.value ?? undefined);
-    sourceBuilder
-      .fetchAccessPoints()
-      .catch((error) => store.alertService.alertUnhandledError(error));
+  const renderDataProductLabel = (
+    dataProduct: V1_EntitlementsDataProductLite,
+  ): ReactNode => {
+    const title = dataProduct.title;
+    const id = guaranteeNonNullable(dataProduct.id);
+    // If title is empty, show id only. Otherwise show title (prominent) and id (muted)
+    // Main text: slightly larger and normal weight. Subtext: grey and slightly smaller.
+    return (
+      <div className="flex w-full flex-col items-start">
+        <div className="w-full whitespace-normal break-words text-base font-normal text-black">
+          {title ?? id}
+        </div>
+        {title ? (
+          <div className="mt-1 w-full truncate text-sm text-neutral-500">
+            {id}
+          </div>
+        ) : null}
+      </div>
+    );
+  };
+  const renderDataProductMainText = (
+    dataProduct: V1_EntitlementsDataProductLite,
+  ): ReactNode => {
+    const title = dataProduct.title;
+    const id = guaranteeNonNullable(dataProduct.id);
+    // Ensure selected value is vertically centered inside the control by
+    // making the label container full-height and centering its content.
+    return (
+      <div className="flex h-full w-full items-center whitespace-normal break-words text-base font-normal text-black">
+        {title ?? id}
+      </div>
+    );
   };
 
   useEffect(() => {
     sourceBuilder.reset();
     try {
       sourceBuilder.loadDataProducts(auth.user?.access_token);
-      sourceBuilder.fetchEnvironment(auth.user?.access_token);
-      sourceBuilder.fetchUserEntitlementEnvs(auth.user?.access_token);
     } catch (error) {
       assertErrorThrown(error);
       store.alertService.alertUnhandledError(error);
@@ -74,104 +80,161 @@ export const LakehouseConsumerDataCubeSourceBuilder: React.FC<{
     <div className="flex h-full w-full">
       <div className="m-3 flex w-full flex-col items-stretch gap-2 text-neutral-500">
         <div className="query-setup__wizard__group mt-3">
-          <div className="query-setup__wizard__group__title">Data Product</div>
+          <div className="query-setup__wizard__group__title">Mode</div>
           <CustomSelectorInput
-            className="query-setup__wizard__selector text-nowrap"
-            options={sourceBuilder.dataProducts.map((dataProduct) => ({
-              label: guaranteeNonNullable(dataProduct.id),
-              value: guaranteeNonNullable(dataProduct.fullPath),
-            }))}
-            disabled={
-              sourceBuilder.dataProductLoadingState.isInProgress ||
-              sourceBuilder.dataProductLoadingState.hasFailed
-            }
-            isLoading={sourceBuilder.dataProductLoadingState.isInProgress}
-            onChange={(newValue: { label: string; value: string } | null) => {
-              sourceBuilder.setSelectedDataProduct(newValue?.value ?? '');
-              sourceBuilder
-                .fetchDataProduct(auth.user?.access_token)
-                .catch((error) =>
-                  store.alertService.alertUnhandledError(error),
-                );
+            className="query-setup__wizard__selector"
+            options={[
+              {
+                label: 'Production',
+                value: V1_EntitlementsLakehouseEnvironmentType.PRODUCTION,
+              },
+              {
+                label: 'Production (parallel)',
+                value:
+                  V1_EntitlementsLakehouseEnvironmentType.PRODUCTION_PARALLEL,
+              },
+            ]}
+            onChange={(
+              newVal: {
+                label: string;
+                value: V1_EntitlementsLakehouseEnvironmentType;
+              } | null,
+            ) => {
+              sourceBuilder.setEnvMode(newVal?.value);
+              sourceBuilder.setSelectedDataProduct(undefined);
             }}
             value={
-              sourceBuilder.selectedDataProduct
+              sourceBuilder.envMode
                 ? {
-                    label: sourceBuilder.selectedDataProduct,
-                    value: sourceBuilder.selectedDataProduct,
+                    label:
+                      sourceBuilder.envMode ===
+                      V1_EntitlementsLakehouseEnvironmentType.PRODUCTION
+                        ? 'Production'
+                        : sourceBuilder.envMode ===
+                            V1_EntitlementsLakehouseEnvironmentType.PRODUCTION_PARALLEL
+                          ? 'Production (parallel)'
+                          : 'Development',
+                    value: sourceBuilder.envMode,
                   }
                 : null
             }
-            placeholder={`Choose a Data Product`}
-            isClearable={false}
+            placeholder="Choose mode"
+            isClearable={true}
             escapeClearsValue={true}
           />
         </div>
-        {sourceBuilder.environments.length > 0 && (
+        {sourceBuilder.envMode && (
           <div className="query-setup__wizard__group mt-3">
-            <div className="query-setup__wizard__group__title">Environment</div>
+            <div className="query-setup__wizard__group__title">
+              Data Product
+            </div>
             <CustomSelectorInput
               className="query-setup__wizard__selector text-nowrap"
-              options={envOptions}
+              // give the option rows more height to accommodate wrapped titles
+              optionCustomization={{ rowHeight: 56 }}
+              options={sourceBuilder.filteredDataProducts.map((dataProduct) => {
+                const title = dataProduct.title;
+                const id = guaranteeNonNullable(dataProduct.id);
+                return {
+                  label: renderDataProductLabel(dataProduct),
+                  // include a searchable text so react-select filter can match id/title
+                  searchText: `${title} ${id}`.trim(),
+                  value: guaranteeNonNullable(dataProduct),
+                };
+              })}
+              filterOption={(option, rawInput) => {
+                const input = rawInput.toLowerCase();
+                // option.data may contain our custom searchText
+                const data = option.data as { searchText?: string } | undefined;
+
+                // Prefer a non-empty searchText from data when available
+                const searchTextSource =
+                  data?.searchText && data.searchText.trim() !== ''
+                    ? data.searchText.toLowerCase().trim()
+                    : ''.trim();
+
+                // If user hasn't typed anything, show all options
+                if (input === '') {
+                  return true;
+                }
+
+                return searchTextSource.includes(input);
+              }}
               disabled={
-                sourceBuilder.ingestEnvLoadingState.isInProgress ||
-                sourceBuilder.ingestEnvLoadingState.hasFailed ||
-                !sourceBuilder.selectedDataProduct
+                sourceBuilder.dataProductLoadingState.isInProgress ||
+                sourceBuilder.dataProductLoadingState.hasFailed
               }
-              isLoading={sourceBuilder.ingestEnvLoadingState.isInProgress}
+              isLoading={sourceBuilder.dataProductLoadingState.isInProgress}
               onChange={(
-                newValue: IngestDeploymentServerConfigOption | null,
+                newValue: {
+                  label: ReactNode;
+                  searchText?: string;
+                  value: V1_EntitlementsDataProductLite;
+                } | null,
               ) => {
-                onEnvChange(newValue);
+                sourceBuilder.setSelectedDataProduct(newValue?.value);
+                sourceBuilder
+                  .fetchDataProduct(auth.user?.access_token)
+                  .catch((error) =>
+                    store.alertService.alertUnhandledError(error),
+                  );
               }}
-              value={selectedEnvOption}
-              placeholder={`Choose an Environment`}
+              value={
+                sourceBuilder.selectedDataProduct
+                  ? {
+                      label: renderDataProductMainText(
+                        sourceBuilder.selectedDataProduct,
+                      ),
+                      searchText:
+                        `${sourceBuilder.selectedDataProduct.title ?? ''} ${sourceBuilder.selectedDataProduct.id}`.trim(),
+                      value: sourceBuilder.selectedDataProduct,
+                    }
+                  : null
+              }
+              placeholder={`Choose a Data Product`}
               isClearable={false}
               escapeClearsValue={true}
             />
           </div>
         )}
-        {sourceBuilder.accessPoints.length > 0 &&
-          sourceBuilder.selectedEnvironment && (
-            <div className="query-setup__wizard__group mt-2">
-              <div className="query-setup__wizard__group__title">
-                Access Point
-              </div>
-              <CustomSelectorInput
-                className="query-setup__wizard__selector"
-                options={sourceBuilder.accessPoints.map((accessPoint) => ({
-                  label: accessPoint,
-                  value: accessPoint,
-                }))}
-                disabled={false}
-                isLoading={false}
-                onChange={(
-                  newValue: { label: string; value: string } | null,
-                ) => {
-                  const accessPoint = newValue?.value ?? '';
-                  sourceBuilder.setSelectedAccessPoint(accessPoint);
-                  sourceBuilder.setWarehouse(
-                    sourceBuilder.DEFAULT_CONSUMER_WAREHOUSE,
-                  );
-                  sourceBuilder
-                    .initializeQuery()
-                    .catch((error) =>
-                      store.alertService.alertUnhandledError(error),
-                    );
-                }}
-                value={
-                  sourceBuilder.selectedAccessPoint
-                    ? {
-                        label: sourceBuilder.selectedAccessPoint,
-                        value: sourceBuilder.selectedAccessPoint,
-                      }
-                    : null
-                }
-                isClearable={false}
-                escapeClearsValue={true}
-              />
+        {sourceBuilder.accessPoints.length > 0 && (
+          <div className="query-setup__wizard__group mt-2">
+            <div className="query-setup__wizard__group__title">
+              Access Point
             </div>
-          )}
+            <CustomSelectorInput
+              className="query-setup__wizard__selector"
+              options={sourceBuilder.accessPoints.map((accessPoint) => ({
+                label: accessPoint,
+                value: accessPoint,
+              }))}
+              disabled={false}
+              isLoading={false}
+              onChange={(newValue: { label: string; value: string } | null) => {
+                const accessPoint = newValue?.value ?? '';
+                sourceBuilder.setSelectedAccessPoint(accessPoint);
+                sourceBuilder.setWarehouse(
+                  sourceBuilder.DEFAULT_CONSUMER_WAREHOUSE,
+                );
+                sourceBuilder
+                  .initializeQuery()
+                  .catch((error) =>
+                    store.alertService.alertUnhandledError(error),
+                  );
+              }}
+              value={
+                sourceBuilder.selectedAccessPoint
+                  ? {
+                      label: sourceBuilder.selectedAccessPoint,
+                      value: sourceBuilder.selectedAccessPoint,
+                    }
+                  : null
+              }
+              isClearable={false}
+              escapeClearsValue={true}
+            />
+          </div>
+        )}
         {sourceBuilder.selectedAccessPoint && (
           <div className="query-setup__wizard__group mt-2">
             <div className="query-setup__wizard__group__title">Warehouse</div>