[Lens] move reference handling to server (#239029)

## Summary

Move Lens `reference` handling to server side.

Close #221943

## Details

Lens no longer relies on `references` from the `buildEmbeddable`. We now
inject/extract references on the server.

Lens duplicates `references` in it's own serialized state so we just
move them there and add the `savedObjectId` for by-ref Lens panels. Lens
still inject these `references` internally by type across the Lens code.

For now we still store the `references` duplicated in the Lens SO state,
but we can clean this up when we extract all the internal reference
handling.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] Review the [backport
guidelines](https://docs.google.com/document/d/1VyN5k91e5OVumlc0Gb9RPa3h1ewuPE705nRtioPiTvY/edit?usp=sharing)
and apply applicable `backport:*` labels.

---------

Co-authored-by: Marco Liberati <[email protected]>
Co-authored-by: dej611 <[email protected]>
Co-authored-by: kibanamachine <[email protected]>

Author: 3 people

src/platform/packages/shared/kbn-lens-common/embeddable/types.ts

@@ -7,6 +7,9 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import type { BehaviorSubject } from 'rxjs';
+
+import type { HasSerializedChildState } from '@kbn/presentation-containers';
 import type {
   AggregateQuery,
   ExecutionContextSearch,
@@ -44,6 +47,7 @@ import type {
   PublishingSubject,
   SerializedTitles,
   ViewMode,
+  useSearchApi,
 } from '@kbn/presentation-publishing';
 import type { Action } from '@kbn/ui-actions-plugin/public';
 import type {
@@ -126,9 +130,9 @@ export interface PreventableEvent {
   preventDefault(): void;
 }
 
-interface LensByValue {
-  // by-value
-  attributes?: Simplify<LensSavedObjectAttributes>;
+export interface LensByValueBase {
+  savedObjectId?: string; // really should be never but creates type issues
+  attributes?: LensSavedObjectAttributes;
 }
 
 export interface LensOverrides {
@@ -150,20 +154,22 @@ export interface LensOverrides {
 /**
  * Lens embeddable props broken down by type
  */
-
-export interface LensByReference {
-  // by-reference
+interface LensByReferenceBase {
   savedObjectId?: string;
+  attributes?: never;
 }
 
 interface ContentManagementProps {
   sharingSavedObjectProps?: SharingSavedObjectProps;
   managed?: boolean;
 }
 
-export type LensPropsVariants = (LensByValue & LensByReference) & {
+interface LensWithReferences {
+  /**
+   * @deprecated use `state.attributes.references`
+   */
   references?: Reference[];
-};
+}
 
 export interface ViewInDiscoverCallbacks extends LensApiProps {
   canViewUnderlyingData$: PublishingSubject<boolean>;
@@ -259,16 +265,29 @@ interface LensRequestHandlersProps {
  * * Panel settings
  * * other props from the embeddable
  */
-export type LensSerializedState = Simplify<
-  LensPropsVariants &
-    LensOverrides &
+type LensSerializedSharedState = Simplify<
+  LensOverrides &
+    LensWithReferences &
     LensUnifiedSearchContext &
     LensPanelProps &
     SerializedTitles &
     Omit<LensSharedProps, 'noPadding'> &
     Partial<DynamicActionsSerializedState> & { isNewPanel?: boolean }
 >;
 
+export type LensByValueSerializedState = Simplify<LensSerializedSharedState & LensByValueBase>;
+export type LensByRefSerializedState = Simplify<LensSerializedSharedState & LensByReferenceBase>;
+
+/**
+ * Combined properties of serialized state stored on dashboard panel
+ *
+ *  Includes:
+ * - Lens document state (for by-value)
+ * - Panel settings
+ * - other props from the embeddable
+ */
+export type LensSerializedState = LensByRefSerializedState | LensByValueSerializedState;
+
 /**
  * Custom props exposed on the Lens exported component
  */
@@ -516,3 +535,26 @@ export interface ESQLVariablesCompatibleDashboardApi {
   controlGroupApi$: PublishingSubject<Partial<CanAddNewPanel> | undefined>;
   children$: PublishingSubject<{ [key: string]: unknown }>;
 }
+
+type SearchApi = ReturnType<typeof useSearchApi>;
+
+interface GeneralLensApi {
+  searchSessionId$: BehaviorSubject<string | undefined>;
+  disabledActionIds$: BehaviorSubject<string[] | undefined>;
+  setDisabledActionIds: (ids: string[] | undefined) => void;
+  viewMode$: BehaviorSubject<ViewMode | undefined>;
+  settings: {
+    syncColors$: BehaviorSubject<boolean>;
+    syncCursor$: BehaviorSubject<boolean>;
+    syncTooltips$: BehaviorSubject<boolean>;
+  };
+  forceDSL?: boolean;
+  esqlVariables$: BehaviorSubject<ESQLControlVariable[] | undefined>;
+  hideTitle$: BehaviorSubject<boolean | undefined>;
+  reload$: BehaviorSubject<void>;
+}
+
+export type LensParentApi = SearchApi &
+  LensRuntimeState &
+  GeneralLensApi &
+  HasSerializedChildState<LensSerializedState>;

src/platform/packages/shared/kbn-lens-common/index.ts

@@ -252,8 +252,8 @@ export type {
   DocumentToExpressionReturnType,
   PreventableEvent,
   LensOverrides,
-  LensByReference,
-  LensPropsVariants,
+  LensByValueSerializedState,
+  LensByRefSerializedState,
   ViewInDiscoverCallbacks,
   IntegrationCallbacks,
   LensPublicCallbacks,
@@ -268,6 +268,7 @@ export type {
   LensHasEditPanel,
   LensInspectorAdapters,
   LensApi,
+  LensParentApi,
   LensInternalApi,
   ExpressionWrapperProps,
   GetStateType,
@@ -276,6 +277,7 @@ export type {
   TypedLensSerializedState,
   LensEmbeddableOutput,
   ESQLVariablesCompatibleDashboardApi,
+  LensByValueBase,
 } from './embeddable/types';
 export type {
   LensAppLocatorParams,

src/platform/packages/shared/kbn-lens-common/types.ts

@@ -79,11 +79,7 @@ import type { InspectorOptions } from '@kbn/inspector-plugin/public';
 import type { OnSaveProps } from '@kbn/saved-objects-plugin/public';
 import type { NavigateToLensContext } from './convert_to_lens_types';
 import type { LensAppLocator, MainHistoryLocationState } from './locator_types';
-import type {
-  LensRuntimeState,
-  LensSavedObjectAttributes,
-  StructuredDatasourceStates,
-} from './embeddable/types';
+import type { LensSavedObjectAttributes, StructuredDatasourceStates } from './embeddable/types';
 import type {
   DimensionLink,
   LensConfiguration,
@@ -144,14 +140,6 @@ export interface LensAttributesService {
     savedObjectId?: string
   ) => Promise<string>;
   checkForDuplicateTitle: (props: CheckDuplicateTitleProps) => Promise<{ isDuplicate: boolean }>;
-  injectReferences: (
-    runtimeState: LensRuntimeState,
-    references: Reference[] | undefined
-  ) => LensRuntimeState;
-  extractReferences: (runtimeState: LensRuntimeState) => {
-    rawState: LensRuntimeState;
-    references: Reference[];
-  };
 }
 
 export interface LensAppServices extends StartServices {

src/platform/packages/shared/kbn-lens-embeddable-utils/package.json

@@ -3,5 +3,6 @@
   "private": true,
   "version": "1.0.0",
   "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0",
-  "homepage": "https://docs.elastic.dev/kibana-dev-docs/lens/config-builder"
+  "homepage": "https://docs.elastic.dev/kibana-dev-docs/lens/config-builder",
+  "sideEffects": false
 }

x-pack/platform/plugins/shared/cases/server/common/utils.ts

@@ -15,6 +15,7 @@ import type {
 import { flatMap, uniqWith, xorWith } from 'lodash';
 import type { LensServerPluginSetup } from '@kbn/lens-plugin/server';
 import { addSpaceIdToPath } from '@kbn/spaces-plugin/common';
+import type { LensEmbeddableStateWithType } from '@kbn/lens-plugin/server/embeddable/types';
 import type {
   ActionsAttachmentPayload,
   AlertAttachmentPayload,
@@ -402,15 +403,16 @@ export const extractLensReferencesFromCommentString = (
   lensEmbeddableFactory: LensServerPluginSetup['lensEmbeddableFactory'],
   comment: string
 ): SavedObjectReference[] => {
-  const extract = lensEmbeddableFactory()?.extract;
+  const extract = lensEmbeddableFactory().extract;
 
   if (extract) {
     const parsedComment = parseCommentString(comment);
     const lensVisualizations = getLensVisualizations(parsedComment.children);
-    const flattenRefs = flatMap(
-      lensVisualizations,
-      (lensObject) => extract(lensObject)?.references ?? []
-    );
+    const flattenRefs = flatMap(lensVisualizations, (vis) => {
+      // TODO: Improve these types
+      const lensVis = vis as unknown as LensEmbeddableStateWithType;
+      return extract(lensVis).references;
+    });
 
     const uniqRefs = uniqWith(
       flattenRefs,

x-pack/platform/plugins/shared/lens/common/embeddable_factory/index.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloneDeep, uniqBy } from 'lodash';
+
+import type { Reference } from '@kbn/content-management-utils';
+
+import type { LensSerializedState } from '../../public';
+
+export const injectLensReferences = (
+  state: LensSerializedState,
+  references: Reference[] = []
+): LensSerializedState => {
+  const clonedState = cloneDeep(state);
+
+  if (clonedState.savedObjectId || !clonedState.attributes) {
+    return clonedState;
+  }
+
+  // TODO: find a way to cull erroneous dashboard references
+  const combinedReferences = uniqBy([...references, ...clonedState.attributes.references], 'name');
+
+  clonedState.attributes.references = combinedReferences;
+
+  return clonedState;
+};
+
+export const extractLensReferences = (
+  state: LensSerializedState
+): {
+  state: LensSerializedState;
+  references: Reference[];
+} => {
+  return {
+    state,
+    references: state.attributes?.references ?? state.references ?? [],
+  };
+};

x-pack/platform/plugins/shared/lens/common/references/index.ts

@@ -5,4 +5,21 @@
  * 2.0.
  */
 
-export { ConfigBuilderStub } from './config_builder_stub';
+import type { EnhancementsRegistry } from '@kbn/embeddable-plugin/common/enhancements/registry';
+
+import type { LensTransforms } from './types';
+import { getTransformIn } from './transform_in';
+import { getTransformOut } from './transform_out';
+
+export interface LensTransformDependencies {
+  transformEnhancementsIn?: EnhancementsRegistry['transformIn'];
+  transformEnhancementsOut?: EnhancementsRegistry['transformOut'];
+}
+
+export function getLensTransforms(deps: LensTransformDependencies): LensTransforms {
+  return {
+    transformIn: getTransformIn(deps),
+    transformOut: getTransformOut(deps),
+    transformOutInjectsReferences: true,
+  };
+}