feat: use normalization CDK logic in Builder UI (#16708)

## What
In airbytehq/airbyte-python-cdk#599, I made some changes to the manifest normalizer in the CDK to make it ready to be used by the new Builder implementation. See that PR for details on those changes if interested.

This PR makes changes to both SchemaForm and the Builder to make it properly use the manifest normalizer.

At a high level, this looks like:
- When the Builder is first opened, `/resolve` is called with `should_normalize: true`, which causes the CDK to
  - Resolve all $refs
  - Find matching values for fields with `linkable: true` and point them to shared values placed in `definitions.shared`
  - Replace parent stream configs that just duplicate another stream with a $ref that points to that stream instead
  - Return the resulting manifest back to the UI
- Builder then passes this result to SchemaForm which now:
  - Resolves all $refs pointing to `refTargetPath`, which is the `definitions.shared` field
  - RefsHandler keeps track of where those $refs were pointing and propagates changes back and forth between the shared components (most of this logic was already implemented - this PR just cleans up some parts of this logic)
  - Parent stream $refs are left alone since they aren't pointing to `refTargetPath` - this is desired since a follow-up PR adds an override to the parent stream field to show a dropdown for selecting the parent stream name, and $refs are used for this in the UI logic
- When saving the manifest to the database, publishing, or exporting YAML, the Builder uses the new `exportValuesWithRefs` method from RefsHandler to re-insert the $refs for the linked fields so that the manifests stay concise and non-duplicative
- If the user switches to YAML mode and makes changes, the updated manifest is again passed to `/resolve` with `should_normalize: true` to get the normalized manifest, and then the new `resetFormAndRefState` method in RefsHandler is used to update the ref mappings and form state to reflect the newly-normalized manifest, as if it was loaded for the first time like the above bullets described. 

## Testing
Note that the deploy preview is not enough to test this since this is based on other branches that modify backend services.

To test this, you can deploy locally and see that parent stream $refs and linkable field $refs are properly returned by the `/resolve` call, and that the Builder UI handles this gracefully.

Author: lmossman

airbyte-webapp/src/components/connectorBuilder/MenuBar/DownloadYamlButton.tsx

@@ -1,13 +1,15 @@
 import snakeCase from "lodash/snakeCase";
 import { FormattedMessage } from "react-intl";
 
+import { useRefsHandler } from "components/forms/SchemaForm/RefsHandler";
 import { Button } from "components/ui/Button";
 import { FlexContainer } from "components/ui/Flex";
 import { Icon } from "components/ui/Icon";
 import { Tooltip } from "components/ui/Tooltip";
 
 import { Action, Namespace, useAnalyticsService } from "core/services/analytics";
 import { downloadFile, FILE_TYPE_DOWNLOAD } from "core/utils/file";
+import { removeEmptyProperties } from "core/utils/form";
 import { useConnectorBuilderFormState } from "services/connectorBuilder/ConnectorBuilderStateService";
 
 import styles from "./DownloadYamlButton.module.scss";
@@ -24,13 +26,14 @@ export const DownloadYamlButton: React.FC<DownloadYamlButtonProps> = ({ classNam
   const { validateAndTouch } = useBuilderErrors();
   const connectorNameField = useBuilderWatch("name");
   const { yamlIsValid } = useConnectorBuilderFormState();
-  const manifest = useBuilderWatch("manifest");
+  const { exportValuesWithRefs } = useRefsHandler();
   const yaml = useBuilderWatch("yaml");
   const mode = useBuilderWatch("mode");
   const { hasErrors } = useBuilderErrors();
 
   const downloadYaml = () => {
-    const yamlToDownload = mode === "ui" ? convertJsonToYaml(manifest) : yaml;
+    const yamlToDownload =
+      mode === "ui" ? convertJsonToYaml(removeEmptyProperties(exportValuesWithRefs().manifest, true)) : yaml;
     const file = new Blob([yamlToDownload], { type: FILE_TYPE_DOWNLOAD });
     downloadFile(file, connectorNameField ? `${snakeCase(connectorNameField)}.yaml` : "connector_builder.yaml");
     analyticsService.track(Namespace.CONNECTOR_BUILDER, Action.DOWNLOAD_YAML, {

airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx

@@ -253,7 +253,9 @@ export const StreamTester: React.FC<{
           <Text bold>{resolveErrorMessage}</Text>
           {errorExceptionStack && (
             <Collapsible label={formatMessage({ id: "connectorBuilder.tracebackLabel" })} className={styles.traceback}>
-              <Pre longLines>{errorExceptionStack}</Pre>
+              <Pre longLines>
+                {isString(errorExceptionStack) ? errorExceptionStack : JSON.stringify(errorExceptionStack, null, 2)}
+              </Pre>
             </Collapsible>
           )}
           <Text>

airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlManifestEditor.tsx

@@ -5,6 +5,7 @@ import React, { useMemo, useRef, useState } from "react";
 import { useFormContext } from "react-hook-form";
 import { FormattedMessage } from "react-intl";
 
+import { useRefsHandler } from "components/forms/SchemaForm/RefsHandler";
 import { FlexItem } from "components/ui/Flex";
 import { ExternalLink } from "components/ui/Link";
 import { ButtonTab, Tabs } from "components/ui/Tabs";
@@ -39,7 +40,8 @@ export const YamlManifestEditor: React.FC = () => {
     setYamlIsDirty,
   } = useConnectorBuilderFormState();
   const { resolveManifest } = useConnectorBuilderResolve();
-  const { setValue } = useFormContext();
+  const { setValue, getValues } = useFormContext();
+  const { resetFormAndRefState } = useRefsHandler();
   const yamlManifestValue = useBuilderWatch("yaml");
 
   // Add a simple counter reference to track the latest call
@@ -51,17 +53,25 @@ export const YamlManifestEditor: React.FC = () => {
         // Increment counter to track this call
         const thisCallId = ++lastCallIdRef.current;
 
-        resolveManifest(newManifest).then((resolvedManifest) => {
+        resolveManifest(newManifest, true).then((resolvedManifest) => {
           // Only update state if this is still the latest call
           // This prevents yamlIsDirty from being set back to false when subsequent YAML changes are made
           // while previous resolve calls are still in progress.
           if (thisCallId === lastCallIdRef.current) {
-            setValue("manifest", resolvedManifest.manifest);
+            // shouldNormalize is set to true in the above resolveManifest call, which can result in $refs
+            // pointing to the refTargetPath in the result.
+            // To ensure that the linking behavior works as expected, we must update the refMappings to account
+            // for these $refs, and then manually resolve them when updating the manifest form state.
+            const newBuilderState = {
+              ...getValues(),
+              manifest: resolvedManifest.manifest,
+            };
+            resetFormAndRefState(newBuilderState);
             setYamlIsDirty(false);
           }
         });
       }, 500),
-    [resolveManifest, setValue, setYamlIsDirty]
+    [getValues, resetFormAndRefState, resolveManifest, setYamlIsDirty]
   );
 
   const areCustomComponentsEnabled = useCustomComponentsEnabled();

airbyte-webapp/src/components/forms/SchemaForm/Controls/SchemaFormControl.tsx

@@ -5,6 +5,7 @@ import { ReactMarkdown } from "react-markdown/lib/react-markdown";
 
 import { LabelInfo } from "components/Label";
 import { Badge } from "components/ui/Badge";
+import { FlexContainer } from "components/ui/Flex";
 import { Tooltip } from "components/ui/Tooltip";
 
 import { ArrayOfObjectsControl } from "./ArrayOfObjectsControl";
@@ -122,10 +123,11 @@ export const SchemaFormControl = ({
         <LabelInfo description={targetSchema.description} examples={targetSchema.examples} />
       ) : undefined,
     optional: isOptional,
-    header: targetSchema.deprecated ? (
-      <DeprecatedBadge message={targetSchema.deprecation_message} />
-    ) : (
-      <LinkComponentsToggle path={path} fieldSchema={targetSchema} />
+    header: (
+      <FlexContainer alignItems="center">
+        {targetSchema.deprecated && <DeprecatedBadge message={targetSchema.deprecation_message} />}
+        <LinkComponentsToggle path={path} fieldSchema={targetSchema} />
+      </FlexContainer>
     ),
     containerControlClassName: className,
     onlyShowErrorIfTouched,