Simple-Schema and to stored refactor (microsoft#25962)

## Description

Since simple schema can now contain view schema specific data, but might
also be derived from stored schema, and might also be storing only a
subset of its view specific data that it can hold (like when
snapshotting for compatibility tests) using it is rather error prone.

This refactoring attempts to better clarify what the semantics of a
given simple schema are by allowing them to be typed as either view or
stored, and providing more specific names and docs to the compatibility
snapshotting APIs.

This also refactors how we generate stored schema: now we transform a
simple schema for a view to a simple schema for stored schema, then
convert that to the persisted format. This redesign separates the
concerns for persisted format conversion and the semantics of things
like staged schema which get processes when converting from view to
stored.

This results in some deduplication of logic, and allows all schema
transformation logic to be applied directly to simple-schema.
This change could be followed up with some further changes to better
remove stored schema from the alpha APIs, and replace those APIs with
use of stored-simple-schema and thus shrinking the package API surface
area and making its types more interoperable.

`getUnhydratedContext` has been improved to give better asserts when
uses re-entrantly.

## Breaking Changes

Several stored and simple schema alpha APIs have been impacted, but all
stable APIs should behave as is.

Author: CraigMacomber

packages/dds/tree/api-report/tree.alpha.api.md

@@ -850,7 +850,6 @@ export interface TreeViewBeta<in out TSchema extends ImplicitFieldSchema> extend
 // @public @sealed
 export class TreeViewConfiguration<const TSchema extends ImplicitFieldSchema = ImplicitFieldSchema> implements Required<ITreeViewConfiguration<TSchema>> {
     constructor(props: ITreeViewConfiguration<TSchema>);
-    protected readonly definitionsInternal: ReadonlyMap<string, TreeNodeSchema>;
     readonly enableSchemaValidation: boolean;
     readonly preventAmbiguity: boolean;
     readonly schema: TSchema;

packages/dds/tree/api-report/tree.beta.api.md

@@ -862,7 +862,6 @@ export interface TreeViewBeta<in out TSchema extends ImplicitFieldSchema> extend
 // @public @sealed
 export class TreeViewConfiguration<const TSchema extends ImplicitFieldSchema = ImplicitFieldSchema> implements Required<ITreeViewConfiguration<TSchema>> {
     constructor(props: ITreeViewConfiguration<TSchema>);
-    protected readonly definitionsInternal: ReadonlyMap<string, TreeNodeSchema>;
     readonly enableSchemaValidation: boolean;
     readonly preventAmbiguity: boolean;
     readonly schema: TSchema;

packages/dds/tree/api-report/tree.legacy.beta.api.md

@@ -585,7 +585,6 @@ export interface TreeView<in out TSchema extends ImplicitFieldSchema> extends ID
 // @public @sealed
 export class TreeViewConfiguration<const TSchema extends ImplicitFieldSchema = ImplicitFieldSchema> implements Required<ITreeViewConfiguration<TSchema>> {
     constructor(props: ITreeViewConfiguration<TSchema>);
-    protected readonly definitionsInternal: ReadonlyMap<string, TreeNodeSchema>;
     readonly enableSchemaValidation: boolean;
     readonly preventAmbiguity: boolean;
     readonly schema: TSchema;

packages/dds/tree/api-report/tree.legacy.public.api.md

@@ -585,7 +585,6 @@ export interface TreeView<in out TSchema extends ImplicitFieldSchema> extends ID
 // @public @sealed
 export class TreeViewConfiguration<const TSchema extends ImplicitFieldSchema = ImplicitFieldSchema> implements Required<ITreeViewConfiguration<TSchema>> {
     constructor(props: ITreeViewConfiguration<TSchema>);
-    protected readonly definitionsInternal: ReadonlyMap<string, TreeNodeSchema>;
     readonly enableSchemaValidation: boolean;
     readonly preventAmbiguity: boolean;
     readonly schema: TSchema;

packages/dds/tree/api-report/tree.public.api.md

@@ -185,6 +185,7 @@ export {
 	type AnnotateAllowedTypesList,
 	type AllowedTypesFull,
 	type AllowedTypesFullFromMixed,
+	type SchemaType,
 	// Beta APIs
 	TreeBeta,
 	type TreeChangeEventsBeta,
@@ -283,8 +284,8 @@ export {
 	type SchemaFactory_base,
 	type NumberKeys,
 	type SimpleAllowedTypeAttributes,
-	encodeSimpleSchema,
-	decodeSimpleSchema,
+	encodeSchemaCompatibilitySnapshot,
+	decodeSchemaCompatibilitySnapshot,
 	exportCompatibilitySchemaSnapshot,
 	importCompatibilitySchemaSnapshot,
 	checkCompatibility,

packages/dds/tree/src/index.ts

@@ -97,6 +97,7 @@ import {
 	type ITreeAlpha,
 	type SimpleObjectFieldSchema,
 	type SimpleAllowedTypeAttributes,
+	type SchemaType,
 } from "../simple-tree/index.js";
 
 import { SchematizingSimpleTreeView } from "./schematizingTreeView.js";
@@ -507,7 +508,9 @@ export class SharedTreeKernel
 	public onDisconnect(): void {}
 }
 
-export function exportSimpleSchema(storedSchema: TreeStoredSchema): SimpleTreeSchema {
+export function exportSimpleSchema(
+	storedSchema: TreeStoredSchema,
+): SimpleTreeSchema<SchemaType.Stored> {
 	return {
 		root: exportSimpleFieldSchemaStored(storedSchema.rootFieldSchema),
 		definitions: new Map(
@@ -823,16 +826,18 @@ export const defaultSharedTreeOptions: Required<SharedTreeOptionsInternal> = {
  */
 function buildSimpleAllowedTypeAttributesForStoredSchema(
 	types: TreeTypeSet,
-): ReadonlyMap<string, SimpleAllowedTypeAttributes> {
-	const allowedTypesInfo = new Map<string, SimpleAllowedTypeAttributes>();
+): ReadonlyMap<string, SimpleAllowedTypeAttributes<SchemaType.Stored>> {
+	const allowedTypesInfo = new Map<string, SimpleAllowedTypeAttributes<SchemaType.Stored>>();
 	for (const type of types) {
 		// Stored schemas do not have staged upgrades
 		allowedTypesInfo.set(type, { isStaged: undefined });
 	}
 	return allowedTypesInfo;
 }
 
-function exportSimpleFieldSchemaStored(schema: TreeFieldStoredSchema): SimpleFieldSchema {
+function exportSimpleFieldSchemaStored(
+	schema: TreeFieldStoredSchema,
+): SimpleFieldSchema<SchemaType.Stored> {
 	let kind: FieldKind;
 	switch (schema.kind) {
 		case FieldKinds.identifier.identifier:
@@ -866,7 +871,9 @@ function exportSimpleFieldSchemaStored(schema: TreeFieldStoredSchema): SimpleFie
  * Note on SimpleNodeSchema construction: In the persisted format `persistedMetadata` is just called `metadata` whereas the `metadata`
  * field on SimpleNodeSchema is not persisted.
  */
-function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeSchema {
+function exportSimpleNodeSchemaStored(
+	schema: TreeNodeStoredSchema,
+): SimpleNodeSchema<SchemaType.Stored> {
 	const arrayTypes = tryStoredSchemaAsArray(schema);
 	if (arrayTypes !== undefined) {
 		return {
@@ -877,7 +884,7 @@ function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeS
 		};
 	}
 	if (schema instanceof ObjectNodeStoredSchema) {
-		const fields = new Map<FieldKey, SimpleObjectFieldSchema>();
+		const fields = new Map<FieldKey, SimpleObjectFieldSchema<SchemaType.Stored>>();
 		for (const [storedKey, field] of schema.objectNodeFields) {
 			fields.set(storedKey, { ...exportSimpleFieldSchemaStored(field), storedKey });
 		}

packages/dds/tree/src/shared-tree/sharedTree.ts

@@ -50,8 +50,6 @@ import {
 	isObjectNodeSchema,
 	isTreeNode,
 	toInitialSchema,
-	convertField,
-	toUnhydratedSchema,
 	type TreeParsingOptions,
 	type NodeChangedData,
 	type ConciseTree,
@@ -60,6 +58,7 @@ import {
 	borrowCursorFromTreeNodeOrValue,
 	contentSchemaSymbol,
 	type TreeNodeSchema,
+	getUnhydratedContext,
 } from "../simple-tree/index.js";
 import { brand, extractFromOpaque, type JsonCompatible } from "../util/index.js";
 import {
@@ -824,7 +823,7 @@ export const TreeAlpha: TreeAlpha = {
 		return createFromCursor(
 			schema,
 			cursor,
-			convertField(normalizeFieldSchema(schema), toUnhydratedSchema),
+			getUnhydratedContext(schema).flexContext.schema.rootFieldSchema,
 		);
 	},
 

packages/dds/tree/src/shared-tree/treeAlpha.ts

@@ -3,31 +3,19 @@
  * Licensed under the MIT License.
  */
 
-import {
-	assert,
-	debugAssert,
-	fail,
-	oob,
-	unreachableCase,
-} from "@fluidframework/core-utils/internal";
+import { assert, fail, oob, unreachableCase } from "@fluidframework/core-utils/internal";
 import { UsageError } from "@fluidframework/telemetry-utils/internal";
 
-import {
-	type FieldSchemaAlpha,
-	type ImplicitFieldSchema,
-	FieldKind,
-	normalizeFieldSchema,
-} from "../fieldSchema.js";
+import { type FieldSchemaAlpha, type ImplicitFieldSchema, FieldKind } from "../fieldSchema.js";
 import {
 	type AllowedTypesFullEvaluated,
 	NodeKind,
 	type TreeNodeSchema,
-	markSchemaMostDerived,
 } from "../core/index.js";
 import {
-	permissiveStoredSchemaGenerationOptions,
-	restrictiveStoredSchemaGenerationOptions,
-	toStoredSchema,
+	toInitialSchema,
+	toUnhydratedSchema,
+	transformSimpleSchema,
 } from "../toStoredSchema.js";
 import {
 	isArrayNodeSchema,
@@ -42,7 +30,8 @@ import {
 import { getOrCreate } from "../../util/index.js";
 import type { MakeNominal } from "../../util/index.js";
 import { walkFieldSchema } from "../walkFieldSchema.js";
-import type { SimpleNodeSchema, SimpleTreeSchema } from "../simpleSchema.js";
+import type { SchemaType, SimpleNodeSchema } from "../simpleSchema.js";
+import { createTreeSchema, type TreeSchema } from "../treeSchema.js";
 
 /**
  * Options when constructing a tree view.
@@ -187,11 +176,6 @@ export class TreeViewConfiguration<
 	 */
 	public readonly preventAmbiguity!: boolean;
 
-	/**
-	 * {@link TreeSchema.definitions} but with public types.
-	 */
-	protected readonly definitionsInternal!: ReadonlyMap<string, TreeNodeSchema>;
-
 	/**
 	 * Construct a new {@link TreeViewConfiguration}.
 	 *
@@ -225,23 +209,10 @@ export class TreeViewConfiguration<
 		// Ambiguity errors are lower priority to report than invalid schema errors, so collect these in an array and report them all at once.
 		const ambiguityErrors: string[] = [];
 
-		// Eagerly perform this conversion to surface errors sooner.
-		// Includes detection of duplicate schema identifiers.
-		toStoredSchema(config.schema, restrictiveStoredSchemaGenerationOptions);
-		toStoredSchema(config.schema, permissiveStoredSchemaGenerationOptions);
-
-		const definitions = new Map<string, SimpleNodeSchema & TreeNodeSchema>();
-
+		// Validate the schema and collect ambiguity errors.
+		// This does a lot of validation (throwing usage errors as a side effect) in addition to just collecting ambiguity errors.
+		// ambiguityErrors are considered a lower priority, so only thrown if no other errors are found.
 		walkFieldSchema(config.schema, {
-			node: (schema) => {
-				// Ensure all reachable schema are marked as most derived.
-				// This ensures if multiple schema extending the same schema factory generated class are present (or have had instances of them constructed, or get instances of them constructed in the future),
-				// an error is reported.
-				markSchemaMostDerived(schema, true);
-
-				debugAssert(() => !definitions.has(schema.identifier));
-				definitions.set(schema.identifier, schema as SimpleNodeSchema & TreeNodeSchema);
-			},
 			allowedTypes({ types }: AllowedTypesFullEvaluated): void {
 				checkUnion(
 					types.map((t) => t.type),
@@ -251,8 +222,6 @@ export class TreeViewConfiguration<
 			},
 		});
 
-		this.definitionsInternal = definitions;
-
 		if (ambiguityErrors.length !== 0) {
 			// Duplicate errors are common since when two types conflict, both orders error:
 			const deduplicated = new Set(ambiguityErrors);
@@ -273,38 +242,22 @@ export class TreeViewConfigurationAlpha<
 	extends TreeViewConfiguration<TSchema>
 	implements TreeSchema
 {
-	/**
-	 * {@inheritDoc TreeSchema.root}
-	 */
 	public readonly root: FieldSchemaAlpha;
-
-	/**
-	 * {@inheritDoc TreeSchema.definitions}
-	 */
-	public get definitions(): ReadonlyMap<string, SimpleNodeSchema & TreeNodeSchema> {
-		return this.definitionsInternal as ReadonlyMap<string, SimpleNodeSchema & TreeNodeSchema>;
-	}
+	public readonly definitions: ReadonlyMap<
+		string,
+		SimpleNodeSchema<SchemaType.View> & TreeNodeSchema
+	>;
 
 	public constructor(props: ITreeViewConfiguration<TSchema>) {
 		super(props);
-		this.root = normalizeFieldSchema(props.schema);
-	}
-}
+		const treeSchema = createTreeSchema(this.schema);
+		this.root = treeSchema.root;
+		this.definitions = treeSchema.definitions;
 
-/**
- * {@link TreeViewConfigurationAlpha}
- * @sealed @alpha
- */
-export interface TreeSchema extends SimpleTreeSchema {
-	/**
-	 * {@inheritDoc SimpleTreeSchema.root}
-	 */
-	readonly root: FieldSchemaAlpha;
-
-	/**
-	 * {@inheritDoc SimpleTreeSchema.definitions}
-	 */
-	readonly definitions: ReadonlyMap<string, SimpleNodeSchema & TreeNodeSchema>;
+		// Eagerly perform these conversions to surface errors sooner.
+		toInitialSchema(this.root);
+		transformSimpleSchema(treeSchema, toUnhydratedSchema);
+	}
 }
 
 /**

packages/dds/tree/src/simple-tree/api/configuration.ts

@@ -43,7 +43,7 @@ import {
 	type FieldSchema,
 } from "../fieldSchema.js";
 import { LeafNodeSchema } from "../leafNodeSchema.js";
-import type { TreeSchema } from "./configuration.js";
+import type { TreeSchema } from "../treeSchema.js";
 import { tryStoredSchemaAsArray } from "./customTree.js";
 import { FieldKinds } from "../../feature-libraries/index.js";