Support geometry collection for reality meshes with glTF tiles (backport #9015) [release/5.6.x] (#9035)

Co-authored-by: Erin Ingram <47707444+eringram@users.noreply.github.com>
Co-authored-by: Mark Schlosser <47000437+markschlosseratbentley@users.noreply.github.com>
Co-authored-by: Ben Polinsky <78756012+ben-polinsky@users.noreply.github.com>
Co-authored-by: Nam Le <50554904+hl662@users.noreply.github.com>

Author: 5 people

common/api/core-frontend.api.md

@@ -4119,7 +4119,7 @@ export abstract class GltfReader {
     // (undocumented)
     protected readFeatureIndices(_json: any): number[] | undefined;
     // (undocumented)
-    readGltfAndCreateGeometry(transformToRoot?: Transform, needNormals?: boolean, needParams?: boolean): RealityTileGeometry;
+    readGltfAndCreateGeometry(transformToRoot?: Transform, needNormals?: boolean, needParams?: boolean): Promise<RealityTileGeometry>;
     // (undocumented)
     protected readGltfAndCreateGraphics(isLeaf: boolean, featureTable: FeatureTable | undefined, contentRange: ElementAlignedBox3d | undefined, transformToRoot?: Transform, pseudoRtcBias?: Vector3d, instances?: InstancedGraphicParams): GltfReaderResult;
     // (undocumented)

common/changes/@itwin/core-frontend/eringram-cesium-tiles-collect-geometry_2026-02-26-14-54.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@itwin/core-frontend",
+      "comment": "Support geometry collection for reality meshes with glTF tiles",
+      "type": "none"
+    }
+  ],
+  "packageName": "@itwin/core-frontend"
+}

core/frontend/src/internal/tile/RealityTileLoader.ts

@@ -80,20 +80,43 @@ export abstract class RealityTileLoader {
 
   public async loadGeometryFromStream(tile: RealityTile, streamBuffer: ByteStream, system: RenderSystem): Promise<RealityTileContent> {
     const format = this._getFormat(streamBuffer);
-    if (format !== TileFormat.B3dm)
+    if (format !== TileFormat.B3dm && format !== TileFormat.Gltf) {
       return {};
+    }
 
     const { is3d, yAxisUp, iModel, modelId } = tile.realityRoot;
-    const reader = B3dmReader.create(streamBuffer, iModel, modelId, is3d, tile.contentRange, system, yAxisUp, tile.isLeaf, tile.center, tile.transformToRoot, undefined, this.getBatchIdMap());
-    if (reader)
-      reader.defaultWrapMode = GltfWrapMode.ClampToEdge;
+    let reader: GltfReader | undefined;
 
+    // Create final transform from tree's iModelTransform and transformToRoot
     let transform = tile.tree.iModelTransform;
     if (tile.transformToRoot) {
       transform = transform.multiplyTransformTransform(tile.transformToRoot);
     }
 
-    const geom = reader?.readGltfAndCreateGeometry(transform);
+    switch (format) {
+      case TileFormat.Gltf:
+        const props = createReaderPropsWithBaseUrl(streamBuffer, yAxisUp, tile.tree.baseUrl);
+
+        if (props) {
+          reader = new GltfGraphicsReader(props, {
+            iModel,
+            gltf: props.glTF,
+            contentRange: tile.contentRange,
+            transform: tile.transformToRoot,
+            hasChildren: !tile.isLeaf,
+            pickableOptions: { id: modelId },
+            idMap: this.getBatchIdMap()
+          });
+        }
+        break;
+      case TileFormat.B3dm:
+        reader = B3dmReader.create(streamBuffer, iModel, modelId, is3d, tile.contentRange, system, yAxisUp, tile.isLeaf, tile.center, tile.transformToRoot, undefined, this.getBatchIdMap());
+        if (reader)
+          reader.defaultWrapMode = GltfWrapMode.ClampToEdge;
+        break;
+    }
+    const geom = await reader?.readGltfAndCreateGeometry(transform);
+
     // See RealityTileTree.reprojectAndResolveChildren for how reprojectionTransform is calculated
     const xForm = tile.reprojectionTransform;
     if (tile.tree.reprojectGeometry && geom?.polyfaces && xForm) {

core/frontend/src/test/tile/RealityTile.test.ts

@@ -5,14 +5,14 @@
 
 import { afterEach, beforeEach, describe, expect, it, MockInstance, vi } from "vitest";
 import { ByteStream } from "@itwin/core-bentley";
-import { TileFormat } from "@itwin/core-common";
+import { GltfV2ChunkTypes, GltfVersions, TileFormat } from "@itwin/core-common";
 import { Point3d, PolyfaceBuilder, Range3d, StrokeOptions, Transform } from "@itwin/core-geometry";
 import { IModelConnection } from "../../IModelConnection";
 import { IModelApp } from "../../IModelApp";
 import { MockRender } from "../../internal/render/MockRender";
 import { RenderMemory } from "../../render/RenderMemory";
 import {
-  B3dmReader, RealityTile, RealityTileGeometry, RealityTileLoader, RealityTileTree,
+  B3dmReader, GltfGraphicsReader, RealityTile, RealityTileGeometry, RealityTileLoader, RealityTileTree,
   Tile, TileDrawArgs, TileLoadPriority, TileRequest, TileRequestChannel
 } from "../../tile/internal";
 import { createBlankConnection } from "../createBlankConnection";
@@ -153,7 +153,7 @@ class TestRealityTree extends RealityTileTree {
 }
 
 class TestB3dmReader extends B3dmReader {
-  public override readGltfAndCreateGeometry(_transformToRoot?: Transform, _needNormals?: boolean, _needParams?: boolean): RealityTileGeometry {
+  public override async readGltfAndCreateGeometry(_transformToRoot?: Transform, _needNormals?: boolean, _needParams?: boolean): Promise<RealityTileGeometry> {
     // Create mock geometry data with a simple polyface
     const options = StrokeOptions.createForFacets();
     const polyBuilder = PolyfaceBuilder.create(options);
@@ -168,6 +168,26 @@ class TestB3dmReader extends B3dmReader {
   }
 }
 
+/** Creates a minimal valid GLB (binary glTF) for testing. */
+function createMinimalGlb(): Uint8Array {
+  const json = JSON.stringify({ asset: { version: "2.0" }, meshes: [] });
+  const jsonBytes = new TextEncoder().encode(json.padEnd(Math.ceil(json.length / 4) * 4));
+  const numBytes = 12 + 8 + jsonBytes.length; // header + JSON chunk header + JSON data
+
+  const glb = new Uint8Array(numBytes);
+  const view = new DataView(glb.buffer);
+
+  view.setUint32(0, TileFormat.Gltf, true);
+  view.setUint32(4, GltfVersions.Version2, true);
+  view.setUint32(8, numBytes, true);
+
+  view.setUint32(12, jsonBytes.length, true);
+  view.setUint32(16, GltfV2ChunkTypes.JSON, true);
+  glb.set(jsonBytes, 20);
+
+  return glb;
+}
+
 function expectPointToEqual(point: Point3d, x: number, y: number, z: number) {
   expect(point.x).toEqual(x);
   expect(point.y).toEqual(y);
@@ -321,4 +341,34 @@ describe("RealityTileLoader", () => {
     const geometryTransform = createGeometrySpy.mock.calls[0][0];
     expect(geometryTransform).toEqual(expectedTransform);
   });
+
+  it("should load geometry from tiles in glTF format", async () => {
+    const gltfStreamBuffer = ByteStream.fromUint8Array(createMinimalGlb());
+
+    const mockPolyface = PolyfaceBuilder.create(StrokeOptions.createForFacets()).claimPolyface();
+    vi.spyOn(GltfGraphicsReader.prototype, "readGltfAndCreateGeometry")
+      .mockResolvedValue({ polyfaces: [mockPolyface] });
+
+    const tree = new TestRealityTree(0, imodel, reader, false);
+    const tile = tree.rootTile;
+
+    const result = await reader.loadGeometryFromStream(tile, gltfStreamBuffer, IModelApp.renderSystem);
+
+    expect(result.geometry).to.not.be.undefined;
+    expect(result.geometry?.polyfaces).to.have.length(1);
+  });
+
+  it("should return empty content for unsupported tile format", async () => {
+    const buffer = new Uint8Array(16);
+    const view = new DataView(buffer.buffer);
+    view.setUint32(0, 0x12345678, true);
+    const invalidStreamBuffer = ByteStream.fromUint8Array(buffer);
+
+    const tree = new TestRealityTree(0, imodel, reader, false);
+    const tile = tree.rootTile;
+
+    const result = await reader.loadGeometryFromStream(tile, invalidStreamBuffer, IModelApp.renderSystem);
+
+    expect(result.geometry).to.be.undefined;
+  });
 });

core/frontend/src/tile/GltfReader.ts

@@ -22,7 +22,7 @@ import { IModelApp } from "../IModelApp";
 import { InstancedGraphicParams } from "../common/render/InstancedGraphicParams";
 import { RealityMeshParams } from "../render/RealityMeshParams";
 import { Mesh } from "../common/internal/render/MeshPrimitives";
-import { Triangle } from "../common/internal/render/Primitives";
+import { Triangle, TriangleList } from "../common/internal/render/Primitives";
 import { RenderGraphic } from "../render/RenderGraphic";
 import { RenderSystem } from "../render/RenderSystem";
 import { BatchedTileIdMap, decodeMeshoptBuffer, RealityTileGeometry,TileContent } from "./internal";
@@ -31,7 +31,7 @@ import { CreateRenderMaterialArgs } from "../render/CreateRenderMaterialArgs";
 import { DisplayParams } from "../common/internal/render/DisplayParams";
 import { FrontendLoggerCategory } from "../common/FrontendLoggerCategory";
 import { getImageSourceFormatForMimeType, imageBitmapFromImageSource, imageElementFromImageSource, tryImageElementFromUrl } from "../common/ImageUtil";
-import { MeshPrimitiveType } from "../common/internal/render/MeshPrimitive";
+import { MeshPointList, MeshPrimitiveType } from "../common/internal/render/MeshPrimitive";
 import { PointCloudArgs } from "../common/internal/render/PointCloudPrimitive";
 import { TextureImageSource } from "../common/render/TextureParams";
 import {
@@ -258,7 +258,7 @@ export class GltfReaderProps {
 
 /** The GltfMeshData contains the raw GLTF mesh data. If the data is suitable to create a [[RealityMesh]] directly, basically in the quantized format produced by
   * ContextCapture, then a RealityMesh is created directly from this data. Otherwise, the mesh primitive is populated from the raw data and a MeshPrimitive
-  * is generated. The MeshPrimitve path is much less efficient but should be rarely used.
+  * is generated. The MeshPrimitive path is much less efficient but should be rarely used.
   *
   * @internal
   */
@@ -664,7 +664,8 @@ export abstract class GltfReader {
     };
   }
 
-  public readGltfAndCreateGeometry(transformToRoot?: Transform, needNormals = false, needParams = false): RealityTileGeometry {
+  public async readGltfAndCreateGeometry(transformToRoot?: Transform, needNormals = false, needParams = false): Promise<RealityTileGeometry> {
+    await this.resolveResources();
     const transformStack = new TransformStack(this.getTileTransform(transformToRoot));
     const polyfaces: IndexedPolyface[] = [];
     for (const nodeKey of this._sceneNodes) {
@@ -949,11 +950,31 @@ export abstract class GltfReader {
   }
 
   private polyfaceFromGltfMesh(mesh: GltfMeshData, transform: Transform | undefined , needNormals: boolean, needParams: boolean): IndexedPolyface | undefined {
-    if (!mesh.pointQParams || !mesh.points || !mesh.indices)
-      return undefined;
+    if (mesh.pointQParams && mesh.points && mesh.indices)
+      return this.polyfaceFromQuantizedData(mesh.pointQParams, mesh.points, mesh.indices, mesh.normals, mesh.uvQParams, mesh.uvs, transform, needNormals, needParams);
 
-    const { points, pointQParams, normals, uvs, uvQParams, indices } = mesh;
+    const meshPrim = mesh.primitive;
+    const triangles = meshPrim.triangles;
+    const points = meshPrim.points;
+    if (!triangles || triangles.isEmpty || points.length === 0)
+      return undefined;
 
+    // This will likely only be the case for Draco-compressed meshes-- see where readDracoMeshPrimitive is called within readMeshPrimitive
+    // That is the only case where mesh.primitive is populated but mesh.pointQParams, mesh.points, & mesh.indices are not
+    return this.polyfaceFromMeshPrimitive(triangles, points, meshPrim.normals, meshPrim.uvParams, transform, needNormals, needParams);
+  }
+
+  private polyfaceFromQuantizedData(
+    pointQParams: QParams3d,
+    points: Uint16Array,
+    indices: Uint8Array | Uint16Array | Uint32Array,
+    normals: Uint16Array | undefined,
+    uvQParams: QParams2d | undefined,
+    uvs: Uint16Array | undefined,
+    transform: Transform | undefined,
+    needNormals: boolean,
+    needParams: boolean
+  ): IndexedPolyface {
     const includeNormals = needNormals && undefined !== normals;
     const includeParams = needParams && undefined !== uvQParams && undefined !== uvs;
 
@@ -990,6 +1011,62 @@ export abstract class GltfReader {
     return polyface;
   }
 
+  private polyfaceFromMeshPrimitive(
+    triangles: TriangleList,
+    points: MeshPointList,
+    normals: OctEncodedNormal[],
+    uvParams: Point2d[],
+    transform: Transform | undefined,
+    needNormals: boolean,
+    needParams: boolean
+  ): IndexedPolyface {
+    const includeNormals = needNormals && normals.length > 0;
+    const includeParams = needParams && uvParams.length > 0;
+
+    const polyface = IndexedPolyface.create(includeNormals, includeParams);
+
+    if (points instanceof QPoint3dList) {
+      for (let i = 0; i < points.length; i++) {
+        const point = points.unquantize(i);
+        if (transform)
+          transform.multiplyPoint3d(point, point);
+        polyface.addPoint(point);
+      }
+    } else {
+      const center = points.range.center;
+      for (const pt of points) {
+        const point = pt.plus(center);
+        if (transform)
+          transform.multiplyPoint3d(point, point);
+        polyface.addPoint(point);
+      }
+    }
+
+    if (includeNormals)
+      for (const normal of normals)
+        polyface.addNormal(OctEncodedNormal.decodeValue(normal.value));
+
+    if (includeParams)
+      for (const uv of uvParams)
+        polyface.addParam(uv);
+
+    const indices = triangles.indices;
+    let j = 0;
+    for (const index of indices) {
+      polyface.addPointIndex(index);
+      if (includeNormals)
+        polyface.addNormalIndex(index);
+
+      if (includeParams)
+        polyface.addParamIndex(index);
+
+      if (0 === (++j % 3))
+        polyface.terminateFacet();
+    }
+
+    return polyface;
+  }
+
   // ###TODO what is the actual type of `json`?
   public getBufferView(json: { [k: string]: any }, accessorName: string): GltfBufferView | undefined {
     try {