fix(getAnchoredPosition): add better support for floatingElements inside dialogs (#681)

* add repro

* lock scroll

* implement new "displayInVisibleViewport" setting for getAnchoredPosition

* rename story

* Fix getAnchoredPosition for better dialog support

Enhance getAnchoredPosition to support floatingElements in dialogs with a new setting.

* Refactor pureCalculateAnchoredPosition to eliminate DOM dependency (#683)

* Initial plan

* Refactor pureCalculateAnchoredPosition to accept viewport dimensions as parameter

Co-authored-by: francinelucca <40550942+francinelucca@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: francinelucca <40550942+francinelucca@users.noreply.github.com>

* rename displayInVisibleViewport to displayInViewport

* Update quick-masks-camp.md

---------

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>

Author: francinelucca

.changeset/quick-masks-camp.md

@@ -0,0 +1,5 @@
+---
+"@primer/behaviors": patch
+---
+
+fix(getAnchoredPosition): add better support for floatingElements inside dialogs through new "displayInViewport" setting

src/__tests__/anchored-position.test.ts

@@ -573,4 +573,128 @@ describe('getAnchoredPosition', () => {
       expect(result.left).toEqual(290) // anchorRect.left - parentRect.left
     })
   })
+
+  describe('displayInViewport option', () => {
+    beforeEach(() => {
+      // Mock window dimensions for consistent testing
+      Object.defineProperty(window, 'innerWidth', {value: 1024, writable: true})
+      Object.defineProperty(window, 'innerHeight', {value: 768, writable: true})
+    })
+
+    it('defaults to false when not specified', () => {
+      const anchorRect = makeDOMRect(300, 200, 50, 50)
+      const floatingRect = makeDOMRect(NaN, NaN, 100, 100)
+      const {float, anchor} = createVirtualDOM(makeDOMRect(20, 20, 1920, 1080), anchorRect, floatingRect)
+
+      const {top, left} = getAnchoredPosition(float, anchor, {side: 'outside-bottom'})
+
+      // Should use normal clipping behavior (relative to clipping parent)
+      expect(top).toEqual(234) // anchorRect.top + anchorRect.height + 4 - parentRect.top
+      expect(left).toEqual(280) // anchorRect.left - parentRect.left
+    })
+
+    it('constrains to visible viewport when set to true', () => {
+      const anchorRect = makeDOMRect(950, 700, 50, 50) // Near bottom-right of viewport
+      const floatingRect = makeDOMRect(NaN, NaN, 200, 200) // Large floating element
+      const {float, anchor} = createVirtualDOM(
+        makeDOMRect(0, 0, 2000, 2000), // Large container
+        anchorRect,
+        floatingRect,
+      )
+
+      const {top, left} = getAnchoredPosition(float, anchor, {
+        side: 'outside-bottom',
+        displayInViewport: true,
+      })
+
+      // Should be constrained to fit within viewport (1024x768)
+      expect(left).toBeLessThanOrEqual(1024 - 200) // left + width <= viewport width
+      expect(top + 200).toBeLessThanOrEqual(768) // top + height <= viewport height
+    })
+
+    it('flips to top when bottom would exceed viewport height', () => {
+      const anchorRect = makeDOMRect(400, 700, 50, 50) // Near bottom of viewport
+      const floatingRect = makeDOMRect(NaN, NaN, 100, 100)
+      const {float, anchor} = createVirtualDOM(makeDOMRect(0, 0, 1920, 1080), anchorRect, floatingRect)
+
+      const {top, anchorSide} = getAnchoredPosition(float, anchor, {
+        side: 'outside-bottom',
+        displayInViewport: true,
+      })
+
+      // Should flip to top side to fit in viewport
+      expect(anchorSide).toEqual('outside-top')
+      expect(top).toBeGreaterThanOrEqual(0)
+      expect(top + 100).toBeLessThanOrEqual(768)
+    })
+
+    it('adjusts left position when right edge would exceed viewport width', () => {
+      const anchorRect = makeDOMRect(950, 300, 50, 50) // Near right edge of viewport
+      const floatingRect = makeDOMRect(NaN, NaN, 150, 100)
+      const {float, anchor} = createVirtualDOM(makeDOMRect(0, 0, 1920, 1080), anchorRect, floatingRect)
+
+      const {left} = getAnchoredPosition(float, anchor, {
+        side: 'outside-bottom',
+        align: 'start',
+        displayInViewport: true,
+      })
+
+      // Should be pushed left to fit in viewport
+      expect(left + 150).toBeLessThanOrEqual(1024) // left + width <= viewport width
+    })
+
+    it('works with inside positioning', () => {
+      const anchorRect = makeDOMRect(100, 100, 800, 600) // Large anchor covering most viewport
+      const floatingRect = makeDOMRect(NaN, NaN, 200, 100)
+      const {float, anchor} = createVirtualDOM(makeDOMRect(0, 0, 1920, 1080), anchorRect, floatingRect)
+
+      const {top, left} = getAnchoredPosition(float, anchor, {
+        side: 'inside-center',
+        align: 'center',
+        displayInViewport: true,
+      })
+
+      // Should position inside the anchor and within viewport
+      expect(left).toBeGreaterThanOrEqual(0)
+      expect(top).toBeGreaterThanOrEqual(0)
+      expect(left + 200).toBeLessThanOrEqual(1024)
+      expect(top + 100).toBeLessThanOrEqual(768)
+    })
+
+    it('still respects allowOutOfBounds when both options are set', () => {
+      const anchorRect = makeDOMRect(950, 700, 50, 50)
+      const floatingRect = makeDOMRect(NaN, NaN, 200, 200)
+      const {float, anchor} = createVirtualDOM(makeDOMRect(0, 0, 1920, 1080), anchorRect, floatingRect)
+
+      const {top, left} = getAnchoredPosition(float, anchor, {
+        side: 'outside-bottom',
+        displayInViewport: true,
+        allowOutOfBounds: true,
+      })
+
+      // When allowOutOfBounds is true, should allow overflow even with displayInViewport
+      // The exact behavior depends on implementation, but it should not throw an error
+      expect(typeof top).toBe('number')
+      expect(typeof left).toBe('number')
+    })
+
+    it('handles edge case when anchor is outside viewport', () => {
+      const anchorRect = makeDOMRect(1200, 900, 50, 50) // Outside 1024x768 viewport
+      const floatingRect = makeDOMRect(NaN, NaN, 100, 100)
+      const {float, anchor} = createVirtualDOM(makeDOMRect(0, 0, 2000, 2000), anchorRect, floatingRect)
+
+      const {top, left} = getAnchoredPosition(float, anchor, {
+        side: 'outside-bottom',
+        displayInViewport: true,
+      })
+
+      // When anchor is outside viewport, the position calculation should still work
+      // but the final nudging should ensure no part of the floating element exceeds viewport
+      // Since the anchor is at (1200, 900) with 'outside-bottom', the floating element
+      // would initially be positioned around (1200, 954), which exceeds viewport bounds
+      // The displayInViewport option should constrain this
+      expect(left + 100).toBeLessThanOrEqual(1024) // right edge within viewport
+      expect(top + 100).toBeLessThanOrEqual(768) // bottom edge within viewport
+    })
+  })
 })

src/anchored-position.ts

@@ -87,6 +87,14 @@ export interface PositionSettings {
    * this direction.
    */
   allowOutOfBounds: boolean
+
+  /**
+   * If true, attempt to position the floating element entirely within the visible
+   * viewport (window.innerWidth/innerHeight) without requiring scrolling.
+   * This is useful when the anchor is inside a dialog or other container that
+   * prevents scrolling of the main document.
+   */
+  displayInViewport?: boolean
 }
 
 // For each outside anchor position, list the order of alternate positions to try in
@@ -167,6 +175,7 @@ export function getAnchoredPosition(
     floatingElement.getBoundingClientRect(),
     anchorElement instanceof Element ? anchorElement.getBoundingClientRect() : anchorElement,
     getDefaultSettings(settings),
+    {width: window.innerWidth, height: window.innerHeight},
   )
 }
 
@@ -292,6 +301,7 @@ const positionDefaults: PositionSettings = {
   alignmentOffset: 4,
 
   allowOutOfBounds: false,
+  displayInViewport: false,
 }
 
 /**
@@ -311,6 +321,7 @@ function getDefaultSettings(settings: Partial<PositionSettings> = {}): PositionS
       settings.alignmentOffset ??
       (align !== 'center' && side.startsWith('inside') ? positionDefaults.alignmentOffset : 0),
     allowOutOfBounds: settings.allowOutOfBounds ?? positionDefaults.allowOutOfBounds,
+    displayInViewport: settings.displayInViewport ?? positionDefaults.displayInViewport,
   }
 }
 
@@ -324,20 +335,34 @@ function getDefaultSettings(settings: Partial<PositionSettings> = {}): PositionS
  * @param floatingRect WidthAndHeight for the floating element
  * @param anchorRect BoxPosition for the anchor element
  * @param PositionSettings to customize the calculated position for the floating element.
+ * @param visibleViewportSize Size of the visible viewport (window dimensions), used when displayInVisibleViewport is true
  */
 function pureCalculateAnchoredPosition(
   viewportRect: BoxPosition,
   relativePosition: Position,
   floatingRect: Size,
   anchorRect: BoxPosition,
-  {side, align, allowOutOfBounds, anchorOffset, alignmentOffset}: PositionSettings,
+  {side, align, allowOutOfBounds, anchorOffset, alignmentOffset, displayInViewport}: PositionSettings,
+  visibleViewportSize: Size,
 ): AnchorPosition {
   // Compute the relative viewport rect, to bring it into the same coordinate space as `pos`
+  let effectiveViewportRect = viewportRect
+
+  // If displayInViewport is true, constrain to the actual visible viewport
+  if (displayInViewport) {
+    effectiveViewportRect = {
+      top: 0,
+      left: 0,
+      width: visibleViewportSize.width,
+      height: visibleViewportSize.height,
+    }
+  }
+
   const relativeViewportRect: BoxPosition = {
-    top: viewportRect.top - relativePosition.top,
-    left: viewportRect.left - relativePosition.left,
-    width: viewportRect.width,
-    height: viewportRect.height,
+    top: effectiveViewportRect.top - relativePosition.top,
+    left: effectiveViewportRect.left - relativePosition.left,
+    width: effectiveViewportRect.width,
+    height: effectiveViewportRect.height,
   }
 
   let pos = calculatePosition(floatingRect, anchorRect, side, align, anchorOffset, alignmentOffset)
@@ -400,16 +425,14 @@ function pureCalculateAnchoredPosition(
     if (pos.left < relativeViewportRect.left) {
       pos.left = relativeViewportRect.left
     }
-    if (pos.left + floatingRect.width > viewportRect.width + relativeViewportRect.left) {
-      pos.left = viewportRect.width + relativeViewportRect.left - floatingRect.width
+    if (pos.left + floatingRect.width > effectiveViewportRect.width + relativeViewportRect.left) {
+      pos.left = effectiveViewportRect.width + relativeViewportRect.left - floatingRect.width
     }
-    // If we have exhausted all possible positions and none of them worked, we
-    // say that overflowing the bottom of the screen is acceptable since it is
-    // likely to be able to scroll.
-    if (alternateOrder && positionAttempt < alternateOrder.length) {
-      if (pos.top + floatingRect.height > viewportRect.height + relativeViewportRect.top) {
+    // Always apply bottom constraint when displayInViewport is true
+    if (displayInViewport || (alternateOrder && positionAttempt < alternateOrder.length)) {
+      if (pos.top + floatingRect.height > effectiveViewportRect.height + relativeViewportRect.top) {
         // This prevents top from being a negative value
-        pos.top = Math.max(viewportRect.height + relativeViewportRect.top - floatingRect.height, 0)
+        pos.top = Math.max(effectiveViewportRect.height + relativeViewportRect.top - floatingRect.height, 0)
       }
     }
   }