Refactor window layout calculation into iTermLayoutCalculator. Issue 12724

Extract layout logic from iTermRootTerminalView into a pure
iTermLayoutCalculator class for testability. This fixes a bug where
the tab bar height was double-deducted when calculating the toolbelt
frame in fullscreen.

The bug occurred because two mechanisms could both subtract tab bar
height:
1. shouldLeaveEmptyAreaAtTop added it to decorationHeightTop
2. tabViewFrameByShrinkingForFullScreenTabBar shrunk the frame

These are now mutually exclusive - if shouldLeaveEmptyAreaAtTop is
true, the frame shrinking is skipped.

Also rename rootTerminalViewWindowHasFullSizeContentView to
rootTerminalViewFullScreenTabBarAccessoryOverlapsContent for clarity.

Author: gnachman

ModernTests/iTermLayoutCalculatorTest.swift

@@ -659,6 +659,7 @@
 		4887C7B3CAB5465296225860 /* MenuTips.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 904AE312A0E0489899D9ECB8 /* MenuTips.xcassets */; };
 		49A6E4091211CC6000D9AD6F /* Compatability.h in Headers */ = {isa = PBXBuildFile; fileRef = 49A6E4081211CC6000D9AD6F /* Compatability.h */; };
 		4BC1E97CEA4441CF9390DD52 /* MenuTips.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 904AE312A0E0489899D9ECB8 /* MenuTips.xcassets */; };
+		4C4BE71D1C8948BD8CD672BC /* iTermLayoutCalculator.m in Sources */ = {isa = PBXBuildFile; fileRef = AF19E441FF824539A275D8DA /* iTermLayoutCalculator.m */; };
 		5300984C2259365B00A69348 /* iTermHapticActuator.h in Headers */ = {isa = PBXBuildFile; fileRef = 5300984A2259365B00A69348 /* iTermHapticActuator.h */; };
 		5300984D2259365B00A69348 /* iTermHapticActuator.m in Sources */ = {isa = PBXBuildFile; fileRef = 5300984B2259365B00A69348 /* iTermHapticActuator.m */; };
 		530098502259A44D00A69348 /* iTermSoundPlayer.h in Headers */ = {isa = PBXBuildFile; fileRef = 5300984E2259A44D00A69348 /* iTermSoundPlayer.h */; };
@@ -949,6 +950,7 @@
 		9DB3D6F0176CCABE0071CCF8 /* PrefsArrangements.png in Resources */ = {isa = PBXBuildFile; fileRef = 9DB3D6EE176CCABE0071CCF8 /* PrefsArrangements.png */; };
 		9DB3D6F1176CCABE0071CCF8 /* PrefsArrangements@2x.png in Resources */ = {isa = PBXBuildFile; fileRef = 9DB3D6EF176CCABE0071CCF8 /* PrefsArrangements@2x.png */; };
 		A073973E14C768E400786414 /* ColorsMenuItemView.h in Headers */ = {isa = PBXBuildFile; fileRef = A073973D14C768E400786414 /* ColorsMenuItemView.h */; };
+		A445EEDC85D346B39596FDC3 /* iTermLayoutCalculator.h in Headers */ = {isa = PBXBuildFile; fileRef = 93EDFB853885460AA504C6B9 /* iTermLayoutCalculator.h */; };
 		A60014F618550F3900CE38D8 /* NSMutableAttributedString+iTerm.h in Headers */ = {isa = PBXBuildFile; fileRef = A60014F418550F3900CE38D8 /* NSMutableAttributedString+iTerm.h */; };
 		A60014FB18552BDF00CE38D8 /* iTermNSKeyBindingEmulator.h in Headers */ = {isa = PBXBuildFile; fileRef = A60014F918552BDF00CE38D8 /* iTermNSKeyBindingEmulator.h */; };
 		A600A93B2E3EDD5200D05EAD /* iTermBrowserTriggerHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = A67238B72E3EDC8E003227C3 /* iTermBrowserTriggerHandler.swift */; };
@@ -5127,6 +5129,7 @@
 		C6675EBC1C4FE96B0041173B /* iTermSelectorSwizzler.m in Sources */ = {isa = PBXBuildFile; fileRef = C6675EBB1C4FE96B0041173B /* iTermSelectorSwizzler.m */; };
 		CDCD9FF08FCE42F7A6963859 /* PillBackgroundGenerator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7EA289FFE2BF4090A65931BA /* PillBackgroundGenerator.swift */; };
 		D3CE2D4E1A00936F0098ED99 /* PSMDarkTabStyle.h in Headers */ = {isa = PBXBuildFile; fileRef = D3CE2D4C1A00936F0098ED99 /* PSMDarkTabStyle.h */; };
+		D5EF7F4036204859817200E0 /* iTermLayoutCalculator.h in Headers */ = {isa = PBXBuildFile; fileRef = 93EDFB853885460AA504C6B9 /* iTermLayoutCalculator.h */; };
 		DDF0FD65062916F70080EF74 /* iTermApplication.h in Headers */ = {isa = PBXBuildFile; fileRef = DDF0FD63062916F70080EF74 /* iTermApplication.h */; };
 		FB4CEC973C7E9235362E3F26 /* iTermRemotePreferences.h in Headers */ = {isa = PBXBuildFile; fileRef = FB4CECF4AC4392B21E87A07B /* iTermRemotePreferences.h */; };
 /* End PBXBuildFile section */
@@ -6202,6 +6205,7 @@
 		904AE312A0E0489899D9ECB8 /* MenuTips.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = MenuTips.xcassets; sourceTree = "<group>"; };
 		90A1E139186F9EA4003EC3E8 /* AppleScriptTest.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AppleScriptTest.h; sourceTree = "<group>"; };
 		90A1E13A186F9EA4003EC3E8 /* AppleScriptTest.m */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 4; lastKnownFileType = sourcecode.c.objc; path = AppleScriptTest.m; sourceTree = "<group>"; };
+		93EDFB853885460AA504C6B9 /* iTermLayoutCalculator.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = iTermLayoutCalculator.h; sourceTree = "<group>"; };
 		9D737D5D17495B4100B3334D /* PrefsGeneral.png */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; name = PrefsGeneral.png; path = images/PrefsGeneral.png; sourceTree = "<group>"; };
 		9D737D5E17495B4100B3334D /* PrefsGeneral@2x.png */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; name = "PrefsGeneral@2x.png"; path = "images/PrefsGeneral@2x.png"; sourceTree = "<group>"; };
 		9DB3D6DE176C8F0B0071CCF8 /* PrefsAppearance.png */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; name = PrefsAppearance.png; path = images/PrefsAppearance.png; sourceTree = "<group>"; };
@@ -9090,6 +9094,7 @@
 		A6FFAE702CF123CC0031FB21 /* CoreTextLineRenderingHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreTextLineRenderingHelper.swift; sourceTree = "<group>"; };
 		A6FFAE722CF682B60031FB21 /* iTermDoubleWidthCharacterCache.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = iTermDoubleWidthCharacterCache.h; sourceTree = "<group>"; };
 		A6FFAE742CF682D40031FB21 /* iTermDoubleWidthCharacterCache.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iTermDoubleWidthCharacterCache.m; sourceTree = "<group>"; };
+		AF19E441FF824539A275D8DA /* iTermLayoutCalculator.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = iTermLayoutCalculator.m; sourceTree = "<group>"; };
 		C6675EBA1C4FE96B0041173B /* iTermSelectorSwizzler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = iTermSelectorSwizzler.h; sourceTree = "<group>"; };
 		C6675EBB1C4FE96B0041173B /* iTermSelectorSwizzler.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = iTermSelectorSwizzler.m; sourceTree = "<group>"; };
 		D3CE2D4C1A00936F0098ED99 /* PSMDarkTabStyle.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PSMDarkTabStyle.h; sourceTree = "<group>"; };
@@ -9537,6 +9542,7 @@
 				FB4CECF4AC4392B21E87A07B /* iTermRemotePreferences.h */,
 				A6C537C11939507100A08C18 /* iTermRestorableSession.h */,
 				A6BDB0B91B470CBE00F511E6 /* iTermRootTerminalView.h */,
+				93EDFB853885460AA504C6B9 /* iTermLayoutCalculator.h */,
 				1D8CE03B195A143100FE1BEE /* iTermRule.h */,
 				A63BA39318A9CB43002BE075 /* iTermSelection.h */,
 				A6E77F791A23D1A5009B1CB6 /* iTermSelectionScrollHelper.h */,
@@ -10290,6 +10296,7 @@
 				1DB40AA01B221028005B83C7 /* iTermNoColorAccessoryButton.m */,
 				A682DE95190C5E7000BE8758 /* iTermProgressIndicator.m */,
 				A6BDB0BA1B470CBE00F511E6 /* iTermRootTerminalView.m */,
+				AF19E441FF824539A275D8DA /* iTermLayoutCalculator.m */,
 				1DDC093D1B4DB97500B1A910 /* iTermRoundedCornerScrollView.h */,
 				1DDC093E1B4DB97500B1A910 /* iTermRoundedCornerScrollView.m */,
 				A667F3951B4F71AE00705186 /* iTermSavePanel.h */,
@@ -15219,6 +15226,7 @@
 			files = (
 				A626DF0B217D3034005600F9 /* iTermTabBarAccessoryViewController.h in Headers */,
 				A667190A1DCE36C3000CE608 /* iTermRootTerminalView.h in Headers */,
+				D5EF7F4036204859817200E0 /* iTermLayoutCalculator.h in Headers */,
 				A667190B1DCE36C3000CE608 /* iTermCarbonHotKeyController.h in Headers */,
 				A6E7352A20EBF3270034FCFD /* iTermSetCurrentTerminalHelper.h in Headers */,
 				A6F66BA32C2E2C0700DC3569 /* iTermMutableOrderedSet.h in Headers */,
@@ -15770,6 +15778,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				A6BDB0BB1B470CBE00F511E6 /* iTermRootTerminalView.h in Headers */,
+				A445EEDC85D346B39596FDC3 /* iTermLayoutCalculator.h in Headers */,
 				A6CEC11B1DCE8146009F4FD2 /* GPBCodedOutputStream.h in Headers */,
 				A6CEC1251DCE8146009F4FD2 /* GPBProtocolBuffers.h in Headers */,
 				A6CC16541CF012E300E8C148 /* iTermCarbonHotKeyController.h in Headers */,
@@ -19056,6 +19065,7 @@
 				A69FF3CD2E1467D800AFF7A8 /* iTermFileProviderServiceError.swift in Sources */,
 				A619DC4E2CD5A1D0004B8996 /* iTermCoreTextLineRenderingHelper.m in Sources */,
 				A606CBF42145AF4800B3A97E /* iTermRootTerminalView.m in Sources */,
+				4C4BE71D1C8948BD8CD672BC /* iTermLayoutCalculator.m in Sources */,
 				A605A1DB2EB292530072ED7E /* NSButton+iTerm.swift in Sources */,
 				A65660D52372A4A600DC6744 /* iTermCache.m in Sources */,
 				A6EC937524E78A5100EEADEF /* iTermEditSnippetWindowController.m in Sources */,

iTerm2.xcodeproj/project.pbxproj

@@ -1156,10 +1156,6 @@ - (void)rootTerminalViewDidResizeContentArea {
     [self fitTabsToWindow];
 }
 
-- (CGFloat)tabviewWidth {
-    return _contentView.tabviewWidth;
-}
-
 - (void)toggleBroadcastingToCurrentSession:(id)sender {
     [_broadcastInputHelper toggleSession:self.currentSession.guid];
 }
@@ -10456,10 +10452,29 @@ - (iTermStatusBarViewController *)rootTerminalViewSharedStatusBarViewController
     return self.currentSession.statusBarViewController;
 }
 
-- (BOOL)rootTerminalViewWindowHasFullSizeContentView {
-    return [PseudoTerminal windowTypeHasFullSizeContentView:self.windowType];
-}
-
+// Returns YES when the tab bar accessory in fullscreen overlaps the content area.
+// This happens when the window uses NSWindowStyleMaskFullSizeContentView, which
+// causes the content view to extend under the title bar area.
+//
+// This is used by tabViewFrameByShrinkingForFullScreenTabBar to determine if
+// frame shrinking is needed. It is mutually exclusive with
+// rootTerminalViewShouldLeaveEmptyAreaAtTop, which handles the transitional state.
+- (BOOL)rootTerminalViewFullScreenTabBarAccessoryOverlapsContent {
+    // Check the actual window's styleMask rather than savedWindowType, because
+    // during window transitions the styleMask is the authoritative source.
+    return (self.window.styleMask & NSWindowStyleMaskFullSizeContentView) == NSWindowStyleMaskFullSizeContentView;
+}
+
+// Returns YES when empty space should be reserved at the top of the content area
+// for a tab bar that is "on loan" to the titlebar accessory system but is not
+// yet positioned as an accessory.
+//
+// This handles the transitional state during fullscreen entry/exit when:
+// 1. The tab bar control has been borrowed (tabBarControlOnLoan == YES)
+// 2. But it's not yet (or no longer) being shown as a titlebar accessory
+//
+// This is mutually exclusive with tabViewFrameByShrinkingForFullScreenTabBar,
+// which handles the case when the tab bar IS a functioning accessory.
 - (BOOL)rootTerminalViewShouldLeaveEmptyAreaAtTop {
     if ([PseudoTerminal windowTypeHasFullSizeContentView:self.windowType]) {
         DLog(@"YES because window type %@ has full size content view", @(self.windowType));
@@ -10732,7 +10747,23 @@ - (BOOL)tabBarIsVisibleInTitleBarAccessory {
     return _contentView.tabBarControlOnLoan;
 }
 
-// See the note in windowDecorationSize about this hack.
+// Returns YES during the transitional state when the tab bar is about to stop
+// being a titlebar accessory.
+//
+// This happens when:
+// 1. tabBarShouldBeAccessory returns NO (tab bar should NOT be an accessory)
+// 2. BUT tabBarControlOnLoan is YES (tab bar IS currently loaned to accessory system)
+//
+// This transitional state occurs when:
+// - Going from 2+ tabs to 1 tab in fullscreen (tab bar will hide)
+// - Exiting fullscreen (tab bar will return to normal position)
+//
+// During this state, we need to compensate in window size calculations because
+// the tab bar is still physically present as an accessory but should not be
+// counted in the window's decoration size.
+//
+// See also: rootTerminalViewShouldLeaveEmptyAreaAtTop (handles layout during transition)
+// See also: windowDecorationSize (uses this to adjust decoration height)
 - (BOOL)shouldCompensateForDisappearingTabBarAccessory {
     return (!self.tabBarShouldBeAccessory && _contentView.tabBarControlOnLoan);
 }

sources/PseudoTerminal.m

@@ -55,6 +55,7 @@
 #import "iTermKeyBindingAction.h"
 #import "iTermKeystroke.h"
 #import "iTermKeystrokeFormatter.h"
+#import "iTermLayoutCalculator.h"
 #import "iTermLogoGenerator.h"
 #import "iTermMenuBarObserver.h"
 #import "iTermMetalBufferPool.h"

sources/iTerm2SharedARC-Bridging-Header.h

@@ -0,0 +1,120 @@
+//
+//  iTermLayoutCalculator.h
+//  iTerm2
+//
+//  Created by George Nachman on 2/25/26.
+//
+//  Pure layout calculation logic extracted from iTermRootTerminalView for testability.
+//  This class takes layout inputs (dimensions, flags) and returns frame rects
+//  with no direct AppKit dependencies.
+//
+
+#import <Foundation/Foundation.h>
+#import <CoreGraphics/CoreGraphics.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+/// Input parameters for layout calculation.
+/// All dimensions are in points.
+typedef struct {
+    /// Size of the window's content view
+    CGFloat contentViewWidth;
+    CGFloat contentViewHeight;
+
+    /// Tab bar dimensions
+    CGFloat tabBarHeight;
+    CGFloat leftTabBarWidth;
+
+    /// Toolbelt dimensions
+    CGFloat toolbeltWidth;
+    BOOL shouldShowToolbelt;
+
+    /// Status bar
+    CGFloat statusBarHeight;
+    BOOL hasStatusBar;
+    BOOL statusBarOnTop;  // YES for top, NO for bottom
+
+    /// Tab bar state flags
+    BOOL tabBarVisible;
+    BOOL tabBarOnLoan;
+    BOOL tabBarFlashing;
+    BOOL tabBarShouldBeAccessory;
+    BOOL tabBarAccessoryOverlapsContent;
+
+    /// Fullscreen state
+    BOOL enteringFullscreen;
+    BOOL inFullscreen;
+
+    /// Tab position (PSMTab_TopTab, PSMTab_BottomTab, PSMTab_LeftTab)
+    int tabPosition;
+
+    /// Division view
+    BOOL divisionViewVisible;
+    CGFloat divisionViewHeight;
+
+    /// Notch inset (for MacBooks with notch in fullscreen)
+    CGFloat notchInset;
+
+    /// Whether to leave empty area at top for transitional tab bar state
+    BOOL shouldLeaveEmptyAreaAtTop;
+
+    /// Whether to draw window title in place of tab bar
+    BOOL drawWindowTitleInPlaceOfTabBar;
+} iTermLayoutInputs;
+
+/// Output frame rects from layout calculation.
+typedef struct {
+    /// Frame for the tab view (main content area)
+    CGRect tabViewFrame;
+
+    /// Frame for the status bar container
+    CGRect statusBarFrame;
+
+    /// Frame for the toolbelt
+    CGRect toolbeltFrame;
+
+    /// Frame for the tab bar
+    CGRect tabBarFrame;
+
+    /// Decoration heights (space consumed by decorations at top/bottom)
+    CGFloat decorationHeightTop;
+    CGFloat decorationHeightBottom;
+} iTermLayoutOutputs;
+
+/// Pure layout calculator with no AppKit dependencies.
+/// All layout logic is encapsulated here for easy unit testing.
+@interface iTermLayoutCalculator : NSObject
+
+/// Calculate layout frames given input parameters.
+/// This is a pure function with no side effects.
++ (iTermLayoutOutputs)calculateLayoutWithInputs:(iTermLayoutInputs)inputs;
+
+/// Calculate the tab view frame, optionally shrinking for fullscreen tab bar.
+/// This handles the case when the tab bar is a titlebar accessory that overlaps content.
++ (CGRect)tabViewFrameByShrinkingForFullScreenTabBar:(CGRect)frame
+                                          withInputs:(iTermLayoutInputs)inputs
+    NS_SWIFT_NAME(tabViewFrame(byShrinkingForFullScreenTabBar:with:));
+
+/// Calculate the toolbelt frame.
++ (CGRect)toolbeltFrameWithInputs:(iTermLayoutInputs)inputs;
+
+/// Calculate layout for hidden tab bar case.
++ (iTermLayoutOutputs)calculateLayoutWithHiddenTabBarInputs:(iTermLayoutInputs)inputs;
+
+/// Calculate layout for visible top tab bar.
++ (iTermLayoutOutputs)calculateLayoutWithVisibleTopTabBarInputs:(iTermLayoutInputs)inputs;
+
+/// Calculate layout for visible bottom tab bar.
++ (iTermLayoutOutputs)calculateLayoutWithVisibleBottomTabBarInputs:(iTermLayoutInputs)inputs;
+
+/// Calculate layout for visible left tab bar.
++ (iTermLayoutOutputs)calculateLayoutWithVisibleLeftTabBarInputs:(iTermLayoutInputs)inputs;
+
+@end
+
+// Tab position constants (matching PSMTabBarControl)
+extern const int kLayoutTabPositionTop;
+extern const int kLayoutTabPositionBottom;
+extern const int kLayoutTabPositionLeft;
+
+NS_ASSUME_NONNULL_END