Skip to content

Commit 6e08b62

Browse files
authored
Virtual Keyboard Handling on iOS (#10020)
On iOS/winit, listen to keyboard show/hide/change notifications and scroll Flickables to keep the element in focus visible. (#9857)
1 parent 3f12bba commit 6e08b62

File tree

13 files changed

+532
-17
lines changed

13 files changed

+532
-17
lines changed

docs/astro/src/content/docs/reference/window/window.mdx

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,23 @@ Some devices, such as mobile phones, allow programs to overlap the system UI. A
8888
<SlintProperty propName="safe-area-inset-right" typeName="length" propertyVisibility="out">
8989
Some devices, such as mobile phones, allow programs to overlap the system UI. A few examples for this are the notch on iPhones, the window buttons on macOS on windows that extend their content over the titlebar and the system bar on Android. This property exposes the amount of space at the right of the window that can be drawn to but where no interactive elements should be placed.
9090
</SlintProperty>
91+
92+
### virtual-keyboard-x
93+
<SlintProperty propName="virtual-keyboard-x" typeName="length" propertyVisibility="out">
94+
On mobile devices, virtual keyboards (aka software keyboards or onscreen keyboards) are displayed on top of the application. When such a keyboard is shown, this property denotes the horizontal position of the left boundary of the rectangle covered by it in window coordinates.
95+
</SlintProperty>
96+
97+
### virtual-keyboard-y
98+
<SlintProperty propName="virtual-keyboard-y" typeName="length" propertyVisibility="out">
99+
On mobile devices, virtual keyboards (aka software keyboards or onscreen keyboards) are displayed on top of the application. When such a keyboard is shown, this property denotes the vertical position of the top boundary of the rectangle covered by it in window coordinates.
100+
</SlintProperty>
101+
102+
### virtual-keyboard-width
103+
<SlintProperty propName="virtual-keyboard-width" typeName="length" propertyVisibility="out">
104+
On mobile devices, virtual keyboards (aka software keyboards or onscreen keyboards) are displayed on top of the application. When such a keyboard is shown, this property denotes the width of the rectangle covered by it in window coordinates or 0 otherwise.
105+
</SlintProperty>
106+
107+
### virtual-keyboard-height
108+
<SlintProperty propName="virtual-keyboard-height" typeName="length" propertyVisibility="out">
109+
On mobile devices, virtual keyboards (aka software keyboards or onscreen keyboards) are displayed on top of the application. When such a keyboard is shown, this property denotes the height of the rectangle covered by it in window coordinates or 0 otherwise.
110+
</SlintProperty>

internal/backends/winit/Cargo.toml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,14 @@ objc2-app-kit = { version = "0.3.2" }
124124
# Enable Skia by default on Apple platforms with iOS, etc. (but not macOS). See also enable_skia_renderer in build.rs
125125
i-slint-renderer-skia = { workspace = true, features = ["default"] }
126126

127+
[target.'cfg(target_os = "ios")'.dependencies]
128+
objc2 = "0.6.3"
129+
objc2-foundation = { version = "0.3.2", default-features = false, features = ["std", "block2", "NSString", "NSNotification", "NSOperation", "NSGeometry", "objc2-core-foundation"] }
130+
objc2-ui-kit = { version = "0.3.2", default-features = false, features = ["UIScreen", "UIWindow", "UIView", "UIViewAnimating", "UIViewPropertyAnimator", "UIResponder", "objc2-core-foundation", "objc2-quartz-core", "block2"] }
131+
objc2-quartz-core = { version = "0.3.2", default-features = false, features = ["CADisplayLink", "CATransaction"] }
132+
# Match version in objc2
133+
block2 = "0.6.2"
134+
127135
[target.'cfg(target_os = "windows")'.dependencies]
128136
windows = { workspace = true, features = ["Win32_UI_WindowsAndMessaging"] }
129137

internal/backends/winit/ios.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
// Copyright © SixtyFPS GmbH <[email protected]>
2+
// SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0
3+
4+
mod keyboard_animator;
5+
mod virtual_keyboard;
6+
7+
pub(crate) use keyboard_animator::KeyboardCurveSampler;
8+
pub(crate) use virtual_keyboard::{register_keyboard_notifications, KeyboardNotifications};
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
// Copyright © SixtyFPS GmbH <[email protected]>
2+
// SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0
3+
4+
use std::cell::{OnceCell, RefCell};
5+
6+
use block2::RcBlock;
7+
use objc2::{define_class, msg_send, rc::Retained, DefinedClass, MainThreadMarker, MainThreadOnly};
8+
use objc2_foundation::{NSDefaultRunLoopMode, NSObject, NSObjectProtocol, NSRect, NSRunLoop};
9+
use objc2_quartz_core::{CADisplayLink, CATransaction};
10+
use objc2_ui_kit::{UIView, UIViewAnimating as _, UIViewAnimationCurve, UIViewPropertyAnimator};
11+
12+
struct DisplayLinkTargetIvars {
13+
view: Retained<UIView>,
14+
callback: Box<dyn Fn(NSRect)>,
15+
animator: RefCell<Option<Retained<UIViewPropertyAnimator>>>,
16+
display_link: OnceCell<Retained<CADisplayLink>>,
17+
}
18+
19+
define_class!(
20+
#[unsafe(super = NSObject)]
21+
#[thread_kind = MainThreadOnly]
22+
#[ivars = DisplayLinkTargetIvars]
23+
struct DisplayLinkTarget;
24+
25+
unsafe impl NSObjectProtocol for DisplayLinkTarget {}
26+
27+
impl DisplayLinkTarget {
28+
#[unsafe(method(tick:))]
29+
fn tick(&self, display_link: &CADisplayLink) {
30+
let this = self.ivars();
31+
if let Some(layer) = unsafe { this.view.layer().presentationLayer() } {
32+
(this.callback)(layer.frame());
33+
}
34+
let mut animator_ref = this.animator.borrow_mut();
35+
if let Some(false) = animator_ref.as_ref().map(|animator| animator.isRunning()) {
36+
display_link.setPaused(true);
37+
*animator_ref = None;
38+
}
39+
}
40+
}
41+
);
42+
43+
impl DisplayLinkTarget {
44+
fn new(
45+
mtm: MainThreadMarker,
46+
view: Retained<UIView>,
47+
callback: impl Fn(NSRect) + 'static,
48+
) -> Retained<Self> {
49+
let this = Self::alloc(mtm).set_ivars(DisplayLinkTargetIvars {
50+
view,
51+
callback: Box::new(callback),
52+
animator: Default::default(),
53+
display_link: OnceCell::new(),
54+
});
55+
unsafe { msg_send![super(this), init] }
56+
}
57+
58+
fn set_display_link(&self, display_link: Retained<CADisplayLink>) {
59+
self.ivars().display_link.set(display_link).unwrap();
60+
}
61+
62+
fn stop(&self) {
63+
let ivars = self.ivars();
64+
ivars.display_link.get().unwrap().setPaused(true);
65+
if let Some(animator) = ivars.animator.borrow_mut().take() {
66+
animator.stopAnimation(true);
67+
}
68+
}
69+
70+
fn start(&self, animator: Retained<UIViewPropertyAnimator>) {
71+
let ivars = self.ivars();
72+
animator.startAnimation();
73+
if let Some(old_animator) = ivars.animator.borrow_mut().replace(animator) {
74+
old_animator.stopAnimation(true);
75+
}
76+
ivars.display_link.get().unwrap().setPaused(false);
77+
}
78+
}
79+
80+
/// A helper to sample keyboard animation curves.
81+
/// Since the iOS keyboard animation is not directly accessible, we create a hidden UIView
82+
/// and animate its frame using the same parameters as the keyboard animation.
83+
/// The animation curve used by iOS is private and not documented, but using UIViewPropertyAnimator
84+
/// with the same duration and curve produces identical results.
85+
pub(crate) struct KeyboardCurveSampler {
86+
view: Retained<UIView>,
87+
target: Retained<DisplayLinkTarget>,
88+
mtm: MainThreadMarker,
89+
}
90+
91+
impl KeyboardCurveSampler {
92+
pub(crate) fn new(content_view: &UIView, sampler: impl Fn(NSRect) + 'static) -> Self {
93+
let mtm = MainThreadMarker::new().expect("Must be created on main thread");
94+
let view = UIView::new(mtm);
95+
content_view.addSubview(&view);
96+
97+
let target = DisplayLinkTarget::new(mtm, view.clone(), sampler);
98+
let display_link =
99+
unsafe { CADisplayLink::displayLinkWithTarget_selector(&target, objc2::sel!(tick:)) };
100+
101+
unsafe {
102+
display_link.addToRunLoop_forMode(&NSRunLoop::currentRunLoop(), NSDefaultRunLoopMode);
103+
}
104+
105+
display_link.setPaused(true);
106+
target.set_display_link(display_link);
107+
108+
Self { view, target, mtm }
109+
}
110+
111+
pub(crate) fn start(
112+
&self,
113+
duration: f64,
114+
curve: UIViewAnimationCurve,
115+
begin: NSRect,
116+
end: NSRect,
117+
) {
118+
CATransaction::begin();
119+
CATransaction::setDisableActions(true);
120+
self.target.stop();
121+
self.view.setFrame(begin);
122+
CATransaction::commit();
123+
124+
let view = self.view.clone();
125+
let animations = RcBlock::new(move || {
126+
view.setFrame(end);
127+
});
128+
129+
let animator = UIViewPropertyAnimator::initWithDuration_curve_animations(
130+
UIViewPropertyAnimator::alloc(self.mtm),
131+
duration, // duration is already in seconds
132+
curve,
133+
Some(&animations),
134+
);
135+
136+
self.target.start(animator);
137+
}
138+
}
139+
140+
impl Drop for DisplayLinkTargetIvars {
141+
fn drop(&mut self) {
142+
if let Some(display_link) = self.display_link.get() {
143+
display_link.invalidate();
144+
}
145+
if let Some(animator) = self.animator.borrow_mut().take() {
146+
animator.stopAnimation(true);
147+
}
148+
self.view.removeFromSuperview();
149+
}
150+
}
Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
// Copyright © SixtyFPS GmbH <[email protected]>
2+
// SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0
3+
4+
use std::{cell::RefCell, collections::HashMap, ptr::NonNull, rc::Weak};
5+
6+
use block2::RcBlock;
7+
use objc2_foundation::{
8+
NSContainsRect, NSIntersectsRect, NSNotification, NSNotificationCenter, NSNumber,
9+
NSOperationQueue, NSRect, NSValue,
10+
};
11+
use objc2_ui_kit::{UICoordinateSpace, UIScreen, UIViewAnimationCurve};
12+
use raw_window_handle::HasWindowHandle;
13+
use winit::window::WindowId;
14+
15+
use crate::winitwindowadapter::WinitWindowAdapter;
16+
17+
pub(crate) struct KeyboardNotifications(
18+
[objc2::rc::Retained<objc2::runtime::ProtocolObject<dyn objc2_foundation::NSObjectProtocol>>;
19+
3],
20+
);
21+
22+
impl Drop for KeyboardNotifications {
23+
fn drop(&mut self) {
24+
for notification_object in &self.0 {
25+
unsafe {
26+
objc2_foundation::NSNotificationCenter::defaultCenter()
27+
.removeObserver(notification_object.as_ref());
28+
}
29+
}
30+
}
31+
}
32+
33+
pub(crate) fn register_keyboard_notifications(
34+
active_windows: Weak<RefCell<HashMap<WindowId, Weak<WinitWindowAdapter>>>>,
35+
) -> KeyboardNotifications {
36+
let event_block = RcBlock::new(move |notification: NonNull<NSNotification>| {
37+
if let Some(active_windows) = active_windows.upgrade() {
38+
handle_keyboard_notification(
39+
unsafe { notification.as_ref() },
40+
active_windows.borrow().values(),
41+
);
42+
}
43+
});
44+
let default_center = NSNotificationCenter::defaultCenter();
45+
let main_queue = NSOperationQueue::mainQueue();
46+
KeyboardNotifications(unsafe {
47+
[
48+
objc2_ui_kit::UIKeyboardWillShowNotification,
49+
objc2_ui_kit::UIKeyboardWillHideNotification,
50+
objc2_ui_kit::UIKeyboardWillChangeFrameNotification,
51+
]
52+
.map(|name| {
53+
default_center.addObserverForName_object_queue_usingBlock(
54+
Some(name),
55+
None,
56+
Some(&main_queue),
57+
&event_block,
58+
)
59+
})
60+
})
61+
}
62+
63+
fn handle_keyboard_notification<'a>(
64+
notification: &NSNotification,
65+
windows: impl IntoIterator<Item = &'a Weak<WinitWindowAdapter>>,
66+
) -> Option<()> {
67+
let user_info = notification.userInfo()?;
68+
let is_local = user_info
69+
.objectForKey(unsafe { objc2_ui_kit::UIKeyboardIsLocalUserInfoKey })?
70+
.downcast::<NSNumber>()
71+
.ok()?
72+
.as_bool();
73+
if !is_local {
74+
return Some(());
75+
}
76+
let screen = notification.object()?.downcast::<UIScreen>().ok()?;
77+
let coordinate_space = screen.coordinateSpace();
78+
79+
let frame_begin = unsafe {
80+
user_info
81+
.objectForKey(objc2_ui_kit::UIKeyboardFrameBeginUserInfoKey)?
82+
.downcast::<NSValue>()
83+
.ok()?
84+
.rectValue()
85+
};
86+
let frame_end = unsafe {
87+
user_info
88+
.objectForKey(objc2_ui_kit::UIKeyboardFrameEndUserInfoKey)?
89+
.downcast::<NSValue>()
90+
.ok()?
91+
.rectValue()
92+
};
93+
let animation_duration = user_info
94+
.objectForKey(unsafe { objc2_ui_kit::UIKeyboardAnimationDurationUserInfoKey })?
95+
.downcast::<NSNumber>()
96+
.ok()?
97+
.as_f64();
98+
let curve = UIViewAnimationCurve(
99+
user_info
100+
.objectForKey(unsafe { objc2_ui_kit::UIKeyboardAnimationCurveUserInfoKey })?
101+
.downcast::<NSNumber>()
102+
.unwrap()
103+
.as_isize(),
104+
);
105+
106+
let name = notification.name();
107+
if name.isEqualToString(unsafe { objc2_ui_kit::UIKeyboardWillChangeFrameNotification }) {
108+
for adapter in windows.into_iter() {
109+
let adapter = adapter.upgrade()?;
110+
let raw_window_handle::RawWindowHandle::UiKit(window_handle) =
111+
adapter.winit_window()?.window_handle().ok()?.as_raw()
112+
else {
113+
continue;
114+
};
115+
let view = unsafe { &*(window_handle.ui_view.as_ptr() as *const objc2_ui_kit::UIView) };
116+
let frame_begin = view.convertRect_fromCoordinateSpace(frame_begin, &coordinate_space);
117+
let frame_end = view.convertRect_fromCoordinateSpace(frame_end, &coordinate_space);
118+
119+
// Assumes that the keyboard animation doesn't pass over the window without
120+
// starting or ending while intersecting.
121+
// Although, in this strange edge case we should probably ignore the keyboard anyways.
122+
if NSIntersectsRect(view.bounds(), frame_begin)
123+
|| NSIntersectsRect(view.bounds(), frame_end)
124+
{
125+
adapter.with_keyboard_curve_sampler(|kcs| {
126+
kcs.start(animation_duration, curve, frame_begin, frame_end);
127+
});
128+
}
129+
}
130+
}
131+
132+
Some(())
133+
}

internal/backends/winit/lib.rs

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ mod winitwindowadapter;
2727
use winitwindowadapter::*;
2828
pub(crate) mod event_loop;
2929
mod frame_throttle;
30+
#[cfg(target_os = "ios")]
31+
mod ios;
3032

3133
/// Re-export of the winit crate.
3234
pub use winit;
@@ -522,7 +524,7 @@ pub(crate) struct SharedBackendData {
522524
_requested_graphics_api: Option<RequestedGraphicsAPI>,
523525
#[cfg(enable_skia_renderer)]
524526
skia_context: i_slint_renderer_skia::SkiaSharedContext,
525-
active_windows: RefCell<HashMap<winit::window::WindowId, Weak<WinitWindowAdapter>>>,
527+
active_windows: Rc<RefCell<HashMap<winit::window::WindowId, Weak<WinitWindowAdapter>>>>,
526528
/// List of visible windows that have been created when without the event loop and
527529
/// need to be mapped to a winit Window as soon as the event loop becomes active.
528530
inactive_windows: RefCell<Vec<Weak<WinitWindowAdapter>>>,
@@ -531,6 +533,9 @@ pub(crate) struct SharedBackendData {
531533
not_running_event_loop: RefCell<Option<winit::event_loop::EventLoop<SlintEvent>>>,
532534
event_loop_proxy: winit::event_loop::EventLoopProxy<SlintEvent>,
533535
is_wayland: bool,
536+
#[cfg(target_os = "ios")]
537+
#[allow(unused)]
538+
keyboard_notifications: ios::KeyboardNotifications,
534539
}
535540

536541
impl SharedBackendData {
@@ -582,6 +587,13 @@ impl SharedBackendData {
582587
}
583588
}
584589

590+
let active_windows =
591+
Rc::<RefCell<HashMap<winit::window::WindowId, Weak<WinitWindowAdapter>>>>::default();
592+
593+
#[cfg(target_os = "ios")]
594+
let keyboard_notifications =
595+
ios::register_keyboard_notifications(Rc::downgrade(&active_windows));
596+
585597
let event_loop_proxy = event_loop.create_proxy();
586598
#[cfg(not(target_arch = "wasm32"))]
587599
let clipboard = crate::clipboard::create_clipboard(
@@ -593,13 +605,15 @@ impl SharedBackendData {
593605
_requested_graphics_api: requested_graphics_api,
594606
#[cfg(enable_skia_renderer)]
595607
skia_context: i_slint_renderer_skia::SkiaSharedContext::default(),
596-
active_windows: Default::default(),
608+
active_windows,
597609
inactive_windows: Default::default(),
598610
#[cfg(not(target_arch = "wasm32"))]
599611
clipboard: RefCell::new(clipboard),
600612
not_running_event_loop: RefCell::new(Some(event_loop)),
601613
event_loop_proxy,
602614
is_wayland,
615+
#[cfg(target_os = "ios")]
616+
keyboard_notifications,
603617
})
604618
}
605619

0 commit comments

Comments
 (0)