Skip to content

Commit e9753d5

Browse files
committed
Improve TalkBack and screen reader accessibility
Add comprehensive accessibility support for Android screen readers (TalkBack) to improve terminal and Extra Keys navigation for users with visual impairments. Terminal view accessibility: - Add TerminalAccessibilityHelper using ExploreByTouchHelper to expose terminal lines as individual accessible items - Implement line-by-line navigation with automatic hard-wrapped line merging for continuous paragraph reading - Add text sanitization to remove terminal control characters and backslash continuation markers from announcements - Improve accessibility focus tracking during scrolling and text updates - Suppress redundant status bar announcements during navigation - Differentiate between jitter events and intentional scrolling Extra Keys accessibility: - Add content descriptions to all Extra Keys buttons (e.g., "Up", "Backspace", "Control") for better screen reader announcements - Support navigation keys (arrows, Home, End, Page Up/Down) - Support modifier keys (Ctrl, Alt, Function) - Support special keys (Drawer, Keyboard, Scroll, Paste) This implementation allows screen reader users to: - Navigate terminal output line-by-line using swipe gestures - Read hard-wrapped lines as continuous paragraphs - Access all Extra Keys buttons with proper labels - Use the terminal effectively without visual feedback
1 parent 8aca6db commit e9753d5

File tree

7 files changed

+451
-12
lines changed

7 files changed

+451
-12
lines changed

app/src/main/java/com/termux/app/terminal/TermuxTerminalViewClient.java

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import android.view.KeyEvent;
1414
import android.view.MotionEvent;
1515
import android.view.View;
16+
import android.view.accessibility.AccessibilityManager;
1617
import android.widget.EditText;
1718
import android.widget.ListView;
1819
import android.widget.Toast;
@@ -60,7 +61,7 @@ public class TermuxTerminalViewClient extends TermuxTerminalViewClientBase {
6061
final TermuxTerminalSessionActivityClient mTermuxTerminalSessionActivityClient;
6162

6263
/** Keeping track of the special keys acting as Ctrl and Fn for the soft keyboard and other hardware keys. */
63-
boolean mVirtualControlKeyDown, mVirtualFnKeyDown;
64+
boolean mVirtualControlActive, mVirtualFnActive;
6465

6566
private Runnable mShowSoftKeyboardRunnable;
6667

@@ -309,20 +310,40 @@ private boolean handleVirtualKeys(int keyCode, KeyEvent event, boolean down) {
309310
// Do not steal dedicated buttons from a full external keyboard.
310311
return false;
311312
} else if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) {
312-
mVirtualControlKeyDown = down;
313+
if (down) {
314+
mVirtualControlActive = !mVirtualControlActive;
315+
announceAssertively(mVirtualControlActive ? "Control On" : "Control Off");
316+
}
313317
return true;
314318
} else if (keyCode == KeyEvent.KEYCODE_VOLUME_UP) {
315-
mVirtualFnKeyDown = down;
319+
if (down) {
320+
mVirtualFnActive = !mVirtualFnActive;
321+
announceAssertively(mVirtualFnActive ? "Fn On" : "Fn Off");
322+
}
316323
return true;
317324
}
318325
return false;
319326
}
320327

328+
private void announceAssertively(String text) {
329+
AccessibilityManager am = (AccessibilityManager) mActivity.getSystemService(Context.ACCESSIBILITY_SERVICE);
330+
if (am != null && am.isEnabled()) {
331+
am.interrupt();
332+
}
333+
mActivity.getTerminalView().announceForAccessibility(text);
334+
}
335+
321336

322337

323338
@Override
324339
public boolean readControlKey() {
325-
return readExtraKeysSpecialButton(SpecialButton.CTRL) || mVirtualControlKeyDown;
340+
if (readExtraKeysSpecialButton(SpecialButton.CTRL)) return true;
341+
if (mVirtualControlActive) {
342+
mVirtualControlActive = false;
343+
announceAssertively("Control Off");
344+
return true;
345+
}
346+
return false;
326347
}
327348

328349
@Override
@@ -337,7 +358,13 @@ public boolean readShiftKey() {
337358

338359
@Override
339360
public boolean readFnKey() {
340-
return readExtraKeysSpecialButton(SpecialButton.FN);
361+
if (readExtraKeysSpecialButton(SpecialButton.FN)) return true;
362+
if (mVirtualFnActive) {
363+
mVirtualFnActive = false;
364+
announceAssertively("Fn Off");
365+
return true;
366+
}
367+
return false;
341368
}
342369

343370
public boolean readExtraKeysSpecialButton(SpecialButton specialButton) {
@@ -359,7 +386,10 @@ public boolean onLongPress(MotionEvent event) {
359386

360387
@Override
361388
public boolean onCodePoint(final int codePoint, boolean ctrlDown, TerminalSession session) {
362-
if (mVirtualFnKeyDown) {
389+
if (mVirtualFnActive) {
390+
mVirtualFnActive = false;
391+
announceAssertively("Fn Off");
392+
363393
int resultingKeyCode = -1;
364394
int resultingCodePoint = -1;
365395
boolean altDown = false;
@@ -448,7 +478,7 @@ public boolean onCodePoint(final int codePoint, boolean ctrlDown, TerminalSessio
448478
case 'q':
449479
case 'k':
450480
mActivity.toggleTerminalToolbar();
451-
mVirtualFnKeyDown=false; // force disable fn key down to restore keyboard input into terminal view, fixes termux/termux-app#1420
481+
// mVirtualFnActive=false; // Keep Manual Toggle
452482
break;
453483
}
454484

terminal-view/build.gradle

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ android {
66

77
dependencies {
88
implementation "androidx.annotation:annotation:1.3.0"
9+
implementation "androidx.core:core:1.6.0"
10+
implementation "androidx.customview:customview:1.1.0"
911
api project(":terminal-emulator")
1012
}
1113

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package com.termux.view;
2+
3+
import android.content.Context;
4+
import android.graphics.Rect;
5+
import android.os.Bundle;
6+
import android.os.SystemClock;
7+
import android.view.InputDevice;
8+
import android.view.MotionEvent;
9+
import android.view.inputmethod.InputMethodManager;
10+
11+
import androidx.annotation.NonNull;
12+
import androidx.core.view.accessibility.AccessibilityNodeInfoCompat;
13+
import androidx.customview.widget.ExploreByTouchHelper;
14+
15+
import com.termux.terminal.TerminalEmulator;
16+
17+
import java.util.List;
18+
19+
public class TerminalAccessibilityHelper extends ExploreByTouchHelper {
20+
private final TerminalView mView;
21+
22+
public TerminalAccessibilityHelper(@NonNull TerminalView view) {
23+
super(view);
24+
mView = view;
25+
}
26+
27+
@Override
28+
protected int getVirtualViewAt(float x, float y) {
29+
if (mView.mEmulator == null || mView.mRenderer == null) return HOST_ID;
30+
int row = (int) (y / mView.mRenderer.getFontLineSpacing());
31+
if (row >= 0 && row < mView.mEmulator.mRows) {
32+
// Find the start of the logical block (paragraph) this row belongs to
33+
int startRow = row;
34+
while (startRow > 0 && mView.mEmulator.getScreen().getLineWrap(mView.mTopRow + startRow - 1)) {
35+
startRow--;
36+
}
37+
return startRow;
38+
}
39+
return HOST_ID;
40+
}
41+
42+
@Override
43+
protected void getVisibleVirtualViews(List<Integer> virtualViewIds) {
44+
if (mView.mEmulator == null) return;
45+
for (int i = 0; i < mView.mEmulator.mRows; ) {
46+
virtualViewIds.add(i);
47+
int current = i;
48+
while (current < mView.mEmulator.mRows - 1 &&
49+
mView.mEmulator.getScreen().getLineWrap(mView.mTopRow + current)) {
50+
current++;
51+
}
52+
i = current + 1;
53+
}
54+
}
55+
56+
@Override
57+
protected void onPopulateNodeForVirtualView(int virtualViewId, @NonNull AccessibilityNodeInfoCompat node) {
58+
if (mView.mEmulator == null || mView.mRenderer == null) {
59+
node.setText("");
60+
node.setBoundsInParent(new Rect(0, 0, 1, 1));
61+
return;
62+
}
63+
64+
int startRow = virtualViewId;
65+
int endRow = startRow;
66+
while (endRow < mView.mEmulator.mRows - 1 &&
67+
mView.mEmulator.getScreen().getLineWrap(mView.mTopRow + endRow)) {
68+
endRow++;
69+
}
70+
71+
// Get text for the logical block
72+
int externalStartRow = mView.mTopRow + startRow;
73+
int externalEndRow = mView.mTopRow + endRow;
74+
75+
// getSelectedText(..., Y2) is inclusive in TerminalBuffer
76+
String text = mView.mEmulator.getScreen().getSelectedText(0, externalStartRow, mView.mEmulator.mColumns, externalEndRow, true, false).trim();
77+
if (text.isEmpty()) text = "Blank";
78+
79+
node.setText(text);
80+
node.setContentDescription(text);
81+
82+
// Bounds: from top of startRow to bottom of endRow
83+
int top = startRow * mView.mRenderer.getFontLineSpacing();
84+
int bottom = (endRow + 1) * mView.mRenderer.getFontLineSpacing();
85+
int width = mView.getWidth();
86+
node.setBoundsInParent(new Rect(0, top, width, bottom));
87+
88+
node.addAction(AccessibilityNodeInfoCompat.ACTION_CLICK);
89+
}
90+
91+
@Override
92+
protected boolean onPerformActionForVirtualView(int virtualViewId, int action, Bundle arguments) {
93+
if (action == AccessibilityNodeInfoCompat.ACTION_CLICK) {
94+
// 1. Request Focus & Show Keyboard
95+
mView.requestFocus();
96+
InputMethodManager imm = (InputMethodManager) mView.getContext().getSystemService(Context.INPUT_METHOD_SERVICE);
97+
if (imm != null) {
98+
imm.showSoftInput(mView, 0);
99+
}
100+
101+
// 2. Simulate Mouse Click to move cursor (if supported by app like Emacs)
102+
// We aim for the start of the line (x=5 pixels buffer) vertically centered in the row
103+
// Check if mouse tracking is active to avoid sending escape codes to shells that don't support it.
104+
if (mView.mEmulator.isMouseTrackingActive()) {
105+
int row = virtualViewId;
106+
float y = (row + 0.5f) * mView.mRenderer.getFontLineSpacing();
107+
float x = 5.0f;
108+
109+
long downTime = SystemClock.uptimeMillis();
110+
MotionEvent eventDown = MotionEvent.obtain(downTime, downTime, MotionEvent.ACTION_DOWN, x, y, 0);
111+
eventDown.setSource(InputDevice.SOURCE_MOUSE);
112+
mView.sendMouseEventCode(eventDown, TerminalEmulator.MOUSE_LEFT_BUTTON, true);
113+
eventDown.recycle();
114+
115+
MotionEvent eventUp = MotionEvent.obtain(downTime, downTime + 10, MotionEvent.ACTION_UP, x, y, 0);
116+
eventUp.setSource(InputDevice.SOURCE_MOUSE);
117+
mView.sendMouseEventCode(eventUp, TerminalEmulator.MOUSE_LEFT_BUTTON, false);
118+
eventUp.recycle();
119+
}
120+
121+
return true;
122+
}
123+
return false;
124+
}
125+
}

0 commit comments

Comments
 (0)