feat: Add verbosity shortcuts (experimental) (#9481)

## The basics

- [x] I [validated my changes](https://developers.google.com/blockly/guides/contribute/core#making_and_verifying_a_change)

## The details
### Resolves

Fixes part of RaspberryPiFoundation/blockly-keyboard-experimentation#764
Fixes part of #9450 (infrastructure needs)

### Proposed Changes

Introduces support for two new "where am I?" shortcuts for helping to provide location context for users:
- `I`: re-reads the current selected block with full verbosity (i.e. also includes the block's field types with their values in the readout).
- `shift+I`: reads the current selected block's parent with full verbosity.

Note that this includes some functional changes to `Field` to allow for more powerful customization of a field's ARIA representation (by splitting up value and type), though a field's value defaults potentially to null which will be ignored in the final ARIA computed label. This seems necessary per the discussion here: https://github.com/RaspberryPiFoundation/blockly/pull/9470/files#r2541508565 but more consideration may be needed here as part of #9307.

Some limitations in the new shortcuts:
- They will not read out anything if a block is not selected (e.g. for fields and icons).
- They read out input blocks when the input block is selected.
- They cannot read out anything while in move mode (due to the behavior here in the plugin which automatically cancels moves if an unknown shortcut is pressed: https://github.com/RaspberryPiFoundation/blockly-keyboard-experimentation/blob/a36f3662b05c2ddcd18bde8745777fff8dc3df31/src/actions/mover.ts#L166-L191).
- The readout is limited by the problems of dynamic ARIA announcements (per #9460).

### Reason for Changes

RaspberryPiFoundation/blockly-keyboard-experimentation#764 provides context on the specific needs addressed here.

### Test Coverage

Self tested. No new automated tests needed for experimental work.

### Documentation

No new documentation needed for experimental work.

### Additional Information

This was spun out of #9470 with the intent of getting shortcuts initially working checked in even if the entirety of the experience is incomplete.

Author: BenHenning

blocks/math.ts

@@ -32,7 +32,7 @@ export const blocks = createBlockDefinitionsFromJsonArray([
         'type': 'field_number',
         'name': 'NUM',
         'value': 0,
-        'ariaName': 'Number',
+        'ariaTypeName': 'Number',
       },
     ],
     'output': 'Number',
@@ -55,7 +55,7 @@ export const blocks = createBlockDefinitionsFromJsonArray([
       {
         'type': 'field_dropdown',
         'name': 'OP',
-        'ariaName': 'Arithmetic operation',
+        'ariaTypeName': 'Arithmetic operation',
         'options': [
           ['%{BKY_MATH_ADDITION_SYMBOL}', 'ADD', 'Plus'],
           ['%{BKY_MATH_SUBTRACTION_SYMBOL}', 'MINUS', 'Minus'],

core/block_svg.ts

@@ -242,8 +242,11 @@ export class BlockSvg
     );
   }
 
-  private computeAriaLabel(): string {
-    const {commaSeparatedSummary, inputCount} = buildBlockSummary(this);
+  computeAriaLabel(verbose: boolean = false): string {
+    const {commaSeparatedSummary, inputCount} = buildBlockSummary(
+      this,
+      verbose,
+    );
     let inputSummary = '';
     if (inputCount > 1) {
       inputSummary = 'has inputs';
@@ -2029,7 +2032,7 @@ interface BlockSummary {
   inputCount: number;
 }
 
-function buildBlockSummary(block: BlockSvg): BlockSummary {
+function buildBlockSummary(block: BlockSvg, verbose: boolean): BlockSummary {
   let inputCount = 0;
 
   // Produce structured segments
@@ -2059,7 +2062,7 @@ function buildBlockSummary(block: BlockSvg): BlockSummary {
           return true;
         })
         .map((field) => {
-          const text = field.getText() ?? field.getValue();
+          const text = field.computeAriaLabel(verbose);
           // If the block is a full block field, we only want to know if it's an
           // editable field if we're not directly on it.
           if (field.EDITABLE && !field.isFullBlockField() && !isNestedInput) {

core/field.ts

@@ -271,8 +271,67 @@ export abstract class Field<T = any>
     }
   }
 
-  getAriaName(): string | null {
-    return this.config?.ariaName ?? null;
+  /**
+   * Gets a an ARIA-friendly label representation of this field's type.
+   *
+   * @returns An ARIA representation of the field's type or null if it is
+   *     unspecified.
+   */
+  getAriaTypeName(): string | null {
+    return this.config?.ariaTypeName ?? null;
+  }
+
+  /**
+   * Gets a an ARIA-friendly label representation of this field's value.
+   *
+   * Note that implementations should generally always override this value to
+   * ensure a non-null value is returned since the default implementation relies
+   * on 'getValue' which may return null, and a null return value for this
+   * function will prompt ARIA label generation to skip the field's value
+   * entirely when there may be a better contextual placeholder to use, instead,
+   * specific to the field.
+   *
+   * @returns An ARIA representation of the field's value, or null if no value
+   *     is currently defined or known for the field.
+   */
+  getAriaValue(): string | null {
+    const currentValue = this.getValue();
+    return currentValue !== null ? String(currentValue) : null;
+  }
+
+  /**
+   * Computes a descriptive ARIA label to represent this field with configurable
+   * verbosity.
+   *
+   * A 'verbose' label includes type information, if available, whereas a
+   * non-verbose label only contains the field's value.
+   *
+   * Note that this will always return the latest representation of the field's
+   * label which may differ from any previously set ARIA label for the field
+   * itself. Implementations are largely responsible for ensuring that the
+   * field's ARIA label is set correctly at relevant moments in the field's
+   * lifecycle (such as when its value changes).
+   *
+   * Finally, it is never guaranteed that implementations use the label returned
+   * by this method for their actual ARIA label. Some implementations may rely
+   * on other context to convey information like the field's value. Example:
+   * checkboxes represent their checked/non-checked status (i.e. value) through
+   * a separate ARIA property.
+   *
+   * It's possible this returns an empty string if the field doesn't supply type
+   * or value information for certain cases (such as a null value). This will
+   * lead to the field being potentially COMPLETELY HIDDEN for screen reader
+   * navigation.
+   *
+   * @param verbose Whether to include the field's type information in the
+   *     returned label, if available.
+   */
+  computeAriaLabel(verbose: boolean = false): string {
+    const components: Array<string | null> = [this.getAriaValue()];
+    if (verbose) {
+      components.push(this.getAriaTypeName());
+    }
+    return components.filter((item) => item !== null).join(', ');
   }
 
   /**
@@ -1426,7 +1485,7 @@ export interface FieldConfig {
   type: string;
   name?: string;
   tooltip?: string;
-  ariaName?: string;
+  ariaTypeName?: string;
 }
 
 /**

core/field_checkbox.ts

@@ -116,10 +116,18 @@ export class FieldCheckbox extends Field<CheckboxBool> {
     this.recomputeAria();
   }
 
+  override getAriaValue(): string {
+    return this.value_ ? 'checked' : 'not checked';
+  }
+
   private recomputeAria() {
     const element = this.getFocusableElement();
     aria.setRole(element, aria.Role.CHECKBOX);
-    aria.setState(element, aria.State.LABEL, this.getAriaName() ?? 'Checkbox');
+    aria.setState(
+      element,
+      aria.State.LABEL,
+      this.getAriaTypeName() ?? 'Checkbox',
+    );
     aria.setState(element, aria.State.CHECKED, !!this.value_);
   }
 

core/field_dropdown.ts

@@ -202,6 +202,10 @@ export class FieldDropdown extends Field<string> {
     this.recomputeAria();
   }
 
+  override getAriaValue(): string {
+    return this.computeLabelForOption(this.selectedOption);
+  }
+
   protected recomputeAria() {
     if (!this.fieldGroup_) return; // There's no element to set currently.
     const element = this.getFocusableElement();
@@ -214,14 +218,7 @@ export class FieldDropdown extends Field<string> {
       aria.clearState(element, aria.State.CONTROLS);
     }
 
-    const label = [
-      this.computeLabelForOption(this.selectedOption),
-      this.getAriaName(),
-    ]
-      .filter((item) => !!item)
-      .join(', ');
-
-    aria.setState(element, aria.State.LABEL, label);
+    aria.setState(element, aria.State.LABEL, super.computeAriaLabel(true));
   }
 
   /**

core/field_image.ts

@@ -132,6 +132,10 @@ export class FieldImage extends Field<string> {
     }
   }
 
+  override getAriaValue(): string {
+    return this.altText;
+  }
+
   /**
    * Create the block UI for this image.
    */
@@ -159,11 +163,7 @@ export class FieldImage extends Field<string> {
     if (this.isClickable()) {
       this.imageElement.style.cursor = 'pointer';
       aria.setRole(element, aria.Role.BUTTON);
-
-      const label = [this.altText, this.getAriaName()]
-        .filter((item) => !!item)
-        .join(', ');
-      aria.setState(element, aria.State.LABEL, label);
+      aria.setState(element, aria.State.LABEL, super.computeAriaLabel(true));
     } else {
       // The field isn't navigable unless it's clickable.
       aria.setRole(element, aria.Role.PRESENTATION);

core/field_input.ts

@@ -188,13 +188,8 @@ export abstract class FieldInput<T extends InputTypes> extends Field<
    */
   protected recomputeAriaLabel() {
     if (!this.fieldGroup_) return;
-
     const element = this.getFocusableElement();
-    const label = [this.getValue(), this.getAriaName()]
-      .filter((item) => item !== null)
-      .join(', ');
-
-    aria.setState(element, aria.State.LABEL, label);
+    aria.setState(element, aria.State.LABEL, super.computeAriaLabel());
   }
 
   override isFullBlockField(): boolean {

core/shortcut_items.ts

@@ -16,6 +16,7 @@ import {isDeletable as isIDeletable} from './interfaces/i_deletable.js';
 import {isDraggable} from './interfaces/i_draggable.js';
 import {IFocusableNode} from './interfaces/i_focusable_node.js';
 import {KeyboardShortcut, ShortcutRegistry} from './shortcut_registry.js';
+import {aria} from './utils.js';
 import {Coordinate} from './utils/coordinate.js';
 import {KeyCodes} from './utils/keycodes.js';
 import {Rect} from './utils/rect.js';
@@ -33,6 +34,8 @@ export enum names {
   PASTE = 'paste',
   UNDO = 'undo',
   REDO = 'redo',
+  READ_FULL_BLOCK_SUMMARY = 'read_full_block_summary',
+  READ_BLOCK_PARENT_SUMMARY = 'read_block_parent_summary',
 }
 
 /**
@@ -386,6 +389,71 @@ export function registerRedo() {
   ShortcutRegistry.registry.register(redoShortcut);
 }
 
+/**
+ * Registers a keyboard shortcut for re-reading the current selected block's
+ * summary with additional verbosity to help provide context on where the user
+ * is currently navigated (for screen reader users only).
+ */
+export function registerReadFullBlockSummary() {
+  const i = ShortcutRegistry.registry.createSerializedKey(KeyCodes.I, null);
+  const readFullBlockSummaryShortcut: KeyboardShortcut = {
+    name: names.READ_FULL_BLOCK_SUMMARY,
+    preconditionFn(workspace) {
+      return (
+        !workspace.isDragging() &&
+        !getFocusManager().ephemeralFocusTaken() &&
+        !!getFocusManager().getFocusedNode() &&
+        getFocusManager().getFocusedNode() instanceof BlockSvg
+      );
+    },
+    callback(_, e) {
+      const selectedBlock = getFocusManager().getFocusedNode() as BlockSvg;
+      const blockSummary = selectedBlock.computeAriaLabel(true);
+      aria.announceDynamicAriaState(`Current block: ${blockSummary}`);
+      e.preventDefault();
+      return true;
+    },
+    keyCodes: [i],
+  };
+  ShortcutRegistry.registry.register(readFullBlockSummaryShortcut);
+}
+
+/**
+ * Registers a keyboard shortcut for re-reading the current selected block's
+ * parent block summary with additional verbosity to help provide context on
+ * where the user is currently navigated (for screen reader users only).
+ */
+export function registerReadBlockParentSummary() {
+  const shiftI = ShortcutRegistry.registry.createSerializedKey(KeyCodes.I, [
+    KeyCodes.SHIFT,
+  ]);
+  const readBlockParentSummaryShortcut: KeyboardShortcut = {
+    name: names.READ_BLOCK_PARENT_SUMMARY,
+    preconditionFn(workspace) {
+      return (
+        !workspace.isDragging() &&
+        !getFocusManager().ephemeralFocusTaken() &&
+        !!getFocusManager().getFocusedNode() &&
+        getFocusManager().getFocusedNode() instanceof BlockSvg
+      );
+    },
+    callback(_, e) {
+      const selectedBlock = getFocusManager().getFocusedNode() as BlockSvg;
+      const parentBlock = selectedBlock.getParent();
+      if (parentBlock) {
+        const blockSummary = parentBlock.computeAriaLabel(true);
+        aria.announceDynamicAriaState(`Parent block: ${blockSummary}`);
+      } else {
+        aria.announceDynamicAriaState('Current block has no parent');
+      }
+      e.preventDefault();
+      return true;
+    },
+    keyCodes: [shiftI],
+  };
+  ShortcutRegistry.registry.register(readBlockParentSummaryShortcut);
+}
+
 /**
  * Registers all default keyboard shortcut item. This should be called once per
  * instance of KeyboardShortcutRegistry.
@@ -400,6 +468,8 @@ export function registerDefaultShortcuts() {
   registerPaste();
   registerUndo();
   registerRedo();
+  registerReadFullBlockSummary();
+  registerReadBlockParentSummary();
 }
 
 registerDefaultShortcuts();