diff --git a/examples/simple_chat/lib/main.dart b/examples/simple_chat/lib/main.dart index 0802bef55..2f57fe983 100644 --- a/examples/simple_chat/lib/main.dart +++ b/examples/simple_chat/lib/main.dart @@ -48,6 +48,11 @@ class _ChatScreenState extends State { giving exactly one response for each user message. Your responses should contain acknowledgment of the user message. + + When controlling the UI, you can add, update and replace surfaces. + - Use 'add' to create new surfaces. + - Use 'update' to make small changes to existing surfaces. + - Use 'replace' to completely change the content of a surface. ''', catalog: CoreCatalogItems.asCatalog(), onSurfaceAdded: _handleSurfaceAdded, diff --git a/examples/travel_app/lib/main.dart b/examples/travel_app/lib/main.dart index f11ab7125..27339a35b 100644 --- a/examples/travel_app/lib/main.dart +++ b/examples/travel_app/lib/main.dart @@ -132,7 +132,7 @@ class _TravelPlannerPageState extends State { _conversation.removeWhere( (m) => m is AiUiMessage && m.surfaceId == surfaceId, ); - case SurfaceUpdated(:final surfaceId, :final definition): + case SurfaceChanged(:final surfaceId, :final definition): final index = _conversation.lastIndexWhere( (m) => m is AiUiMessage && m.surfaceId == surfaceId, ); @@ -375,7 +375,6 @@ to the user. of time, the budget, preferred activity types etc. Then, when the user clicks search, you should update the surface to have -<<<<<<< HEAD a Column with the existing inputGroup, an itineraryWithDetails. When creating the itinerary, include all necessary `itineraryEntry` items for hotels and transport with generic details and a status of `choiceRequired`. @@ -424,11 +423,15 @@ update existing content. - Adding surfaces: Most of the time, you should only add new surfaces to the conversation. This is less confusing for the user, because they can easily find this new content at the bottom of the conversation. -- Updating surfaces: You should update surfaces when you are running an -iterative search flow, e.g. the user is adjusting filter values and generating -an itinerary or a booking accomodation etc. This is less confusing for the user -because it avoids confusing the conversation with many versions of the same -itinerary etc. +- Updating surfaces: You should update surfaces when you want to make small + changes to an existing surface, for example, when you are running an + iterative search flow, e.g. the user is adjusting filter values and + generating an itinerary or a booking accomodation etc. This is less confusing + for the user because it avoids confusing the conversation with many versions + of the same itinerary etc. +- Replacing surfaces: You should replace surfaces when you want to completely + change the content of a surface, for example, when you are starting a new + search or moving to a different part of the conversation flow. When processing a user message or event, you should add or update one surface and then call provideFinalOutput to return control to the user. Never continue diff --git a/packages/flutter_genui/IMPLEMENTATION.md b/packages/flutter_genui/IMPLEMENTATION.md index 611dbb7eb..1554cf115 100644 --- a/packages/flutter_genui/IMPLEMENTATION.md +++ b/packages/flutter_genui/IMPLEMENTATION.md @@ -61,7 +61,7 @@ This layer is responsible for all communication with the generative AI model. This is the central nervous system of the package, orchestrating the state of all generated UI surfaces. -- **`GenUiManager`**: The core state manager for the dynamic UI. It maintains a map of all active UI "surfaces", where each surface is represented by a `UiDefinition`. It provides the tools (`addOrUpdateSurface`, `deleteSurface`) that the AI uses to manipulate the UI. It exposes a stream of `GenUiUpdate` events (`SurfaceAdded`, `SurfaceUpdated`, `SurfaceRemoved`) so that the application can react to changes. It also acts as the `GenUiHost` for the `GenUiSurface` widget. +- **`GenUiManager`**: The core state manager for the dynamic UI. It maintains a map of all active UI "surfaces", where each surface is represented by a `UiDefinition`. It provides the tools (`addOrUpdateSurface`, `deleteSurface`) that the AI uses to manipulate the UI. It exposes a stream of `GenUiUpdate` events (`SurfaceAdded`, `SurfaceChanged`, `SurfaceRemoved`) so that the application can react to changes. It also acts as the `GenUiHost` for the `GenUiSurface` widget. - **`ui_tools.dart`**: Contains the `AddOrUpdateSurfaceTool` and `DeleteSurfaceTool` classes that wrap the `GenUiManager`'s methods, making them available to the AI. ### 3. UI Model Layer (`lib/src/model/`) diff --git a/packages/flutter_genui/lib/src/core/genui_manager.dart b/packages/flutter_genui/lib/src/core/genui_manager.dart index 829fa7e95..e2e9c1535 100644 --- a/packages/flutter_genui/lib/src/core/genui_manager.dart +++ b/packages/flutter_genui/lib/src/core/genui_manager.dart @@ -21,7 +21,7 @@ import 'ui_tools.dart'; /// A sealed class representing an update to the UI managed by [GenUiManager]. /// -/// This class has three subclasses: [SurfaceAdded], [SurfaceUpdated], and +/// This class has three subclasses: [SurfaceAdded], [SurfaceChanged], and /// [SurfaceRemoved]. sealed class GenUiUpdate { /// Creates a [GenUiUpdate] for the given [surfaceId]. @@ -42,10 +42,10 @@ class SurfaceAdded extends GenUiUpdate { } /// Fired when an existing surface is modified. -class SurfaceUpdated extends GenUiUpdate { - /// Creates a [SurfaceUpdated] event for the given [surfaceId] and +class SurfaceChanged extends GenUiUpdate { + /// Creates a [SurfaceChanged] event for the given [surfaceId] and /// [definition]. - const SurfaceUpdated(super.surfaceId, this.definition); + const SurfaceChanged(super.surfaceId, this.definition); /// The new definition of the surface. final UiDefinition definition; @@ -119,7 +119,7 @@ class GenUiManager implements GenUiHost { if (event is! UiActionEvent) throw ArgumentError('Unexpected event type'); final stateValue = valueStore.forSurface(event.surfaceId); final eventString = - 'Action: ${jsonEncode(event.value)}\n' + 'Action: ${jsonEncode(event.value)}\\n' 'Current state: ${jsonEncode(stateValue)}'; _onSubmit.add(UserMessage([TextPart(eventString)])); } @@ -164,19 +164,58 @@ class GenUiManager implements GenUiHost { /// If a surface with the given ID does not exist, a new one is created. /// Otherwise, the existing surface is updated. void addOrUpdateSurface(String surfaceId, JsonMap definition) { - final uiDefinition = UiDefinition.fromMap({ - 'surfaceId': surfaceId, - ...definition, - }); + final action = definition['action'] as String? ?? 'replace'; final notifier = surface(surfaceId); // Gets or creates the notifier. final isNew = notifier.value == null; - notifier.value = uiDefinition; + if (isNew) { + final uiDefinition = UiDefinition.fromMap({ + 'surfaceId': surfaceId, + ...definition, + }); + notifier.value = uiDefinition; genUiLogger.info('Adding surface $surfaceId'); _surfaceUpdates.add(SurfaceAdded(surfaceId, uiDefinition)); } else { - genUiLogger.info('Updating surface $surfaceId'); - _surfaceUpdates.add(SurfaceUpdated(surfaceId, uiDefinition)); + switch (action) { + case 'replace': + final uiDefinition = UiDefinition.fromMap({ + 'surfaceId': surfaceId, + ...definition, + }); + notifier.value = uiDefinition; + genUiLogger.info('Replacing surface $surfaceId'); + _surfaceUpdates.add(SurfaceChanged(surfaceId, uiDefinition)); + case 'update': + final currentDefinition = notifier.value!; + assert( + definition['root'] == currentDefinition.root, + 'The root widget ID must be the same for update actions.', + ); + final newWidgetsList = definition['widgets'] as List; + final newWidgetsMap = {}; + for (final widget in newWidgetsList) { + final typedWidget = widget as JsonMap; + newWidgetsMap[typedWidget['id'] as String] = typedWidget; + } + + final updatedWidgets = { + ...currentDefinition.widgets, + ...newWidgetsMap, + }; + + // TODO(andrewkolb): Prune orphaned widgets. + final uiDefinition = UiDefinition.fromMap({ + 'surfaceId': surfaceId, + 'root': currentDefinition.root, + 'widgets': updatedWidgets.values.toList(), + }); + notifier.value = uiDefinition; + genUiLogger.info('Updating surface $surfaceId'); + _surfaceUpdates.add(SurfaceChanged(surfaceId, uiDefinition)); + default: + throw ArgumentError('Invalid action: $action'); + } } } diff --git a/packages/flutter_genui/lib/src/core/ui_tools.dart b/packages/flutter_genui/lib/src/core/ui_tools.dart index fc60a752d..b84b6e704 100644 --- a/packages/flutter_genui/lib/src/core/ui_tools.dart +++ b/packages/flutter_genui/lib/src/core/ui_tools.dart @@ -30,12 +30,23 @@ class AddOrUpdateSurfaceTool extends AiTool { 'action': S.string( description: 'The action to perform. You must choose from the available ' - 'actions. If you choose the `add` action, you must choose a ' - 'new unique surfaceId. If you choose the `update` action, ' - 'you must choose an existing surfaceId.', + 'actions.\n' + '- `add`: Creates a new surface. You must choose a new, ' + 'unique `surfaceId`.\n' + '- `update`: Updates an existing surface by adding or ' + 'replacing individual widgets. This is efficient for small ' + 'changes, as it preserves the rest of the widget tree. The ' + '`root` widget ID must be the same as the original ' + 'surface.\n' + '- `replace`: Replaces the entire content of an existing ' + 'surface. This is for when the entire UI needs to be ' + 'changed.', enumValues: [ if (configuration.actions.allowCreate) 'add', - if (configuration.actions.allowUpdate) 'update', + if (configuration.actions.allowUpdate) ...[ + 'update', + 'replace', + ], ], ), 'surfaceId': S.string( @@ -50,7 +61,9 @@ class AddOrUpdateSurfaceTool extends AiTool { 'root': S.string( description: 'The ID of the root widget. This ID must correspond to ' - 'the ID of one of the widgets in the `widgets` list.', + 'the ID of one of the widgets in the `widgets` list. ' + 'For `update` actions, this must be the same as the ' + 'original surface.', ), 'widgets': S.list( items: catalog.schema, @@ -78,6 +91,8 @@ class AddOrUpdateSurfaceTool extends AiTool { Future invoke(JsonMap args) async { final surfaceId = args['surfaceId'] as String; final definition = args['definition'] as JsonMap; + final action = args['action'] as String; + definition['action'] = action; onAddOrUpdate(surfaceId, definition); return {'surfaceId': surfaceId, 'status': 'SUCCESS'}; } diff --git a/packages/flutter_genui/lib/src/facade/ui_agent.dart b/packages/flutter_genui/lib/src/facade/ui_agent.dart index dd563c804..ef942bffb 100644 --- a/packages/flutter_genui/lib/src/facade/ui_agent.dart +++ b/packages/flutter_genui/lib/src/facade/ui_agent.dart @@ -162,7 +162,7 @@ class UiAgent { ); _addMessage(message); onSurfaceDeleted!.call(update); - } else if (update is SurfaceUpdated) { + } else if (update is SurfaceChanged) { final message = AiUiMessage( definition: update.definition, surfaceId: update.surfaceId, diff --git a/packages/flutter_genui/test/core/genui_manager_test.dart b/packages/flutter_genui/test/core/genui_manager_test.dart index 57883cdc7..8c0361cc1 100644 --- a/packages/flutter_genui/test/core/genui_manager_test.dart +++ b/packages/flutter_genui/test/core/genui_manager_test.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'package:flutter_genui/flutter_genui.dart'; +import 'package:flutter_genui/src/primitives/simple_items.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { @@ -55,15 +56,15 @@ void main() { expect(manager.surfaces['s1']!.value!.root, 'root'); }); - test('addOrUpdateSurface updates an existing surface and fires ' - 'SurfaceUpdated', () async { + test('addOrUpdateSurface with "replace" action updates an existing surface ' + 'and fires SurfaceChanged', () async { final oldDefinition = { 'root': 'root', 'widgets': [ { 'id': 'root', 'widget': { - 'text': {'text': 'Old'}, + 'Text': {'text': 'Old'}, }, }, ], @@ -71,12 +72,13 @@ void main() { manager.addOrUpdateSurface('s1', oldDefinition); final newDefinition = { + 'action': 'replace', 'root': 'root', 'widgets': [ { 'id': 'root', 'widget': { - 'text': {'text': 'New'}, + 'Text': {'text': 'New'}, }, }, ], @@ -86,16 +88,69 @@ void main() { manager.addOrUpdateSurface('s1', newDefinition); final update = await futureUpdate; - expect(update, isA()); + expect(update, isA()); expect(update.surfaceId, 's1'); - final updatedDefinition = (update as SurfaceUpdated).definition; - expect(updatedDefinition.widgets['root'], { - 'id': 'root', - 'widget': { - 'text': {'text': 'New'}, - }, - }); - expect(manager.surfaces['s1']!.value, updatedDefinition); + final changedUpdate = update as SurfaceChanged; + expect( + ((changedUpdate.definition.widgets['root']! as JsonMap)['widget'] + as JsonMap)['Text'], + {'text': 'New'}, + ); + expect(manager.surfaces['s1']!.value, changedUpdate.definition); + }); + + test('addOrUpdateSurface with "update" action updates an existing surface ' + 'and fires SurfaceChanged', () async { + final oldDefinition = { + 'root': 'root', + 'widgets': [ + { + 'id': 'root', + 'widget': { + 'Text': {'text': 'Old'}, + }, + }, + { + 'id': 'child', + 'widget': { + 'Text': {'text': 'Child'}, + }, + }, + ], + }; + manager.addOrUpdateSurface('s1', oldDefinition); + + final newDefinition = { + 'action': 'update', + 'root': 'root', + 'widgets': [ + { + 'id': 'root', + 'widget': { + 'Text': {'text': 'New'}, + }, + }, + ], + }; + + final futureUpdate = manager.surfaceUpdates.first; + manager.addOrUpdateSurface('s1', newDefinition); + final update = await futureUpdate; + + expect(update, isA()); + expect(update.surfaceId, 's1'); + final changedUpdate = update as SurfaceChanged; + expect( + ((changedUpdate.definition.widgets['root']! as JsonMap)['widget'] + as JsonMap)['Text'], + {'text': 'New'}, + ); + expect( + ((changedUpdate.definition.widgets['child']! as JsonMap)['widget'] + as JsonMap)['Text'], + {'text': 'Child'}, + ); + expect(manager.surfaces['s1']!.value, changedUpdate.definition); }); test('deleteSurface removes a surface and fires SurfaceRemoved', () async { @@ -105,7 +160,7 @@ void main() { { 'id': 'root', 'widget': { - 'text': {'text': 'Hello'}, + 'Text': {'text': 'Hello'}, }, }, ], diff --git a/packages/flutter_genui/test/core/ui_tools_test.dart b/packages/flutter_genui/test/core/ui_tools_test.dart index b024245c2..6d2de14e0 100644 --- a/packages/flutter_genui/test/core/ui_tools_test.dart +++ b/packages/flutter_genui/test/core/ui_tools_test.dart @@ -28,6 +28,7 @@ void main() { ); final args = { + 'action': 'add', 'surfaceId': 'testSurface', 'definition': { 'root': 'rootWidget',