Skip to content

Added connection support for both omi + omi glasses #4843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MithilSaiReddy wants to merge 4 commits into
base: main
Choose a base branch
from
+147 −61
Open

Added connection support for both omi + omi glasses #4843

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b93d9af
Added connection support for both omi + omi glasses
MithilSaiReddy Feb 17, 2026
5f3acc7
Merge branch 'BasedHardware:main' into main
MithilSaiReddy Mar 3, 2026
9fb3725
chore: consolidate changelog for v0.0.1
github-actions[bot] Mar 3, 2026
999da42
Merge pull request #1 from MithilSaiReddy/changelog/v0.0.1
github-actions[bot] Mar 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 70 additions & 40 deletions app/lib/providers/capture_provider.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ class CaptureProvider extends ChangeNotifier
DebugLogManager.logEvent('app_lifecycle_changed', {
'state': state.name,
'recording_state': recordingState.name,
'has_device': _recordingDevice != null,
'has_device': _audioDevice != null || _glassDevice != null,
'socket_connected': _socket?.state == SocketServiceState.connected,
});

Expand Down Expand Up @@ -236,13 +236,24 @@ class CaptureProvider extends ChangeNotifier
notifyListeners();
}

BtDevice? _recordingDevice;
BtDevice? _audioDevice; // was: BtDevice? _recordingDevice
BtDevice? _glassDevice;

BtDevice? get recordingDevice => _audioDevice ?? _glassDevice;
BtDevice? get _recordingDevice => recordingDevice;

bool get havingRecordingDevice => _audioDevice != null || _glassDevice != null;
bool get isMultiModal => _audioDevice != null && _glassDevice != null;

String? _getConversationSourceFromDevice() {
if (_recordingDevice == null) {
if (_audioDevice != null && _glassDevice != null) {
return 'multimodal';
}
final device = _audioDevice ?? _glassDevice;
if (device == null){
return null;
}
switch (_recordingDevice!.type) {
switch (device.type) {
case DeviceType.friendPendant:
return 'friend_com';
case DeviceType.omi:
Expand Down Expand Up @@ -279,6 +290,7 @@ class CaptureProvider extends ChangeNotifier

StreamSubscription? _bleBytesStream;
StreamSubscription? _blePhotoStream;
StreamSubscription? _glassAudioStream;

get bleBytesStream => _bleBytesStream;

Expand Down Expand Up @@ -324,13 +336,11 @@ class CaptureProvider extends ChangeNotifier

// having a connected device or using the phone's mic for recording
bool get recordingDeviceServiceReady =>
_recordingDevice != null ||
_audioDevice != null ||
_glassDevice != null ||
recordingState == RecordingState.record ||
recordingState == RecordingState.systemAudioRecord;

bool get havingRecordingDevice => _recordingDevice != null;

BtDevice? get recordingDevice => _recordingDevice;

void setHasTranscripts(bool value) {
hasTranscripts = value;
Expand All @@ -344,8 +354,18 @@ class CaptureProvider extends ChangeNotifier
}

void _updateRecordingDevice(BtDevice? device) {
Logger.debug('connected device changed from ${_recordingDevice?.id} to ${device?.id}');
_recordingDevice = device;
if (device == null) {
// null clears BOTH slots (e.g. on full disconnect)
Logger.debug('connected device cleared (both slots)');
_audioDevice = null;
_glassDevice = null;
} else if (device.type == DeviceType.openglass) {
Logger.debug('glass device set: ${device.id}');
_glassDevice = device;
} else {
Logger.debug('audio device set: ${device.id}');
_audioDevice = device;
}
notifyListeners();
}

Expand Down Expand Up @@ -373,9 +393,9 @@ class CaptureProvider extends ChangeNotifier
Logger.debug("Transcription settings changed, refreshing socket connection...");

// Handle device recording
if (_recordingDevice != null) {
if (_audioDevice != null) {
await _socket?.stop(reason: 'transcription settings changed');
BleAudioCodec codec = await _getAudioCodec(_recordingDevice!.id);
BleAudioCodec codec = await _getAudioCodec(_audioDevice!.id);
await _initiateWebsocket(
audioCodec: codec,
force: true,
Expand Down Expand Up @@ -480,12 +500,12 @@ class CaptureProvider extends ChangeNotifier
return;
}

if (_recordingDevice == null) {
Logger.debug("Recording device is null, cannot process voice command");
if (_audioDevice == null) {
Logger.debug("Audio device is null, cannot process voice command");
return;
}

BleAudioCodec codec = await _getAudioCodec(_recordingDevice!.id);
BleAudioCodec codec = await _getAudioCodec(_audioDevice!.id);
if (messageProvider != null) {
await messageProvider?.sendVoiceMessageStreamToServer(
data,
Expand Down Expand Up @@ -635,16 +655,16 @@ class CaptureProvider extends ChangeNotifier
_blesBytesReceived += snapshot.length;

// Command button triggered
bool voiceCommandSupported = _recordingDevice != null
? (_recordingDevice?.type == DeviceType.omi || _recordingDevice?.type == DeviceType.openglass)
bool voiceCommandSupported = _audioDevice != null
? (_audioDevice?.type == DeviceType.omi || _audioDevice?.type == DeviceType.openglass)
: false;
if (_voiceCommandSession != null && voiceCommandSupported) {
_commandBytes.add(snapshot.sublist(3));
}

// Local storage syncs
var checkWalSupported =
(_recordingDevice?.type == DeviceType.omi || _recordingDevice?.type == DeviceType.openglass) &&
(_audioDevice?.type == DeviceType.omi || _audioDevice?.type == DeviceType.openglass) &&
codec.isOpusSupported() &&
(_socket?.state != SocketServiceState.connected || SharedPreferencesUtil().unlimitedLocalStorageEnabled);
if (checkWalSupported != _isWalSupported) {
Expand All @@ -657,7 +677,7 @@ class CaptureProvider extends ChangeNotifier
// Send WS
if (_socket?.state == SocketServiceState.connected) {
final paddingLeft =
(_recordingDevice?.type == DeviceType.omi || _recordingDevice?.type == DeviceType.openglass) ? 3 : 0;
(_audioDevice?.type == DeviceType.omi || _audioDevice?.type == DeviceType.openglass) ? 3 : 0;
final trimmedValue = paddingLeft > 0 ? value.sublist(paddingLeft) : value;
_socket?.send(trimmedValue);

Expand All @@ -677,13 +697,13 @@ class CaptureProvider extends ChangeNotifier
Logger.debug('resetState');
await _cleanupCurrentState();

// Always try to stream audio if a device is present
// Start audio streaming for _audioDevice if present.
await _ensureDeviceSocketConnection();
await _initiateDeviceAudioStreaming();

// Additionally, stream photos if the device supports it
if (_recordingDevice != null) {
var connection = await ServiceManager.instance().device.ensureConnection(_recordingDevice!.id);
// Start photo streaming for _glassDevice if present.
if (_glassDevice != null) {
var connection = await ServiceManager.instance().device.ensureConnection(_glassDevice!.id);
if (connection != null && await connection.hasPhotoStreamingCharacteristic()) {
await _initiateDevicePhotoStreaming();
}
Expand Down Expand Up @@ -736,10 +756,11 @@ class CaptureProvider extends ChangeNotifier
}

Future<void> _ensureDeviceSocketConnection() async {
if (_recordingDevice == null) {
final primaryDevice = _audioDevice ?? _glassDevice;
if (primaryDevice == null) {
return;
}
BleAudioCodec codec = await _getAudioCodec(_recordingDevice!.id);
BleAudioCodec codec = await _getAudioCodec(primaryDevice.id);
var language =
SharedPreferencesUtil().hasSetPrimaryLanguage ? SharedPreferencesUtil().userPrimaryLanguage : "multi";
final customSttConfig = SharedPreferencesUtil().customSttConfig;
Expand All @@ -754,7 +775,7 @@ class CaptureProvider extends ChangeNotifier
}

Future<void> _initiateDeviceAudioStreaming() async {
final device = _recordingDevice;
final device = _audioDevice;
if (device == null) {
return;
}
Expand All @@ -781,8 +802,10 @@ class CaptureProvider extends ChangeNotifier
}

Future<void> _initiateDevicePhotoStreaming() async {
if (_recordingDevice == null) return;
final deviceId = _recordingDevice!.id;
final device = _glassDevice;
if (device == null) return;

final deviceId = device.id;
var connection = await ServiceManager.instance().device.ensureConnection(deviceId);
if (connection == null) return;

Expand All @@ -792,13 +815,11 @@ class CaptureProvider extends ChangeNotifier
final String tempId = 'temp_img_${DateTime.now().millisecondsSinceEpoch}';
final String base64Image = base64Encode(rotatedImageBytes);

// Add placeholder to UI for immediate feedback
photos.add(ConversationPhoto(id: tempId, base64: base64Image, createdAt: DateTime.now()));
photos = List.from(photos);
notifyListeners();

// Chunking Logic
const int chunkSize = 8192; // 8KB chunks
const int chunkSize = 8192;
final totalChunks = (base64Image.length / chunkSize).ceil();

for (int i = 0; i < totalChunks; i++) {
Expand All @@ -815,9 +836,9 @@ class CaptureProvider extends ChangeNotifier
});

if (_socket?.state == SocketServiceState.connected) {
_socket?.send(payload); // Send the JSON string
_socket?.send(payload);
}
await Future.delayed(const Duration(milliseconds: 20)); // Small delay to prevent flooding
await Future.delayed(const Duration(milliseconds: 20));
}
});
notifyListeners();
Expand Down Expand Up @@ -890,9 +911,11 @@ class CaptureProvider extends ChangeNotifier
Future _closeBleStream() async {
await _bleBytesStream?.cancel();
await _blePhotoStream?.cancel();
await _glassAudioStream?.cancel();
_stopMetricsTracking();
if (_recordingDevice != null) {
var connection = await ServiceManager.instance().device.ensureConnection(_recordingDevice!.id);

if (_glassDevice != null) {
var connection = await ServiceManager.instance().device.ensureConnection(_glassDevice!.id);
if (connection != null && await connection.hasPhotoStreamingCharacteristic()) {
await connection.performCameraStopPhotoController();
}
Expand All @@ -904,6 +927,7 @@ class CaptureProvider extends ChangeNotifier
void dispose() {
_bleBytesStream?.cancel();
_blePhotoStream?.cancel();
_glassAudioStream?.cancel();
_socket?.unsubscribe(this);
_keepAliveTimer?.cancel();
_connectionStateListener?.cancel();
Expand Down Expand Up @@ -1338,8 +1362,14 @@ class CaptureProvider extends ChangeNotifier
return;
}

if (_recordingDevice != null) {
BleAudioCodec codec = await _getAudioCodec(_recordingDevice!.id);
if (_audioDevice != null) {
BleAudioCodec codec = await _getAudioCodec(_audioDevice!.id);
await _initiateWebsocket(audioCodec: codec, source: _getConversationSourceFromDevice());
return;
}

if (_glassDevice != null) {
BleAudioCodec codec = await _getAudioCodec(_glassDevice!.id);
await _initiateWebsocket(audioCodec: codec, source: _getConversationSourceFromDevice());
return;
}
Expand Down Expand Up @@ -1825,7 +1855,7 @@ class CaptureProvider extends ChangeNotifier
}

Future<void> pauseDeviceRecording() async {
if (_recordingDevice == null) return;
if (_audioDevice == null) return;

// Pause the BLE stream but keep the device connection
await _bleBytesStream?.cancel();
Expand All @@ -1835,12 +1865,12 @@ class CaptureProvider extends ChangeNotifier
}

Future<void> resumeDeviceRecording() async {
if (_recordingDevice == null) return;
if (_audioDevice == null) return;
_isPaused = false;
// Resume streaming from the device
await _initiateDeviceAudioStreaming();

final deviceId = _recordingDevice!.id;
final deviceId = _audioDevice!.id;
BleAudioCodec codec = await _getAudioCodec(deviceId);
await _wal.getSyncs().phone.onAudioCodecChanged(codec);

Expand Down
Loading