feat(stage-tamagotchi): vision

apps/stage-pocket/package.json

@@ -62,6 +62,7 @@
     "animejs": "^4.2.2",
     "colorjs.io": "^0.6.0",
     "culori": "^4.0.2",
+    "d3": "catalog:",
     "date-fns": "^4.1.0",
     "dompurify": "^3.3.1",
     "driver.js": "^1.4.0",

apps/stage-tamagotchi/package.json

@@ -85,6 +85,7 @@
     "colorjs.io": "^0.6.0",
     "crossws": "^0.4.1",
     "culori": "^4.0.2",
+    "d3": "catalog:",
     "date-fns": "^4.1.0",
     "defu": "^6.1.4",
     "destr": "^2.0.5",

apps/stage-tamagotchi/src/main/windows/settings/rpc/index.electron.ts

@@ -6,9 +6,9 @@ import type { WidgetsWindowManager } from '../../widgets'
 
 import { defineInvokeHandler } from '@moeru/eventa'
 import { createContext } from '@moeru/eventa/adapters/electron/main'
-import { ipcMain } from 'electron'
+import { desktopCapturer, ipcMain, session } from 'electron'
 
-import { electronOpenDevtoolsWindow, electronOpenSettingsDevtools } from '../../../../shared/eventa'
+import { electronOpenDevtoolsWindow, electronOpenSettingsDevtools, modulesVisionPrepareScreenSourceSelection } from '../../../../shared/eventa'
 import { createWidgetsService } from '../../../services/airi/widgets'
 import { createAutoUpdaterService, createScreenService, createWindowService } from '../../../services/electron'
 
@@ -34,4 +34,15 @@ export async function setupSettingsWindowInvokes(params: {
   defineInvokeHandler(context, electronOpenDevtoolsWindow, async (payload) => {
     await params.devtoolsMarkdownStressWindow.openWindow(payload?.route)
   })
+
+  defineInvokeHandler(context, modulesVisionPrepareScreenSourceSelection, async () => {
+    // TODO(@sumimakito): Refactor electron-audio-loopback first then move this to register for beat-sync handler.
+    // TODO(@nekomeowww): Currently, beat-sync and vision cannot be used together, as they both overriding the display media request handler.
+    session.defaultSession.setDisplayMediaRequestHandler((_request, callback) => {
+      desktopCapturer.getSources({ types: ['screen'] }).then((sources) => {
+        // Grant access to the first screen found.
+        callback({ video: sources[0], audio: 'loopback' })
+      })
+    }, { useSystemPicker: false })
+  })
 }

apps/stage-tamagotchi/src/renderer/components/WithScreenCapture.vue

@@ -21,6 +21,7 @@ const { t } = useI18n()
 const {
   getSources,
   setSource,
+  resetSource,
   selectWithSource,
   checkMacOSPermission,
   requestMacOSPermission,
@@ -62,6 +63,7 @@ watch(focused, async (isFocused) => {
     v-bind="{
       getSources,
       setSource,
+      resetSource,
       selectWithSource,
       hasPermissions,
       checkPermissions,

apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts

@@ -0,0 +1,146 @@
+import type { SerializableDesktopCapturerSource } from '@proj-airi/electron-screen-capture'
+import type { SourcesOptions } from 'electron'
+import type { MaybeRefOrGetter } from 'vue'
+
+import { useElectronScreenCapture } from '@proj-airi/electron-screen-capture/vue'
+import { computed, ref, toRaw, toValue } from 'vue'
+
+interface ScreenCaptureSource extends SerializableDesktopCapturerSource {
+  appIconURL?: string
+  thumbnailURL?: string
+}
+
+function toLocalArrayBuffer(bytes: Uint8Array) {
+  if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) {
+    return bytes.slice().buffer
+  }
+  return bytes.buffer as ArrayBuffer
+}
+
+function toObjectUrl(bytes: Uint8Array, mime: string) {
+  return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime }))
+}
+
+export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesOptions>) {
+  const sources = ref<ScreenCaptureSource[]>([])
+  const isRefetching = ref(false)
+  const hasFetchedOnce = ref(false)
+  const activeSourceId = ref('')
+  const activeStream = ref<MediaStream | null>(null)
+
+  const {
+    getSources,
+    setSource,
+    resetSource,
+  } = useElectronScreenCapture(window.electron.ipcRenderer, sourcesOptions)
+
+  const activeSource = computed(() => sources.value.find(source => source.id === activeSourceId.value) || null)
+
+  async function refetchSources() {
+    try {
+      isRefetching.value = true
+      const nextSources = (await getSources())
+        .sort((a, b) => {
+          const aIsScreen = a.id.startsWith('screen:')
+          const bIsScreen = b.id.startsWith('screen:')
+          if (aIsScreen !== bIsScreen)
+            return aIsScreen ? -1 : 1
+          return a.name.localeCompare(b.name)
+        })
+
+      sources.value.forEach((oldSource) => {
+        if (oldSource.appIconURL)
+          URL.revokeObjectURL(oldSource.appIconURL)
+        if (oldSource.thumbnailURL)
+          URL.revokeObjectURL(oldSource.thumbnailURL)
+      })
+
+      sources.value = nextSources.map(source => ({
+        ...source,
+        appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined,
+        thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
+      }))
+
+      if (!activeSourceId.value && sources.value.length > 0) {
+        activeSourceId.value = sources.value[0]?.id || ''
+      }
+    }
+    finally {
+      isRefetching.value = false
+      hasFetchedOnce.value = true
+    }
+  }
+
+  async function startStream() {
+    if (!activeSourceId.value)
+      throw new Error('No active source selected')
+
+    if (activeStream.value) {
+      activeStream.value.getTracks().forEach(track => track.stop())
+    }
+
+    const handle = await setSource({
+      options: toRaw(toValue(sourcesOptions)),
+      sourceId: activeSourceId.value,
+    })
+
+    try {
+      const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false })
+      activeStream.value = stream
+      return stream
+    }
+    finally {
+      await resetSource(handle)
+    }
+  }
+
+  function stopStream() {
+    if (activeStream.value) {
+      activeStream.value.getTracks().forEach(track => track.stop())
+    }
+    activeStream.value = null
+  }
+
+  function cleanup() {
+    stopStream()
+    sources.value.forEach((oldSource) => {
+      if (oldSource.appIconURL)
+        URL.revokeObjectURL(oldSource.appIconURL)
+      if (oldSource.thumbnailURL)
+        URL.revokeObjectURL(oldSource.thumbnailURL)
+    })
+  }
+
+  function captureFrame(video: HTMLVideoElement, quality = 0.82, maxWidth = 1280, maxHeight = 720) {
+    if (!video || video.readyState < 2)
+      return null
+
+    const canvas = document.createElement('canvas')
+    const sourceWidth = video.videoWidth
+    const sourceHeight = video.videoHeight
+    const scale = Math.min(maxWidth / sourceWidth, maxHeight / sourceHeight, 1)
+    canvas.width = Math.round(sourceWidth * scale)
+    canvas.height = Math.round(sourceHeight * scale)
+
+    const ctx = canvas.getContext('2d')
+    if (!ctx)
+      throw new Error('Failed to create canvas context')
+
+    ctx.drawImage(video, 0, 0, canvas.width, canvas.height)
+    return canvas.toDataURL('image/jpeg', quality)
+  }
+
+  return {
+    sources,
+    activeSourceId,
+    activeSource,
+    activeStream,
+    isRefetching,
+    hasFetchedOnce,
+    refetchSources,
+    startStream,
+    stopStream,
+    cleanup,
+    captureFrame,
+  }
+}

apps/stage-tamagotchi/src/renderer/layouts/settings.vue

@@ -68,6 +68,10 @@ const routeHeaderMetadataMap = computed(() => {
       subtitle: t('settings.title'),
       title: t('settings.pages.modules.hearing.title'),
     },
+    '/settings/modules/vision': {
+      subtitle: t('settings.title'),
+      title: t('settings.pages.modules.vision.title'),
+    },
     '/settings/modules/memory-short-term': {
       subtitle: t('settings.title'),
       title: t('settings.pages.modules.memory-short-term.title'),
@@ -133,6 +137,10 @@ const routeHeaderMetadataMap = computed(() => {
       subtitle: t('tamagotchi.settings.devtools.title'),
       title: t('tamagotchi.settings.devtools.pages.context-flow.title'),
     },
+    '/devtools/vision': {
+      subtitle: t('tamagotchi.settings.devtools.title'),
+      title: 'Vision Capture',
+    },
   }
 
   for (const metadata of allProvidersMetadata.value) {