diff --git a/maps/function-declarations.js b/maps/function-declarations.js index c80b15ca..f09cb757 100644 --- a/maps/function-declarations.js +++ b/maps/function-declarations.js @@ -57,6 +57,7 @@ export function embed(location) { loading="lazy" allowfullscreen referrerpolicy="no-referrer-when-downgrade" +sandbox="allow-scripts allow-popups allow-forms allow-popups-to-escape-sandbox" src="https://www.google.com/maps/embed/v1/place?key=${API_KEY} &q=${location}" > diff --git a/maps/index.html b/maps/index.html index db0d00fa..4085e7c8 100644 --- a/maps/index.html +++ b/maps/index.html @@ -7,6 +7,15 @@
+
+ +
+
+
+ + + +
diff --git a/maps/script.js b/maps/script.js index 5649c7f9..7d0b6bb6 100644 --- a/maps/script.js +++ b/maps/script.js @@ -54,11 +54,20 @@ const chat = async (userText) => { if (call) { functionDeclarations[0].callback(call.args); + } else { + const geminiResponse = response.text(); + messagesDiv.innerHTML += `

Gemini: ${geminiResponse}

`; + messagesDiv.scrollTop = messagesDiv.scrollHeight; + + // Text-to-speech for Gemini's response + const utterance = new SpeechSynthesisUtterance(geminiResponse); + window.speechSynthesis.speak(utterance); } } catch (e) { console.error(e); - } -}; + messagesDiv.innerHTML += `

Gemini: Error: ${e.message}

`; + messagesDiv.scrollTop = messagesDiv.scrollHeight; + }; async function init() { renderPage("%"); // Start by rendering with empty location query: shows earth @@ -70,6 +79,72 @@ async function init() { } else { document.documentElement.setAttribute("data-theme", "light"); } + + const saveLocalButton = document.querySelector("#save-local-button"); + saveLocalButton.addEventListener("click", () => { + const location = document.querySelector("#map iframe").src; + const caption = document.querySelector("#caption p")?.textContent || ""; + const content = `Location: ${location}\nCaption: ${caption}`; + const blob = new Blob([content], { type: "text/plain" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "map-data.txt"; + a.click(); + URL.revokeObjectURL(url); + }); + + const chatInput = document.querySelector("#chat-input"); + const sendButton = document.querySelector("#send-button"); + const messagesDiv = document.querySelector("#messages"); + const voiceInputButton = document.querySelector("#voice-input-button"); + + const sendMessage = async () => { + const userText = chatInput.value; + if (userText.trim() === "") return; + + messagesDiv.innerHTML += `

You: ${userText}

`; + chatInput.value = ""; + await chat(userText); + messagesDiv.scrollTop = messagesDiv.scrollHeight; + }; + + sendButton.addEventListener("click", sendMessage); + chatInput.addEventListener("keypress", (e) => { + if (e.key === "Enter") { + sendMessage(); + } + }); + + const handleVoiceInput = () => { + const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + if (!SpeechRecognition) { + alert("Speech recognition not supported in this browser."); + return; + } + + const recognition = new SpeechRecognition(); + recognition.interimResults = false; + recognition.lang = 'en-US'; + + recognition.addEventListener('result', (e) => { + const transcript = Array.from(e.results) + .map(result => result[0]) + .map(result => result.transcript) + .join(''); + chatInput.value = transcript; + sendMessage(); + }); + + recognition.addEventListener('error', (e) => { + console.error('Speech recognition error:', e.error); + alert('Speech recognition error: ' + e.error); + }); + + recognition.start(); + }; + + voiceInputButton.addEventListener("click", handleVoiceInput); } init(); diff --git a/spatial/src/TopBar.tsx b/spatial/src/TopBar.tsx index b28d9379..77086264 100644 --- a/spatial/src/TopBar.tsx +++ b/spatial/src/TopBar.tsx @@ -13,7 +13,7 @@ // limitations under the License. import { useAtom } from "jotai"; -import { useResetState } from "./hooks"; +import { useResetState, useSaveState } from "./hooks"; import { DetectTypeAtom, HoverEnteredAtom, @@ -25,6 +25,7 @@ import { modelOptions } from "./consts"; export function TopBar() { const resetState = useResetState(); + const saveState = useSaveState(); const [revealOnHover, setRevealOnHoverMode] = useAtom(RevealOnHoverModeAtom); const [detectType] = useAtom(DetectTypeAtom); const [, setHoverEntered] = useAtom(HoverEnteredAtom); @@ -45,6 +46,17 @@ export function TopBar() { >
Reset session
+
{detectType === "2D bounding boxes" ? ( diff --git a/spatial/src/hooks.tsx b/spatial/src/hooks.tsx index c2f44828..8b075f55 100644 --- a/spatial/src/hooks.tsx +++ b/spatial/src/hooks.tsx @@ -19,6 +19,9 @@ import { BumpSessionAtom, ImageSentAtom, PointsAtom, + ImageSrcAtom, + LinesAtom, + DetectTypeAtom, } from "./atoms"; export function useResetState() { @@ -36,3 +39,32 @@ export function useResetState() { setPoints([]); }; } + +export function useSaveState() { + const [imageSrc] = useAtom(ImageSrcAtom); + const [boundingBoxes2D] = useAtom(BoundingBoxes2DAtom); + const [boundingBoxes3D] = useAtom(BoundingBoxes3DAtom); + const [points] = useAtom(PointsAtom); + const [lines] = useAtom(LinesAtom); + const [detectType] = useAtom(DetectTypeAtom); + + return () => { + const state = { + imageSrc, + boundingBoxes2D, + boundingBoxes3D, + points, + lines, + detectType, + }; + + const content = JSON.stringify(state, null, 2); + const blob = new Blob([content], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "spatial-data.json"; + a.click(); + URL.revokeObjectURL(url); + }; +} diff --git a/spatial/src/styles.css b/spatial/src/styles.css index 3d1b3fbb..303a990d 100644 --- a/spatial/src/styles.css +++ b/spatial/src/styles.css @@ -110,8 +110,8 @@ input[type="range"] { } .box-caption { - color: var(--bg-color); - background: var(--accent-color); + color: var(--text-color-primary); + background: var(--box-color); border-radius: var(--box-radius); padding: 14px 28px; max-width: 340px; diff --git a/video/server/index.mjs b/video/server/index.mjs index 9ed24079..e9b0c728 100644 --- a/video/server/index.mjs +++ b/video/server/index.mjs @@ -17,10 +17,12 @@ import ViteExpress from 'vite-express' import multer from 'multer' import {checkProgress, promptVideo, uploadVideo} from './upload.mjs' +import os from 'os'; + const app = express() app.use(express.json()) -const upload = multer({dest: '/tmp/'}) +const upload = multer({dest: os.tmpdir()}) app.post('/api/upload', upload.single('video'), async (req, res) => { try { const file = req.file diff --git a/video/src/App.jsx b/video/src/App.jsx index f904cddd..c16761be 100644 --- a/video/src/App.jsx +++ b/video/src/App.jsx @@ -47,6 +47,28 @@ export default function App() { const isCustomChartMode = isChartMode && chartMode === 'Custom' const hasSubMode = isCustomMode || isChartMode + const saveState = () => { + const state = { + vidUrl, + timecodeList, + selectedMode, + activeMode, + customPrompt, + chartMode, + chartPrompt, + chartLabel, + }; + + const content = JSON.stringify(state, null, 2); + const blob = new Blob([content], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "video-data.json"; + a.click(); + URL.revokeObjectURL(url); + }; + const setTimecodes = ({timecodes}) => setTimecodeList( timecodes.map(t => ({...t, text: t.text.replaceAll("\\'", "'")})) @@ -238,6 +260,12 @@ export default function App() { > ▶️ Generate +
)} diff --git a/video/src/VideoPlayer.jsx b/video/src/VideoPlayer.jsx index b09bd64b..fecf63ea 100644 --- a/video/src/VideoPlayer.jsx +++ b/video/src/VideoPlayer.jsx @@ -16,10 +16,12 @@ import {useCallback, useEffect, useState, useMemo} from 'react' import c from 'classnames' import {timeToSecs} from './utils' -const formatTime = t => - `${Math.floor(t / 60)}:${Math.floor(t % 60) - .toString() - .padStart(2, '0')}` +const formatTime = t => { + const minutes = Math.floor(t / 60); + const seconds = Math.floor(t % 60).toString().padStart(2, '0'); + const milliseconds = Math.floor((t - Math.floor(t)) * 1000).toString().padStart(3, '0'); + return `${minutes}:${seconds}.${milliseconds}`; +} export default function VideoPlayer({ url,