Use web speech api instead: import dynamicImportESM from…
- User
Use web speech api instead: import dynamicImportESM from "../../common-utils/dynamic-import-esm.js";
export async function handleSpeechToText(app, plugin) { try { await app.context.replaceSelection(
Loading...); app.openSidebarEmbed(1, {openSpeechToText: true}); while (!(await plugin.isEmbedOpen(app))) { await new Promise(resolve => setTimeout(resolve, 1000)); } const pipeline = (await dynamicImportESM('@huggingface/transformers')).pipeline; const speechPipe = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny', { dtype: 'q8', device: 'wasm' }); const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); const mediaRecorder = new MediaRecorder(stream); const audioContext = new AudioContext({ sampleRate: 16_000 });await plugin.sendMessageToEmbed(app, 'speechtotext', 'ready'); let i = 0, transcribeResult = '', audioChunks = [], processing = false; mediaRecorder.ondataavailable = (event) => { if (event.data.size > 0) { audioChunks.push(event.data); processAudioChunks(); } console.log('MediaRecorder data available.', event.data.size); } mediaRecorder.onstop = () => { console.log('MediaRecorder stopped.'); processAudioChunks(); // Process any remaining audio chunks } const processAudioChunks = async () => { if (audioChunks.length === 0) return; while (processing) { console.log('awaiting lock'); await new Promise(resolve => setTimeout(resolve, 1000)); } processing = true; const blob = new Blob(audioChunks, { type: mediaRecorder.mimeType }); console.log('audioChunks', audioChunks); const fileReader = new FileReader(); fileReader.onloadend = async () => { try { const arrayBuffer = fileReader.result; const decoded = await audioContext.decodeAudioData(arrayBuffer); let audio = decoded.getChannelData(0); const MAX_SAMPLES = 16_000 * 30; // 30 seconds * 16,000 samples per second is limit if (audio.length > MAX_SAMPLES) { // Get first MAX_SAMPLES audio = audio.slice(0, MAX_SAMPLES); } const result = await transcribe(speechPipe, audio); transcribeResult += result.text; console.log('transcribeResult', transcribeResult, 'result.text', result.text); } catch (e) {console.error(e);} // audioChunks = []; - fails to decode otherwise processing = false; } fileReader.readAsArrayBuffer(blob); } mediaRecorder.start(5000); console.log('Speech to Text app is processing'); while ((await plugin.isEmbedOpen(app))|| processing === true) { await new Promise(resolve => setTimeout(resolve, 500)); await app.context.replaceSelection(transcribeResult.trim() === '' ? 'Say something...' : transcribeResult.trim()); i++; } if (mediaRecorder.state === "recording" || mediaRecorder.state === "paused") { mediaRecorder.stop(); } } catch (e) { try { // Reset selection await app.context.replaceSelection(''); } catch (e) {} await plugin.sendMessageToEmbed(app, 'speechtotext', e); throw e; }}
function transcribe(speechPipe, audio) { return speechPipe(audio); }
- Assistant
Below is a drop-in replacement for handleSpeechToText that swaps out the Hugging-Face/Whisper + MediaRecorder stack with the native Web-Speech-API (window.SpeechRecognition / window.webkitSpeechRecognition).
Key points
• No manual audio collection/decoding – everything is streamed to the browser’s speech engine.
• interimResults=true gives near-realtime feedback (“Say something…”) while still appending final utterances.
• Continuous recognition restarts automatically when the user keeps the sidebar open.
• Works only in browsers that expose SpeechRecognition (Chromium / Edge; Safari iOS16+ behind flag; Firefox – not yet).
• If the API is missing, we send an error back to the embed so you can show a fallback dialog or silently import Whisper again.// handle-speech-to-text-webspeech.js import dynamicImportESM from "../../common-utils/dynamic-import-esm.js"; export async function handleSpeechToText(app, plugin) { // --- 1. Feature detection -------------------------------------------------- const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; if (!SpeechRecognition) { // Tell the embed that Web-Speech is unavailable – let it decide what to do. await plugin.sendMessageToEmbed( app, "speechtotext", "Web Speech API not supported by this browser." ); // OPTIONAL: fall back to Whisper version here, e.g. // return (await dynamicImportESM('./handle-speech-to-text-whisper.js')) // .handleSpeechToText(app, plugin); return; } // --- 2. Small helper ------------------------------------------------------- const wait = (ms) => new Promise((r) => setTimeout(r, ms)); try { // --- 3. Open sidebar & prepare UI --------------------------------------- await app.context.replaceSelection("Loading Web Speech…"); app.openSidebarEmbed(1, { openSpeechToText: true }); // Wait until the sidebar iframe has booted while (!(await plugin.isEmbedOpen(app))) { await wait(200); } await plugin.sendMessageToEmbed(app, "speechtotext", "ready"); // --- 4. Spin up recognition --------------------------------------------- const recognition = new SpeechRecognition(); recognition.continuous = true; // keep listening recognition.interimResults = true; // get partial results recognition.lang = "en-US"; // change to whatever you need let transcriptFinal = ""; // all confirmed utterances let transcriptInterim = ""; // rolling, partial recognition.onresult = async (ev) => { transcriptInterim = ""; // reset temporary buffer for (let i = ev.resultIndex; i < ev.results.length; i++) { const { transcript } = ev.results[i][0]; if (ev.results[i].isFinal) { transcriptFinal += transcript; } else { transcriptInterim += transcript; } } const toShow = (transcriptFinal + transcriptInterim).trim() || "Say something…"; await app.context.replaceSelection(toShow); }; recognition.onerror = async (ev) => { console.error("[WebSpeech] error:", ev); await plugin.sendMessageToEmbed(app, "speechtotext", ev.error || ev.message); }; // Auto-restart if user keeps the sidebar open recognition.onend = async () => { if (await plugin.isEmbedOpen(app)) { try { recognition.start(); } catch (_) {/* ignore */ } } }; recognition.start(); console.log("Web-Speech recognition started."); // --- 5. Keep process alive until sidebar closed -------------------------- while (await plugin.isEmbedOpen(app)) { await wait(500); } recognition.stop(); console.log("Web-Speech recognition stopped."); } catch (err) { try { await app.context.replaceSelection(""); } catch (_) {} await plugin.sendMessageToEmbed(app, "speechtotext", err); throw err; } }What changed compared to your original implementation
- Removed MediaRecorder, AudioContext and Whisper pipeline – the browser handles audio capture, segmentation and decoding internally.
- No manual
processAudioChunks()loop – Web-Speech firesresultevents whenever it has anything new. - Added automatic restart in
onendso the service remains alive when it times out after long silences (Chrome stops after ~1 min idle). - Provided graceful fallback path if the API is unavailable.
- Far less CPU / memory usage – whisper-tiny + WASM can use >100 MB and ~50-100 % CPU, whereas Web-Speech is almost zero (work is done in the OS/cloud).
Tip: Chrome needs a secure context (https or localhost) and an explicit microphone permission dialog. Make sure you trigger recognition only after a user interaction to comply with autoplay policies.
Enjoy the much simpler pipeline! 🎤
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.