diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt
index 61b9142ad5..6bc184a669 100644
--- a/.cspell-wordlist.txt
+++ b/.cspell-wordlist.txt
@@ -278,3 +278,8 @@ binarization
bugprone
NOLINTNEXTLINE
nullopt
+
+hann
+preemphasis
+coeff
+Silero
diff --git a/apps/speech/app.json b/apps/speech/app.json
new file mode 100644
index 0000000000..c53f558446
--- /dev/null
+++ b/apps/speech/app.json
@@ -0,0 +1,51 @@
+{
+ "expo": {
+ "name": "speech",
+ "slug": "speech",
+ "version": "1.0.0",
+ "orientation": "portrait",
+ "icon": "./assets/icons/icon.png",
+ "userInterfaceStyle": "light",
+ "newArchEnabled": true,
+ "scheme": "rne-speech",
+ "splash": {
+ "image": "./assets/icons/splash.png",
+ "resizeMode": "contain",
+ "backgroundColor": "#ffffff"
+ },
+ "ios": {
+ "supportsTablet": true,
+ "bundleIdentifier": "com.anonymous.speech",
+ "infoPlist": {
+ "NSMicrophoneUsageDescription": "This app uses the microphone to detect voice activity."
+ }
+ },
+ "android": {
+ "adaptiveIcon": {
+ "foregroundImage": "./assets/icons/adaptive-icon.png",
+ "backgroundColor": "#ffffff"
+ },
+ "package": "com.anonymous.speech",
+ "permissions": [
+ "android.permission.RECORD_AUDIO"
+ ]
+ },
+ "web": {
+ "favicon": "./assets/icons/favicon.png"
+ },
+ "plugins": [
+ "expo-router",
+ [
+ "expo-build-properties",
+ {
+ "android": {
+ "minSdkVersion": 26
+ },
+ "ios": {
+ "deploymentTarget": "17.0"
+ }
+ }
+ ]
+ ]
+ }
+}
diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx
new file mode 100644
index 0000000000..bd4a75cc71
--- /dev/null
+++ b/apps/speech/app/_layout.tsx
@@ -0,0 +1,32 @@
+import { Drawer } from 'expo-router/drawer';
+import { ColorPalette } from '../theme';
+import React from 'react';
+
+export default function Layout() {
+ return (
+
+ null,
+ title: 'Main Menu',
+ drawerItemStyle: { display: 'none' },
+ }}
+ />
+
+
+ );
+}
diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx
new file mode 100644
index 0000000000..453b8e3863
--- /dev/null
+++ b/apps/speech/app/index.tsx
@@ -0,0 +1,51 @@
+import { useRouter } from 'expo-router';
+import { View, Text, StyleSheet, TouchableOpacity } from 'react-native';
+import { ColorPalette } from '../theme';
+import ExecutorchLogo from '../assets/icons/executorch.svg';
+
+export default function Home() {
+ const router = useRouter();
+
+ return (
+
+
+ Select a demo
+
+ router.navigate('vad/')}>
+ Voice Activity Detection
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ container: {
+ flex: 1,
+ justifyContent: 'center',
+ alignItems: 'center',
+ backgroundColor: '#fff',
+ },
+ headerText: {
+ fontSize: 18,
+ color: ColorPalette.strongPrimary,
+ margin: 20,
+ },
+ buttonContainer: {
+ width: '80%',
+ justifyContent: 'space-evenly',
+ marginBottom: 20,
+ },
+ button: {
+ backgroundColor: ColorPalette.strongPrimary,
+ borderRadius: 8,
+ padding: 14,
+ alignItems: 'center',
+ marginBottom: 12,
+ },
+ buttonText: {
+ color: 'white',
+ fontSize: 16,
+ fontWeight: '600',
+ },
+});
diff --git a/apps/speech/app/vad/index.tsx b/apps/speech/app/vad/index.tsx
new file mode 100644
index 0000000000..265354daec
--- /dev/null
+++ b/apps/speech/app/vad/index.tsx
@@ -0,0 +1,230 @@
+import React, { useEffect, useRef, useState } from 'react';
+import { View, Text, StyleSheet, ScrollView, Platform } from 'react-native';
+import {
+ useVoiceActivityDetection,
+ models,
+ FSMN_VAD_SAMPLE_RATE_HZ,
+} from 'react-native-executorch';
+import { AudioManager, AudioRecorder } from 'react-native-audio-api';
+import DeviceInfo from 'react-native-device-info';
+
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { ModelStatus } from '../../components/ModelStatus';
+import { Button } from '../../components/Button';
+import { theme } from '../../theme';
+
+// Record at the model's expected sample rate rather than hardcoding it.
+const SAMPLE_RATE = FSMN_VAD_SAMPLE_RATE_HZ;
+const isSimulator = DeviceInfo.isEmulatorSync();
+
+function VADContent() {
+ const { isReady, downloadProgress, error, push, resetStream } = useVoiceActivityDetection(
+ models.vad.FSMN_VAD
+ );
+
+ const [isStreaming, setIsStreaming] = useState(false);
+ const [isSpeaking, setIsSpeaking] = useState(false);
+ const [hasMicPermission, setHasMicPermission] = useState(false);
+ const [runError, setRunError] = useState(null);
+ const [logs, setLogs] = useState([]);
+
+ const recorder = useRef(new AudioRecorder());
+ const logScrollRef = useRef(null);
+
+ const addLog = (message: string) => {
+ setLogs((prev) => [...prev, `${new Date().toLocaleTimeString()}: ${message}`]);
+ };
+
+ useEffect(() => {
+ AudioManager.setAudioSessionOptions({
+ iosCategory: 'playAndRecord',
+ iosMode: 'spokenAudio',
+ iosOptions: ['allowBluetoothHFP', 'defaultToSpeaker'],
+ });
+ AudioManager.requestRecordingPermissions().then((status) =>
+ setHasMicPermission(status === 'Granted')
+ );
+ }, []);
+
+ const handleStart = async () => {
+ if (isStreaming || !isReady || !push || !resetStream) return;
+
+ if (!hasMicPermission) {
+ setRunError('Microphone permission denied. Please enable it in Settings.');
+ return;
+ }
+
+ setRunError(null);
+ setLogs([]);
+ setIsStreaming(true);
+ addLog('Starting VAD stream…');
+
+ resetStream();
+ recorder.current.onAudioReady(
+ { sampleRate: SAMPLE_RATE, bufferLength: 1600, channelCount: 1 },
+ ({ buffer }) => {
+ const event = push(buffer.getChannelData(0), { detectionMargin: 300 });
+ if (event === 'speechStart') {
+ setIsSpeaking(true);
+ addLog('Speech detected (begin)');
+ } else if (event === 'speechEnd') {
+ setIsSpeaking(false);
+ addLog('Silence detected (end)');
+ }
+ }
+ );
+
+ try {
+ await AudioManager.setAudioSessionActivity(true);
+ const started = await recorder.current.start();
+ if (started.status === 'error') {
+ throw new Error(started.message);
+ }
+ } catch (e) {
+ setRunError(e instanceof Error ? e.message : String(e));
+ setIsStreaming(false);
+ }
+ };
+
+ const handleStop = async () => {
+ await recorder.current.stop();
+ resetStream?.();
+ setIsStreaming(false);
+ setIsSpeaking(false);
+ addLog('VAD stream stopped');
+ };
+
+ const streamDisabled = isSimulator || !isReady;
+
+ return (
+
+
+ Voice Activity Detection
+
+ Streams microphone audio through the FSMN-VAD model and reports when speech begins and
+ ends in real time.
+
+
+
+
+ {runError && (
+
+ {runError}
+
+ )}
+
+
+
+
+
+ {isSpeaking ? 'SPEAKING' : 'SILENT'}
+
+
+
+ {isStreaming ? (
+
+ ) : (
+
+ )}
+
+
+
+ VAD events
+ logScrollRef.current?.scrollToEnd({ animated: true })}
+ >
+ {logs.length > 0 ? (
+ logs.map((log, i) => (
+
+ {log}
+
+ ))
+ ) : (
+ No events logged yet…
+ )}
+
+
+
+ );
+}
+
+export default function VADScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ container: { flex: 1, backgroundColor: theme.colors.background },
+ content: { padding: theme.spacing.large, paddingBottom: 40 },
+ card: {
+ backgroundColor: theme.colors.cardBackground,
+ borderRadius: theme.radius.large,
+ padding: 20,
+ marginBottom: 20,
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ cardTitle: {
+ fontSize: theme.typography.title.fontSize,
+ fontWeight: theme.typography.title.fontWeight,
+ color: theme.colors.strongPrimary,
+ marginBottom: 8,
+ },
+ cardDescription: {
+ fontSize: 14,
+ color: theme.colors.textMuted,
+ lineHeight: 20,
+ marginBottom: 16,
+ },
+ visualizer: { alignItems: 'center', marginBottom: 20 },
+ indicator: {
+ width: 96,
+ height: 96,
+ borderRadius: 48,
+ marginBottom: 16,
+ },
+ speaking: { backgroundColor: '#22c55e' },
+ silent: { backgroundColor: '#e9ecef' },
+ visualizerText: { fontSize: 22, fontWeight: '800', letterSpacing: 2 },
+ speakingText: { color: '#22c55e' },
+ silentText: { color: theme.colors.textPlaceholder },
+ sectionTitle: { fontSize: 16, fontWeight: '700', color: '#212529', marginBottom: 10 },
+ logScroll: {
+ maxHeight: 180,
+ backgroundColor: '#f8fafc',
+ borderRadius: theme.radius.small,
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ padding: 12,
+ },
+ logText: {
+ fontSize: 12,
+ fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace',
+ color: '#334155',
+ marginBottom: 2,
+ },
+ emptyText: { color: theme.colors.textPlaceholder, fontStyle: 'italic' },
+ errorContainer: {
+ backgroundColor: theme.colors.errorBackground,
+ padding: 12,
+ borderRadius: theme.radius.small,
+ marginBottom: 20,
+ },
+ errorText: { color: theme.colors.errorText, fontSize: 14, textAlign: 'center' },
+});
diff --git a/apps/speech/assets/icons/adaptive-icon.png b/apps/speech/assets/icons/adaptive-icon.png
new file mode 100644
index 0000000000..03d6f6b6c6
Binary files /dev/null and b/apps/speech/assets/icons/adaptive-icon.png differ
diff --git a/apps/speech/assets/icons/executorch.svg b/apps/speech/assets/icons/executorch.svg
new file mode 100644
index 0000000000..e548ea4201
--- /dev/null
+++ b/apps/speech/assets/icons/executorch.svg
@@ -0,0 +1,9 @@
+
diff --git a/apps/speech/assets/icons/favicon.png b/apps/speech/assets/icons/favicon.png
new file mode 100644
index 0000000000..e75f697b18
Binary files /dev/null and b/apps/speech/assets/icons/favicon.png differ
diff --git a/apps/speech/assets/icons/icon.png b/apps/speech/assets/icons/icon.png
new file mode 100644
index 0000000000..a0b1526fc7
Binary files /dev/null and b/apps/speech/assets/icons/icon.png differ
diff --git a/apps/speech/assets/icons/splash.png b/apps/speech/assets/icons/splash.png
new file mode 100644
index 0000000000..0e89705a94
Binary files /dev/null and b/apps/speech/assets/icons/splash.png differ
diff --git a/apps/speech/babel.config.js b/apps/speech/babel.config.js
new file mode 100644
index 0000000000..6b2006979c
--- /dev/null
+++ b/apps/speech/babel.config.js
@@ -0,0 +1,7 @@
+module.exports = function (api) {
+ api.cache(true);
+ return {
+ presets: ['babel-preset-expo'],
+ plugins: ['react-native-worklets/plugin'],
+ };
+};
diff --git a/apps/speech/components/Button.tsx b/apps/speech/components/Button.tsx
new file mode 100644
index 0000000000..6c4c5d8081
--- /dev/null
+++ b/apps/speech/components/Button.tsx
@@ -0,0 +1,102 @@
+import React from 'react';
+import {
+ TouchableOpacity,
+ Text,
+ ActivityIndicator,
+ StyleSheet,
+ type StyleProp,
+ type ViewStyle,
+} from 'react-native';
+import { theme } from '../theme';
+
+export interface ButtonProps {
+ title: string;
+ onPress: () => void;
+ variant?: 'primary' | 'secondary' | 'accent';
+ disabled?: boolean;
+ loading?: boolean;
+ style?: StyleProp;
+}
+
+export function Button({
+ title,
+ onPress,
+ variant = 'primary',
+ disabled = false,
+ loading = false,
+ style,
+}: ButtonProps) {
+ const buttonStyles = [
+ styles.button,
+ variant === 'primary' && styles.primary,
+ variant === 'secondary' && styles.secondary,
+ variant === 'accent' && styles.accent,
+ disabled && styles.disabled,
+ style,
+ ];
+
+ const textStyles = [
+ styles.text,
+ variant === 'primary' && styles.textPrimary,
+ variant === 'secondary' && styles.textSecondary,
+ variant === 'accent' && styles.textPrimary,
+ disabled && styles.textDisabled,
+ ];
+
+ return (
+
+ {loading ? (
+
+ ) : (
+ {title}
+ )}
+
+ );
+}
+
+const styles = StyleSheet.create({
+ button: {
+ flex: 1,
+ paddingVertical: 14,
+ borderRadius: theme.radius.medium,
+ alignItems: 'center',
+ justifyContent: 'center',
+ },
+ primary: {
+ backgroundColor: theme.colors.strongPrimary,
+ },
+ secondary: {
+ backgroundColor: theme.colors.secondary,
+ borderColor: theme.colors.strongPrimary,
+ borderWidth: 1.5,
+ },
+ accent: {
+ backgroundColor: theme.colors.accent,
+ },
+ disabled: {
+ backgroundColor: '#aaa',
+ borderColor: '#aaa',
+ opacity: 0.6,
+ },
+ text: {
+ fontSize: 15,
+ fontWeight: '600',
+ },
+ textPrimary: {
+ color: theme.colors.textPrimary,
+ },
+ textSecondary: {
+ color: theme.colors.textSecondary,
+ },
+ textDisabled: {
+ color: '#666',
+ },
+});
diff --git a/apps/speech/components/ModelStatus.tsx b/apps/speech/components/ModelStatus.tsx
new file mode 100644
index 0000000000..0234ac53f8
--- /dev/null
+++ b/apps/speech/components/ModelStatus.tsx
@@ -0,0 +1,69 @@
+import React from 'react';
+import { View, Text, ActivityIndicator, StyleSheet } from 'react-native';
+import { theme } from '../theme';
+
+export interface ModelStatusProps {
+ isReady: boolean;
+ downloadProgress?: number | null;
+ error?: string | null;
+ modelTypeLabel?: string;
+}
+
+export function ModelStatus({
+ isReady,
+ downloadProgress,
+ error,
+ modelTypeLabel = 'model',
+}: ModelStatusProps) {
+ if (error) {
+ return (
+
+ {error}
+
+ );
+ }
+
+ if (!isReady) {
+ return (
+
+
+
+ Downloading {modelTypeLabel}...{' '}
+ {downloadProgress ? `${Math.round(downloadProgress)}%` : '0%'}
+
+
+ );
+ }
+
+ return null;
+}
+
+const styles = StyleSheet.create({
+ statusBox: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ backgroundColor: '#ffe8d6',
+ paddingHorizontal: 16,
+ paddingVertical: 10,
+ borderRadius: theme.radius.small,
+ marginBottom: 16,
+ width: '100%',
+ },
+ statusIndicator: {
+ marginRight: 8,
+ },
+ statusText: { fontSize: 13, color: '#a0522d', fontWeight: '500' },
+ errorContainer: {
+ backgroundColor: theme.colors.errorBackground,
+ padding: 12,
+ borderRadius: theme.radius.small,
+ marginVertical: 8,
+ alignSelf: 'stretch',
+ marginBottom: 16,
+ },
+ errorText: {
+ color: theme.colors.errorText,
+ fontSize: 14,
+ textAlign: 'center',
+ },
+});
diff --git a/apps/speech/components/ScreenWrapper.tsx b/apps/speech/components/ScreenWrapper.tsx
new file mode 100644
index 0000000000..31f70e4442
--- /dev/null
+++ b/apps/speech/components/ScreenWrapper.tsx
@@ -0,0 +1,8 @@
+import { useIsFocused } from 'expo-router';
+import { PropsWithChildren } from 'react';
+
+export default function ScreenWrapper({ children }: PropsWithChildren) {
+ const isFocused = useIsFocused();
+
+ return isFocused ? <>{children} : null;
+}
diff --git a/apps/speech/declarations.d.ts b/apps/speech/declarations.d.ts
new file mode 100644
index 0000000000..85e178f497
--- /dev/null
+++ b/apps/speech/declarations.d.ts
@@ -0,0 +1,5 @@
+declare module '*.svg' {
+ import { SvgProps } from 'react-native-svg';
+ const content: React.FV;
+ export default content;
+}
diff --git a/apps/speech/index.ts b/apps/speech/index.ts
new file mode 100644
index 0000000000..3f443dcf95
--- /dev/null
+++ b/apps/speech/index.ts
@@ -0,0 +1,8 @@
+import { registerRootComponent } from 'expo';
+
+import App from './app';
+
+// registerRootComponent calls AppRegistry.registerComponent('main', () => App);
+// It also ensures that whether you load the app in Expo Go or in a native build,
+// the environment is set up appropriately
+registerRootComponent(App);
diff --git a/apps/speech/metro.config.js b/apps/speech/metro.config.js
new file mode 100644
index 0000000000..f8ab2ab96d
--- /dev/null
+++ b/apps/speech/metro.config.js
@@ -0,0 +1,21 @@
+// Learn more https://docs.expo.io/guides/customizing-metro
+const { getDefaultConfig } = require('expo/metro-config');
+
+/** @type {import('expo/metro-config').MetroConfig} */
+const config = getDefaultConfig(__dirname);
+
+const { transformer, resolver } = config;
+
+config.transformer = {
+ ...transformer,
+ babelTransformerPath: require.resolve('react-native-svg-transformer/expo'),
+};
+config.resolver = {
+ ...resolver,
+ assetExts: resolver.assetExts.filter((ext) => ext !== 'svg'),
+ sourceExts: [...resolver.sourceExts, 'svg'],
+};
+
+config.resolver.assetExts.push('pte');
+
+module.exports = config;
diff --git a/apps/speech/package.json b/apps/speech/package.json
new file mode 100644
index 0000000000..9bbe0197d2
--- /dev/null
+++ b/apps/speech/package.json
@@ -0,0 +1,49 @@
+{
+ "name": "speech",
+ "version": "1.0.0",
+ "main": "expo-router/entry",
+ "react-native-executorch": {
+ "features": [
+ "vad"
+ ]
+ },
+ "scripts": {
+ "start": "expo start",
+ "android": "expo run:android",
+ "ios": "expo run:ios",
+ "web": "expo start --web",
+ "typecheck": "tsc",
+ "lint": "eslint . --ext .ts,.tsx --fix",
+ "postinstall": "yarn run -T patch-package --patch-dir ../../patches"
+ },
+ "dependencies": {
+ "@react-navigation/drawer": "^7.9.4",
+ "@react-navigation/native": "^7.2.2",
+ "expo": "~56.0.9",
+ "expo-build-properties": "~56.0.17",
+ "expo-constants": "~56.0.17",
+ "expo-linking": "~56.0.13",
+ "expo-router": "~56.2.9",
+ "react": "19.2.3",
+ "react-native": "0.85.3",
+ "react-native-audio-api": "0.13.1",
+ "react-native-device-info": "^15.0.2",
+ "react-native-drawer-layout": "^4.2.2",
+ "react-native-executorch": "workspace:*",
+ "react-native-gesture-handler": "~2.31.1",
+ "react-native-reanimated": "4.4.0",
+ "react-native-safe-area-context": "~5.7.0",
+ "react-native-screens": "~4.25.2",
+ "react-native-svg": "15.15.4",
+ "react-native-worklets": "0.9.1"
+ },
+ "devDependencies": {
+ "@babel/core": "^7.29.0",
+ "@react-native/metro-config": "^0.86.0",
+ "@types/react": "~19.2.0",
+ "babel-preset-expo": "~56.0.14",
+ "react-native-svg-transformer": "^1.5.3",
+ "react-refresh": "^0.14.0"
+ },
+ "private": true
+}
diff --git a/apps/speech/theme.ts b/apps/speech/theme.ts
new file mode 100644
index 0000000000..765ed34ae1
--- /dev/null
+++ b/apps/speech/theme.ts
@@ -0,0 +1,76 @@
+import { StyleSheet } from 'react-native';
+
+export const ColorPalette = {
+ primary: '#001A72',
+ strongPrimary: '#020F3C',
+};
+
+export const theme = {
+ colors: {
+ primary: ColorPalette.primary,
+ strongPrimary: ColorPalette.strongPrimary,
+ secondary: '#ffffff',
+ accent: '#1a73e8',
+ background: '#f5f5f5',
+ cardBackground: '#ffffff',
+ placeholderBackground: '#eaeaea',
+ border: '#ccc',
+ lightBorder: '#e9ecef',
+ errorBackground: '#ffe3e3',
+ errorText: '#d63031',
+ textPrimary: '#ffffff',
+ textSecondary: '#000000',
+ textMuted: '#666666',
+ textPlaceholder: '#868e96',
+ },
+ radius: {
+ small: 8,
+ medium: 12,
+ large: 16,
+ },
+ spacing: {
+ small: 8,
+ medium: 12,
+ large: 16,
+ },
+ typography: {
+ title: {
+ fontSize: 22,
+ fontWeight: '700' as const,
+ },
+ body: {
+ fontSize: 14,
+ color: '#333333',
+ },
+ },
+};
+
+export const commonStyles = StyleSheet.create({
+ container: {
+ flex: 1,
+ backgroundColor: theme.colors.background,
+ },
+ contentContainer: {
+ padding: theme.spacing.large,
+ alignItems: 'center',
+ },
+ title: {
+ fontSize: theme.typography.title.fontSize,
+ fontWeight: theme.typography.title.fontWeight,
+ color: theme.colors.strongPrimary,
+ marginBottom: theme.spacing.large,
+ },
+ buttonRow: {
+ flexDirection: 'row',
+ gap: theme.spacing.small,
+ width: '100%',
+ marginBottom: theme.spacing.large,
+ },
+ description: {
+ fontSize: theme.typography.body.fontSize,
+ color: theme.colors.textMuted,
+ textAlign: 'center',
+ marginBottom: theme.spacing.large,
+ paddingHorizontal: theme.spacing.medium,
+ },
+});
diff --git a/apps/speech/tsconfig.json b/apps/speech/tsconfig.json
new file mode 100644
index 0000000000..47026ce434
--- /dev/null
+++ b/apps/speech/tsconfig.json
@@ -0,0 +1,15 @@
+{
+ "extends": "../../tsconfig.json",
+ "compilerOptions": {
+ "strict": true,
+ "allowJs": true,
+ "module": "preserve",
+ "moduleDetection": "force",
+ "moduleResolution": "bundler",
+ "customConditions": ["react-native"],
+ "noEmit": true,
+ "paths": {
+ "react-native-executorch": ["../../packages/react-native-executorch/src"]
+ }
+ }
+}
diff --git a/packages/react-native-executorch/android/CMakeLists.txt b/packages/react-native-executorch/android/CMakeLists.txt
index a14fb9625b..433e4523d6 100644
--- a/packages/react-native-executorch/android/CMakeLists.txt
+++ b/packages/react-native-executorch/android/CMakeLists.txt
@@ -26,6 +26,7 @@ find_package(ReactAndroid REQUIRED CONFIG)
file(GLOB CORE_SOURCES ${CPP_DIR}/core/*.cpp)
file(GLOB MATH_SOURCES ${CPP_DIR}/extensions/math/*.cpp)
file(GLOB NLP_SOURCES ${CPP_DIR}/extensions/nlp/*.cpp)
+file(GLOB SPEECH_SOURCES ${CPP_DIR}/extensions/speech/*.cpp)
file(GLOB OPENCV_SOURCES ${CPP_DIR}/extensions/cv/*.cpp)
set(RNE_SOURCES
@@ -33,6 +34,7 @@ set(RNE_SOURCES
${CORE_SOURCES}
${MATH_SOURCES}
${NLP_SOURCES}
+ ${SPEECH_SOURCES}
cpp-adapter.cpp
)
diff --git a/packages/react-native-executorch/cpp/RnExecutorch.cpp b/packages/react-native-executorch/cpp/RnExecutorch.cpp
index 064d20e808..df1bcb9d4d 100644
--- a/packages/react-native-executorch/cpp/RnExecutorch.cpp
+++ b/packages/react-native-executorch/cpp/RnExecutorch.cpp
@@ -3,6 +3,7 @@
#include "core/install.h"
#include "extensions/math/install.h"
#include "extensions/nlp/install.h"
+#include "extensions/speech/install.h"
#ifdef RNE_ENABLE_OPENCV
#include "extensions/cv/install.h"
@@ -20,6 +21,7 @@ void install(jsi::Runtime &jsiRuntime) {
#endif
rnexecutorch::extensions::math::install(jsiRuntime, module);
rnexecutorch::extensions::nlp::install(jsiRuntime, module);
+ rnexecutorch::extensions::speech::install(jsiRuntime, module);
jsiRuntime.global().setProperty(jsiRuntime, "__rnexecutorch_jsi__", std::move(module));
}
diff --git a/packages/react-native-executorch/cpp/extensions/speech/install.cpp b/packages/react-native-executorch/cpp/extensions/speech/install.cpp
new file mode 100644
index 0000000000..78b014eb89
--- /dev/null
+++ b/packages/react-native-executorch/cpp/extensions/speech/install.cpp
@@ -0,0 +1,14 @@
+#include "install.h"
+#include "operations.h"
+
+namespace rnexecutorch::extensions::speech {
+namespace jsi = facebook::jsi;
+
+void install(jsi::Runtime &rt, jsi::Object &module) {
+ jsi::Object speechModule(rt);
+
+ install_extractFrames(rt, speechModule);
+
+ module.setProperty(rt, "speech", speechModule);
+}
+} // namespace rnexecutorch::extensions::speech
diff --git a/packages/react-native-executorch/cpp/extensions/speech/install.h b/packages/react-native-executorch/cpp/extensions/speech/install.h
new file mode 100644
index 0000000000..177d377dd8
--- /dev/null
+++ b/packages/react-native-executorch/cpp/extensions/speech/install.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include
+
+namespace rnexecutorch::extensions::speech {
+void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
+} // namespace rnexecutorch::extensions::speech
diff --git a/packages/react-native-executorch/cpp/extensions/speech/operations.cpp b/packages/react-native-executorch/cpp/extensions/speech/operations.cpp
new file mode 100644
index 0000000000..0014c9e9ec
--- /dev/null
+++ b/packages/react-native-executorch/cpp/extensions/speech/operations.cpp
@@ -0,0 +1,96 @@
+#include "operations.h"
+
+#include
+#include
+#include
+#include
+
+#include "core/tensor.h"
+#include "core/tensor_helpers.h"
+
+namespace rnexecutorch::extensions::speech {
+namespace jsi = facebook::jsi;
+namespace conversions = rnexecutorch::core::conversions;
+namespace tensor = rnexecutorch::core::tensor;
+using rnexecutorch::core::types::DType;
+
+// Slices a mono waveform into overlapping frames, applying per-frame
+// mean-removal, a pre-emphasis filter and a Hann window, writing each frame into
+// a zero-padded row of `dst`. Mirrors the reference FSMN-VAD feature extraction.
+// The whole per-frame inner loop dominates the VAD pipeline (~85% of a detect()
+// call on device), so it lives in native code per the extension guidelines.
+void install_extractFrames(jsi::Runtime &rt, jsi::Object &module) {
+ const auto *name = "extractFrames";
+ auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value {
+ if (count != 4) {
+ throw jsi::JSError(rt, "Usage: extractFrames(waveform, hann, dst, options)");
+ }
+
+ auto waveform = tensor::fromJs(rt, "extractFrames: waveform", args[0], DType::float32, {"length"});
+ auto hann = tensor::fromJs(rt, "extractFrames: hann", args[1], DType::float32, {"frameLength"});
+ auto dst = tensor::fromJs(rt, "extractFrames: dst", args[2], DType::float32, {"frames", "fftLength"});
+
+ auto options = conversions::asType(rt, "extractFrames: options", args[3]);
+ auto numFrames = conversions::getRequiredProperty(rt, "extractFrames", options, "numFrames");
+ auto hopLength = conversions::getRequiredProperty(rt, "extractFrames", options, "hopLength");
+ auto preemphasis = conversions::getRequiredProperty(rt, "extractFrames", options, "preemphasis");
+
+ tensor::checkNotSameTensor(rt, "extractFrames: waveform", waveform, "extractFrames: hann", hann);
+ tensor::checkNotSameTensor(rt, "extractFrames: waveform", waveform, "extractFrames: dst", dst);
+ tensor::checkNotSameTensor(rt, "extractFrames: hann", hann, "extractFrames: dst", dst);
+ auto waveLock = tensor::tryLockShared(rt, "extractFrames: waveform", waveform);
+ auto hannLock = tensor::tryLockShared(rt, "extractFrames: hann", hann);
+ auto dstLock = tensor::tryLockUnique(rt, "extractFrames: dst", dst);
+
+ const auto frameLength = hann->numel_;
+ const auto chunkFrames = static_cast(dst->shape_[0]);
+ const auto fftLength = static_cast(dst->shape_[1]);
+ if (frameLength > fftLength) {
+ throw jsi::JSError(rt, "extractFrames: hann length exceeds dst fftLength");
+ }
+ if (numFrames > chunkFrames) {
+ throw jsi::JSError(rt, "extractFrames: numFrames out of dst frame capacity");
+ }
+
+ if (numFrames > 0) {
+ const uint64_t lastSample = (numFrames - 1) * hopLength + frameLength - 1;
+ if (lastSample >= waveform->numel_) {
+ throw jsi::JSError(rt, "extractFrames: frame window out of waveform bounds");
+ }
+ }
+
+ const auto leftPad = (fftLength - frameLength) / 2;
+ const std::span wave(reinterpret_cast(waveform->data_.get()), waveform->numel_);
+ const std::span win(reinterpret_cast(hann->data_.get()), hann->numel_);
+ const std::span out(reinterpret_cast(dst->data_.get()), dst->numel_);
+
+ // Zero the destination so intra-frame and trailing-row padding stay 0.
+ std::ranges::fill(out, 0.0f);
+
+ for (uint64_t f = 0; f < numFrames; ++f) {
+ const auto frame = wave.subspan(static_cast(f * hopLength), frameLength);
+ const auto base = out.subspan(static_cast(f * fftLength + leftPad), frameLength);
+
+ float sum = 0.0f;
+ for (const float sample : frame) {
+ sum += sample;
+ }
+ const float mean = sum / static_cast(frameLength);
+
+ // Pre-emphasis of a mean-subtracted signal, (raw[j]-mean) -
+ // c*(raw[j-1]-mean), equals raw[j] - c*raw[j-1] - mean*(1-c), so each
+ // output reads only raw samples — no serial dependency, and the fused
+ // mean-removal + pre-emphasis + Hann loop vectorizes.
+ const float meanBias = mean * (1.0f - preemphasis);
+ base[0] = (frame[0] - mean) * win[0];
+ for (std::size_t j = 1; j < frameLength; ++j) {
+ base[j] = (frame[j] - preemphasis * frame[j - 1] - meanBias) * win[j];
+ }
+ }
+
+ return jsi::Value(rt, args[2]);
+ };
+
+ module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 4, fnBody));
+}
+} // namespace rnexecutorch::extensions::speech
diff --git a/packages/react-native-executorch/cpp/extensions/speech/operations.h b/packages/react-native-executorch/cpp/extensions/speech/operations.h
new file mode 100644
index 0000000000..22b0fb6357
--- /dev/null
+++ b/packages/react-native-executorch/cpp/extensions/speech/operations.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include
+
+namespace rnexecutorch::extensions::speech {
+void install_extractFrames(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
+} // namespace rnexecutorch::extensions::speech
diff --git a/packages/react-native-executorch/src/extensions/speech/index.ts b/packages/react-native-executorch/src/extensions/speech/index.ts
new file mode 100644
index 0000000000..d1a0869dfb
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/index.ts
@@ -0,0 +1 @@
+export * from './ops';
diff --git a/packages/react-native-executorch/src/extensions/speech/ops.ts b/packages/react-native-executorch/src/extensions/speech/ops.ts
new file mode 100644
index 0000000000..bfda4c143b
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/ops.ts
@@ -0,0 +1,40 @@
+import { rnexecutorchJsi } from '../../native/bridge';
+import { type Tensor } from '../../core/tensor';
+
+/**
+ * Options controlling how {@link extractFrames} slices and filters the waveform.
+ * @category Types
+ * @property {number} numFrames - Number of frames to write (must be `<= dst.shape[0]`).
+ * @property {number} hopLength - Samples between consecutive frames.
+ * @property {number} preemphasis - Pre-emphasis filter coefficient.
+ */
+export type ExtractFramesOptions = {
+ readonly numFrames: number;
+ readonly hopLength: number;
+ readonly preemphasis: number;
+};
+
+/**
+ * Slices a mono `waveform` into `numFrames` overlapping
+ * frames, applying per-frame mean-removal, a pre-emphasis filter and the `hann`
+ * window, and writing each frame into a zero-padded row of `dst` (shape
+ * `[frames, fftLength]`). `dst` is fully zeroed first, so rows beyond
+ * `numFrames` stay zero (padding). The frame length is taken from `hann`'s
+ * length and the padded width from `dst`'s last dimension.
+ * @category Typescript API
+ * @param waveform Input audio samples, shape `[length]`. Framing starts at the
+ * first sample, so pass only the slice to be framed.
+ * @param hann Precomputed Hann window, shape `[frameLength]`.
+ * @param dst Pre-allocated destination, shape `[frames, fftLength]`.
+ * @param options Framing options.
+ * @returns The `dst` tensor, for convenience.
+ */
+export function extractFrames(
+ waveform: Tensor,
+ hann: Tensor,
+ dst: Tensor,
+ options: ExtractFramesOptions
+): Tensor {
+ 'worklet';
+ return rnexecutorchJsi.speech.extractFrames(waveform, hann, dst, options);
+}
diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts
new file mode 100644
index 0000000000..557c937958
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts
@@ -0,0 +1,352 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
+import { wrapAsync } from '../../../core/runtime';
+import { extractFrames } from '../ops';
+
+/** Sample rate (Hz) the FSMN-VAD model expects its input waveform to be at. */
+export const FSMN_VAD_SAMPLE_RATE_HZ = 16000;
+
+// Feature-extraction geometry baked into the exported FSMN-VAD `.pte`. These are
+// properties of the model rather than user-tunable knobs, so they live here and
+// not in the `models` registry; `fftLength` is read from the model metadata.
+const FRAME_LENGTH = 400; // 25 ms analysis window
+const HOP_LENGTH = 160; // 10 ms between windows
+const PREEMPHASIS = 0.97;
+const MIN_FRAMES = 100; // fewest frames the model accepts per forward pass
+
+const SAMPLES_PER_MS = FSMN_VAD_SAMPLE_RATE_HZ / 1000;
+const HOP_LENGTH_MS = HOP_LENGTH / SAMPLES_PER_MS;
+
+/**
+ * Tunable thresholds controlling how per-frame speech probabilities are turned
+ * into speech {@link Segment}s.
+ * @category Types
+ * @property {number} [speechThreshold] - Minimum speech probability (0-1) for a
+ * frame to count as speech. Defaults to `0.6`.
+ * @property {number} [minSpeechDurationMs] - Minimum duration a region must stay
+ * above the threshold to open a segment. Defaults to `250`.
+ * @property {number} [minSilenceDurationMs] - Minimum duration below the
+ * threshold required to close a segment. Defaults to `100`.
+ * @property {number} [speechPadMs] - Padding added to both ends of every
+ * detected segment. Defaults to `30`.
+ * @property {number} [mergeGapMs] - Segments closer than this gap are merged
+ * into one. Defaults to `0`.
+ */
+export type VadOptions = {
+ readonly speechThreshold?: number;
+ readonly minSpeechDurationMs?: number;
+ readonly minSilenceDurationMs?: number;
+ readonly speechPadMs?: number;
+ readonly mergeGapMs?: number;
+};
+
+/**
+ * Model configuration required to instantiate an FSMN-VAD task runner.
+ * @category Types
+ * @property {string} modelPath - Local path or remote URL of the `.pte` model.
+ * @property {VadOptions} [defaultOptions] - Detection thresholds tuned for this
+ * model; overridable per `detect` call. Falls back to the library defaults.
+ */
+export type FsmnVadModel = {
+ readonly modelPath: string;
+ readonly defaultOptions?: VadOptions;
+};
+
+/**
+ * A detected speech region, with start and end expressed in seconds.
+ * @category Types
+ */
+export type Segment = {
+ readonly start: number;
+ readonly end: number;
+};
+
+/**
+ * Options controlling live detection via `push`. Extends the per-call detection
+ * thresholds ({@link VadOptions}).
+ * @category Types
+ * @property {number} [detectionMargin] - How recent (in milliseconds) the last
+ * detected speech segment must reach toward the end of the window for speech to
+ * still be considered ongoing. Defaults to `100`.
+ */
+export type VadStreamOptions = VadOptions & {
+ readonly detectionMargin?: number;
+};
+
+/**
+ * A speech-activity transition reported by `push`.
+ * @category Types
+ */
+export type VadEvent = 'speechStart' | 'speechEnd';
+
+// Library fallback thresholds. A model may override any of these via
+// `FsmnVadModel.defaultOptions`, and callers via the `detect` options argument.
+const DEFAULT_OPTIONS: Required = {
+ speechThreshold: 0.6,
+ minSpeechDurationMs: 250,
+ minSilenceDurationMs: 100,
+ speechPadMs: 30,
+ mergeGapMs: 0,
+};
+
+// `push` detects over a bounded window of the most recent audio (not the whole
+// growing session): model cost scales with window length, and 2.5s already
+// exceeds FSMN's receptive field (~200ms) and the min-speech duration (250ms).
+const DETECTION_WINDOW_SECONDS = 2.5;
+const DEFAULT_DETECTION_MARGIN_MS = 100;
+
+// Periodic Hann window used to reduce spectral leakage on each frame. Ported
+// from `dsp::hannWindow` (periodic definition, divides by `size`).
+function hannWindow(size: number): Float32Array {
+ 'worklet';
+ const window = new Float32Array(size);
+ for (let i = 0; i < size; i++) {
+ window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / size));
+ }
+ return window;
+}
+
+// Turns per-frame non-speech probabilities into speech segments in seconds:
+// threshold with hysteresis, pad both ends, then merge near-adjacent regions.
+// Mirrors `VoiceActivityDetection::postprocess` + `utils::mergeSegments`.
+// `scores[i]` holds the non-speech probability of frame `i`, so the speech
+// probability is `1 - scores[i]`.
+function postprocess(scores: Float32Array, opts: Required): Segment[] {
+ 'worklet';
+ const threshold = opts.speechThreshold;
+ const minSpeechHops = Math.floor(opts.minSpeechDurationMs / HOP_LENGTH_MS);
+ const minSilenceHops = Math.floor(opts.minSilenceDurationMs / HOP_LENGTH_MS);
+ const speechPadHops = Math.floor(opts.speechPadMs / HOP_LENGTH_MS);
+ const maxMergeGapHops = opts.mergeGapMs / HOP_LENGTH_MS;
+
+ // Threshold with hysteresis: a region must stay above the threshold for
+ // `minSpeechHops` to open a segment, and below it for `minSilenceHops` to
+ // close one. Bounds are in hop (frame) units.
+ const hops: Segment[] = [];
+ let triggered = false;
+ let startHop = -1;
+ let potentialStart = -1;
+ let potentialEnd = -1;
+
+ for (let i = 0; i < scores.length; i++) {
+ const isSpeech = 1 - scores[i]! >= threshold;
+ if (!triggered) {
+ if (!isSpeech) {
+ potentialStart = -1;
+ } else if (potentialStart === -1) {
+ potentialStart = i;
+ } else if (i - potentialStart >= minSpeechHops) {
+ triggered = true;
+ startHop = potentialStart;
+ potentialStart = -1;
+ }
+ } else if (isSpeech) {
+ potentialEnd = -1;
+ } else if (potentialEnd === -1) {
+ potentialEnd = i;
+ } else if (i - potentialEnd >= minSilenceHops) {
+ triggered = false;
+ hops.push({ start: startHop, end: potentialEnd });
+ potentialEnd = -1;
+ }
+ }
+ if (triggered) hops.push({ start: startHop, end: scores.length });
+
+ // Pad both ends, then merge regions separated by at most `mergeGapMs`.
+ const merged: Segment[] = [];
+ for (const hop of hops) {
+ const start = hop.start > speechPadHops ? hop.start - speechPadHops : 0;
+ const end = Math.min(hop.end + speechPadHops, scores.length);
+ const last = merged[merged.length - 1];
+ if (last && (start < last.end || start - last.end <= maxMergeGapHops)) {
+ merged[merged.length - 1] = { start: last.start, end: Math.max(last.end, end) };
+ } else {
+ merged.push({ start, end });
+ }
+ }
+
+ const hopSeconds = HOP_LENGTH / FSMN_VAD_SAMPLE_RATE_HZ;
+ return merged.map((s) => ({ start: s.start * hopSeconds, end: s.end * hopSeconds }));
+}
+
+/**
+ * Creates a Voice Activity Detection runner for the FSMN-VAD model.
+ *
+ * It loads the model, validates its input/output signature, pre-allocates the
+ * static output tensor and registers a disposal hook. The whole pipeline —
+ * feature extraction, chunked inference and segment postprocessing — runs in
+ * TypeScript on top of the core `model.execute` primitive.
+ *
+ * The model exposes a dynamic frame dimension: each forward pass accepts
+ * `[frames, fftLength]` (up to the model-declared maximum) and returns per-frame
+ * class probabilities. Long inputs are split into chunks; a short final chunk is
+ * zero-padded up to the model minimum and its padding scores are discarded.
+ * @category Typescript API
+ * @param config VAD task configuration containing the model path.
+ * @param runtime Optional worklet runtime thread on which to run the model
+ * execution.
+ * @returns A promise resolving to an object containing detection and disposal
+ * controls.
+ */
+export async function createFsmnVoiceActivityDetector(
+ config: FsmnVadModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ /**
+ * Releases all allocated native resources.
+ */
+ dispose: () => void;
+ /**
+ * Asynchronously detects speech segments within a mono waveform sampled at
+ * {@link FSMN_VAD_SAMPLE_RATE_HZ}.
+ * @param waveform The input audio samples.
+ * @param options Optional per-call overrides of the detection thresholds.
+ * @returns A promise resolving to the detected speech segments, in seconds.
+ */
+ detect: (waveform: Float32Array, options?: VadOptions) => Promise;
+ /**
+ * Synchronous version of {@link detect} to be executed directly on the caller
+ * or worklet thread.
+ */
+ detectWorklet: (waveform: Float32Array, options?: VadOptions) => Segment[];
+ /**
+ * Appends a live audio chunk to a bounded rolling window, runs detection over
+ * that window and reports a {@link VadEvent} when speech starts or stops,
+ * otherwise `null`. Designed to be driven straight from a recorder callback.
+ * Detection runs synchronously on the calling thread (a few ms). The rolling
+ * window is required: one short chunk is far too short to satisfy
+ * `minSpeechDurationMs` on its own.
+ * @param chunk The newly captured audio samples.
+ * @param options Optional overrides of the detection thresholds and margin.
+ */
+ push: (chunk: Float32Array, options?: VadStreamOptions) => VadEvent | null;
+ /**
+ * Clears the rolling window and speaking state used by {@link push}.
+ */
+ resetStream: () => void;
+}> {
+ const { modelPath } = config;
+ const modelDefaults: Required = { ...DEFAULT_OPTIONS, ...config.defaultOptions };
+
+ const model = await wrapAsync(loadModel, runtime)(modelPath);
+
+ // Input is [frames, fftLength] with a dynamic frame count; output is per-frame
+ // class probabilities, either [1, frames, classes] or [frames, classes], where
+ // class 0 is the non-speech class. The input and output frame symbols are
+ // deliberately distinct: a symbol only binds within a single tensor, so the
+ // two cannot be cross-constrained here.
+ const meta = validateModelSchema(
+ model,
+ 'forward',
+ [SymbolicTensor('float32', ['inFrames', 'fftLength'])],
+ [SymbolicTensor('float32', [1, 'outFrames', 'classes'], ['outFrames', 'classes'])]
+ );
+ const chunkCapacity = meta.inputTensorMeta[0]!.shape[0]!;
+ const fftLength = meta.inputTensorMeta[0]!.shape[1]!;
+ const outShape = meta.outputTensorMeta[0]!.shape;
+ const numClass = outShape[outShape.length - 1]!;
+ // The output frame count tracks the input's, so the output shape is the
+ // declared one with its frame dimension swapped per chunk (see below).
+ const outFramesDim = outShape.length - 2;
+
+ // The Hann window is uploaded once and reused by the native framing op across
+ // every call. The output tensor cannot be pre-allocated here: this model's
+ // output is dynamic too, so `execute` validates it against the shape produced
+ // for the current chunk, not the model-declared maximum.
+ const tensors = [tensor('float32', [FRAME_LENGTH], hannWindow(FRAME_LENGTH))] as const;
+ const [tHann] = tensors;
+
+ const dispose = () => {
+ tensors.forEach((t) => t.dispose());
+ model.dispose();
+ };
+
+ const detectWorklet = (waveform: Float32Array, options?: VadOptions): Segment[] => {
+ 'worklet';
+ if (waveform.length < FRAME_LENGTH) return [];
+
+ const opts: Required = { ...modelDefaults, ...options };
+ const numFrames = Math.floor((waveform.length - FRAME_LENGTH) / HOP_LENGTH);
+ if (numFrames <= 0) return [];
+
+ const scores = new Float32Array(numFrames);
+ let offset = 0;
+ while (offset < numFrames) {
+ const realFrames = Math.min(numFrames - offset, chunkCapacity);
+ // The model needs at least MIN_FRAMES rows, so a short final chunk is
+ // zero-padded up to it and its padding scores are discarded below.
+ const chunkFrames = Math.max(realFrames, MIN_FRAMES);
+ const startSample = offset * HOP_LENGTH;
+ const sampleCount = (realFrames - 1) * HOP_LENGTH + FRAME_LENGTH;
+
+ const chunkOutShape = outShape.slice();
+ chunkOutShape[outFramesDim] = chunkFrames;
+
+ const tWaveform = tensor(
+ 'float32',
+ [sampleCount],
+ waveform.subarray(startSample, startSample + sampleCount)
+ );
+ const tInput = tensor('float32', [chunkFrames, fftLength]);
+ const tOutput = tensor('float32', chunkOutShape);
+ const outBuffer = new Float32Array(tOutput.numel);
+ try {
+ extractFrames(tWaveform, tHann, tInput, {
+ numFrames: realFrames,
+ hopLength: HOP_LENGTH,
+ preemphasis: PREEMPHASIS,
+ });
+ model.execute('forward', [tInput], [tOutput]);
+ tOutput.getData(outBuffer);
+ } finally {
+ tOutput.dispose();
+ tInput.dispose();
+ tWaveform.dispose();
+ }
+
+ for (let i = 0; i < realFrames; i++) {
+ scores[offset + i] = outBuffer[i * numClass]!;
+ }
+ offset += realFrames;
+ }
+
+ return postprocess(scores, opts);
+ };
+
+ const detect = wrapAsync(detectWorklet, runtime);
+
+ const windowSamples = DETECTION_WINDOW_SECONDS * FSMN_VAD_SAMPLE_RATE_HZ;
+ let window = new Float32Array(0);
+ let isSpeaking = false;
+
+ const push = (chunk: Float32Array, options?: VadStreamOptions): VadEvent | null => {
+ const next = new Float32Array(window.length + chunk.length);
+ next.set(window);
+ next.set(chunk, window.length);
+ window = next.length > windowSamples ? next.slice(next.length - windowSamples) : next;
+
+ // Speech is ongoing if the last detected segment reaches close enough to the
+ // end of the window (within detectionMargin).
+ const segments = detectWorklet(window, options);
+ let speaking = false;
+ if (segments.length > 0) {
+ const windowEndSec = window.length / FSMN_VAD_SAMPLE_RATE_HZ;
+ const diffMs = (windowEndSec - segments[segments.length - 1]!.end) * 1000;
+ speaking = diffMs <= (options?.detectionMargin ?? DEFAULT_DETECTION_MARGIN_MS);
+ }
+
+ if (speaking === isSpeaking) return null;
+ isSpeaking = speaking;
+ return speaking ? 'speechStart' : 'speechEnd';
+ };
+
+ const resetStream = () => {
+ window = new Float32Array(0);
+ isSpeaking = false;
+ };
+
+ return { dispose, detect, detectWorklet, push, resetStream };
+}
diff --git a/packages/react-native-executorch/src/hooks/useVoiceActivityDetection.ts b/packages/react-native-executorch/src/hooks/useVoiceActivityDetection.ts
new file mode 100644
index 0000000000..cfd6fafa58
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useVoiceActivityDetection.ts
@@ -0,0 +1,48 @@
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import {
+ createFsmnVoiceActivityDetector,
+ type FsmnVadModel,
+} from '../extensions/speech/tasks/fsmnVoiceActivityDetection';
+
+/**
+ * React hook to load and run a Voice Activity Detection model.
+ *
+ * This hook manages downloading (if it's a remote URL) and loading the model
+ * file, compiling it, tracking download progress and compilation errors, and
+ * cleaning up native model memory when the component unmounts or configuration
+ * changes. The rolling-window state driven by `push` lives inside the task (see
+ * {@link createFsmnVoiceActivityDetector}), so it is exposed here directly.
+ * @category Hooks
+ * @param config The VAD model configuration.
+ * @param options Hook options.
+ * @param options.preventLoad If true, prevents downloading and compiling the
+ * model.
+ * @returns An object containing the model's loading state, error, download
+ * progress, one-shot detection functions, and live detection controls.
+ */
+export function useVoiceActivityDetection(
+ config: FsmnVadModel,
+ options?: { preventLoad?: boolean }
+) {
+ const { localPath, downloadProgress, downloadError } = useResourceDownload(
+ config.modelPath,
+ options?.preventLoad
+ );
+ const { model, error } = useModel(
+ createFsmnVoiceActivityDetector,
+ localPath ? { ...config, modelPath: localPath } : null,
+ [localPath]
+ );
+
+ return {
+ isReady: !!model,
+ error: downloadError || error,
+ downloadProgress,
+ localPath,
+ detect: model?.detect,
+ detectWorklet: model?.detectWorklet,
+ push: model?.push,
+ resetStream: model?.resetStream,
+ };
+}
diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts
index bc416044a9..1f473a0036 100644
--- a/packages/react-native-executorch/src/index.ts
+++ b/packages/react-native-executorch/src/index.ts
@@ -8,6 +8,7 @@ export * from './hooks/useObjectDetector';
export * from './hooks/useTokenizer';
export * from './hooks/useTextEmbedder';
export * from './hooks/useImageEmbedder';
+export * from './hooks/useVoiceActivityDetection';
export * from './hooks/useResourceDownload';
export * from './hooks/useModel';
@@ -25,6 +26,7 @@ export * from './extensions/cv/tasks/objectDetection';
export * from './extensions/cv/tasks/imageEmbedding';
export * from './extensions/nlp/tasks/tokenization';
export * from './extensions/nlp/tasks/textEmbedding';
+export * from './extensions/speech/tasks/fsmnVoiceActivityDetection';
// Core primitives — for library builders and power users
export { tensor } from './core/tensor';
@@ -48,6 +50,7 @@ export { defaultWorkletRuntime, wrapAsync } from './core/runtime';
export * as math from './extensions/math';
export * as cv from './extensions/cv';
export * as nlp from './extensions/nlp';
+export * as speech from './extensions/speech';
// Utils
export * from './utils';
diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts
index 239655b8e9..8b4e10f1d7 100644
--- a/packages/react-native-executorch/src/models.ts
+++ b/packages/react-native-executorch/src/models.ts
@@ -6,6 +6,7 @@ import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetect
import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation';
import type { ImageEmbedderModel } from './extensions/cv/tasks/imageEmbedding';
import type { TextEmbedderModel } from './extensions/nlp/tasks/textEmbedding';
+import type { FsmnVadModel } from './extensions/speech/tasks/fsmnVoiceActivityDetection';
import {
IMAGENET_NORM,
IMAGENET1K_LABELS,
@@ -590,6 +591,13 @@ const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_INT8: ImageEmbedderModel = {
opts: CLIP_IMAGE_EMBEDDINGS_OPTS,
};
+// =============================================================================
+// Voice Activity Detection
+// =============================================================================
+const FSMN_VAD_XNNPACK_FP32: FsmnVadModel = {
+ modelPath: `${BASE_URL}-fsmn-vad/${NEXT_VERSION_TAG}/xnnpack/fsmn_vad_xnnpack_fp32.pte`,
+};
+
// =============================================================================
// Tokenizers
// =============================================================================
@@ -796,6 +804,12 @@ export const models = {
},
},
},
+ vad: {
+ FSMN_VAD: {
+ ...FSMN_VAD_XNNPACK_FP32,
+ XNNPACK_FP32: FSMN_VAD_XNNPACK_FP32,
+ },
+ },
tokenizer: {
ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER,
},
diff --git a/yarn.lock b/yarn.lock
index bc05759538..503786b0dc 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -12764,6 +12764,24 @@ __metadata:
languageName: node
linkType: hard
+"react-native-audio-api@npm:0.13.1":
+ version: 0.13.1
+ resolution: "react-native-audio-api@npm:0.13.1"
+ dependencies:
+ semver: "npm:^7.7.3"
+ peerDependencies:
+ react: "*"
+ react-native: "*"
+ react-native-worklets: ">= 0.6.0"
+ peerDependenciesMeta:
+ react-native-worklets:
+ optional: true
+ bin:
+ setup-rn-audio-api-web: scripts/setup-rn-audio-api-web.js
+ checksum: 10/30675d28706f4cf03beb925120a6745d858f7dead4bc0df2dd734d6d0feaafc14924bbb1dd261c4f158a4501743f8abe72e029aadac8759b442480c01850ca88
+ languageName: node
+ linkType: hard
+
"react-native-builder-bob@npm:^0.40.18":
version: 0.40.18
resolution: "react-native-builder-bob@npm:0.40.18"
@@ -12796,6 +12814,15 @@ __metadata:
languageName: node
linkType: hard
+"react-native-device-info@npm:^15.0.2":
+ version: 15.0.2
+ resolution: "react-native-device-info@npm:15.0.2"
+ peerDependencies:
+ react-native: "*"
+ checksum: 10/9f945a16bd1aebefc60868e22552375bc630eb98d5505d8cdc847d7841a226540408e7ea174cbb4eb5bfec17235eaca731d154082f64e4935e2a4d2e0244b39b
+ languageName: node
+ linkType: hard
+
"react-native-drawer-layout@npm:^4.2.2, react-native-drawer-layout@npm:^4.2.5":
version: 4.2.5
resolution: "react-native-drawer-layout@npm:4.2.5"
@@ -14036,6 +14063,38 @@ __metadata:
languageName: node
linkType: hard
+"speech@workspace:apps/speech":
+ version: 0.0.0-use.local
+ resolution: "speech@workspace:apps/speech"
+ dependencies:
+ "@babel/core": "npm:^7.29.0"
+ "@react-native/metro-config": "npm:^0.86.0"
+ "@react-navigation/drawer": "npm:^7.9.4"
+ "@react-navigation/native": "npm:^7.2.2"
+ "@types/react": "npm:~19.2.0"
+ babel-preset-expo: "npm:~56.0.14"
+ expo: "npm:~56.0.9"
+ expo-build-properties: "npm:~56.0.17"
+ expo-constants: "npm:~56.0.17"
+ expo-linking: "npm:~56.0.13"
+ expo-router: "npm:~56.2.9"
+ react: "npm:19.2.3"
+ react-native: "npm:0.85.3"
+ react-native-audio-api: "npm:0.13.1"
+ react-native-device-info: "npm:^15.0.2"
+ react-native-drawer-layout: "npm:^4.2.2"
+ react-native-executorch: "workspace:*"
+ react-native-gesture-handler: "npm:~2.31.1"
+ react-native-reanimated: "npm:4.4.0"
+ react-native-safe-area-context: "npm:~5.7.0"
+ react-native-screens: "npm:~4.25.2"
+ react-native-svg: "npm:15.15.4"
+ react-native-svg-transformer: "npm:^1.5.3"
+ react-native-worklets: "npm:0.9.1"
+ react-refresh: "npm:^0.14.0"
+ languageName: unknown
+ linkType: soft
+
"split-on-first@npm:^1.0.0":
version: 1.1.0
resolution: "split-on-first@npm:1.1.0"