diff --git a/apps/speech/screens/TextToSpeechScreen.tsx b/apps/speech/screens/TextToSpeechScreen.tsx index 919076dc35..1bdeb9f542 100644 --- a/apps/speech/screens/TextToSpeechScreen.tsx +++ b/apps/speech/screens/TextToSpeechScreen.tsx @@ -14,34 +14,95 @@ import { models, useTextToSpeech, TextToSpeechModelConfig, + TextToSpeechSupertonicLanguage, } from 'react-native-executorch'; import { ModelPicker, ModelOption } from '../components/ModelPicker'; -const tts = models.text_to_speech.kokoro; +// Supertonic 3: a single multilingual model with 10 built-in voices. Each voice +// works for any of the supported languages, and both the language and the +// number of flow-matching steps are set per synthesis call (no model reload) — +// see the LANGUAGES / STEPS pickers below. +const tts = models.text_to_speech.supertonic; const VOICES: ModelOption[] = [ - { label: '🇺🇸 AF Heart', value: tts.en_us.heart() }, - { label: '🇺🇸 AF River', value: tts.en_us.river() }, - { label: '🇺🇸 AF Sarah', value: tts.en_us.sarah() }, - { label: '🇺🇸 AM Adam', value: tts.en_us.adam() }, - { label: '🇺🇸 AM Michael', value: tts.en_us.michael() }, - { label: '🇺🇸 AM Santa', value: tts.en_us.santa() }, - { label: '🇬🇧 BF Emma', value: tts.en_gb.emma() }, - { label: '🇬🇧 BM Daniel', value: tts.en_gb.daniel() }, - { label: '🇫🇷 FF Siwis', value: tts.fr.siwis() }, - { label: '🇪🇸 EF Dora', value: tts.es.dora() }, - { label: '🇪🇸 EM Alex', value: tts.es.alex() }, - { label: '🇮🇹 IF Sara', value: tts.it.sara() }, - { label: '🇮🇹 IM Nicola', value: tts.it.nicola() }, - { label: '🇵🇹 PF Dora', value: tts.pt.dora() }, - { label: '🇵🇹 PM Santa', value: tts.pt.santa() }, - { label: '🇩🇪 DF Anna', value: tts.de.anna() }, - { label: '🇵🇱 PM Mateusz', value: tts.pl.mateusz() }, - { label: '🇮🇳 HF Alpha', value: tts.hi.alpha() }, - { label: '🇮🇳 HM Omega', value: tts.hi.omega() }, - { label: '🇮🇳 HM Psi', value: tts.hi.psi() }, + { label: '👨 Male 1', value: tts.m1() }, + { label: '👨 Male 2', value: tts.m2() }, + { label: '👨 Male 3', value: tts.m3() }, + { label: '👨 Male 4', value: tts.m4() }, + { label: '👨 Male 5', value: tts.m5() }, + { label: '👩 Female 1', value: tts.f1() }, + { label: '👩 Female 2', value: tts.f2() }, + { label: '👩 Female 3', value: tts.f3() }, + { label: '👩 Female 4', value: tts.f4() }, + { label: '👩 Female 5', value: tts.f5() }, ]; +// A per-call hyperparameter: the `` token. 'na' = unknown/other. +const LANGUAGES: ModelOption[] = [ + { label: '🇺🇸 English', value: 'en' }, + { label: '🇪🇸 Spanish', value: 'es' }, + { label: '🇫🇷 French', value: 'fr' }, + { label: '🇩🇪 German', value: 'de' }, + { label: '🇮🇹 Italian', value: 'it' }, + { label: '🇵🇹 Portuguese', value: 'pt' }, + { label: '🇵🇱 Polish', value: 'pl' }, + { label: '🇷🇺 Russian', value: 'ru' }, + { label: '🇯🇵 Japanese', value: 'ja' }, + { label: '🇰🇷 Korean', value: 'ko' }, + { label: '🇮🇳 Hindi', value: 'hi' }, + { label: '🌐 Other (na)', value: 'na' }, +]; + +// A per-call hyperparameter: number of flow-matching steps (quality vs. speed). +const STEPS: ModelOption[] = [ + { label: '6 (fastest)', value: 6 }, + { label: '8 (default)', value: 8 }, + { label: '16 (best quality)', value: 16 }, +]; + +// Kokoro: language-specific voices. Each voice bundles its own phonemizer +// config, so only the voice picker is shown (no language/steps controls). +const KOKORO_LANG_LABELS: Record = { + en_us: '🇺🇸 US English', + en_gb: '🇬🇧 UK English', + fr: '🇫🇷 French', + es: '🇪🇸 Spanish', + it: '🇮🇹 Italian', + pt: '🇵🇹 Portuguese', + hi: '🇮🇳 Hindi', + pl: '🇵🇱 Polish', + de: '🇩🇪 German', +}; + +const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + +// `models.text_to_speech.kokoro..` are factory functions +// (`() => TextToSpeechModelConfig`); call them to get the actual configs. +const kokoroVoicesByLang = models.text_to_speech.kokoro as unknown as Record< + string, + Record TextToSpeechModelConfig> +>; + +const KOKORO_VOICES: ModelOption[] = Object.entries( + kokoroVoicesByLang +).flatMap(([lang, voices]) => + Object.entries(voices).map(([name, factory]) => ({ + label: `${KOKORO_LANG_LABELS[lang] ?? lang} · ${capitalize(name)}`, + value: factory(), + })) +); + +type TtsModelType = 'supertonic' | 'kokoro'; + +const TTS_MODEL_OPTIONS: ModelOption[] = [ + { label: 'Supertonic', value: 'supertonic' }, + { label: 'Kokoro', value: 'kokoro' }, +]; + +// Supertonic renders at 44.1 kHz, Kokoro at 24 kHz. +const SUPERTONIC_SAMPLE_RATE = 44100; +const KOKORO_SAMPLE_RATE = 24000; + import FontAwesome from '@expo/vector-icons/FontAwesome'; import { AudioManager, @@ -56,13 +117,13 @@ import ErrorBanner from '../components/ErrorBanner'; * Converts an audio vector (Float32Array) to an AudioBuffer for playback * @param audioVector - The generated audio samples from the model * @param audioContext - An optional AudioContext to create the buffer in. If not provided, a new one will be created. - * @param sampleRate - The sample rate (default: 24000 Hz for Kokoro) + * @param sampleRate - The sample rate (default: 44100 Hz for Supertonic) * @returns AudioBuffer ready for playback */ const createAudioBufferFromVector = ( audioVector: Float32Array, audioContext: AudioContext | null = null, - sampleRate: number = 24000 + sampleRate: number = SUPERTONIC_SAMPLE_RATE ): AudioBuffer => { if (audioContext == null) audioContext = new AudioContext({ sampleRate }); @@ -78,11 +139,29 @@ const createAudioBufferFromVector = ( }; export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { + const [selectedTtsModel, setSelectedTtsModel] = + useState('supertonic'); const [selectedSpeaker, setSelectedSpeaker] = - useState(tts.en_us.heart()); + useState(tts.m1()); + const [selectedLang, setSelectedLang] = + useState('en'); + const [totalSteps, setTotalSteps] = useState(8); const model = useTextToSpeech(selectedSpeaker); + const sampleRate = + selectedTtsModel === 'supertonic' + ? SUPERTONIC_SAMPLE_RATE + : KOKORO_SAMPLE_RATE; + + const handleSelectTtsModel = (type: TtsModelType) => { + if (type === selectedTtsModel) return; + setSelectedTtsModel(type); + setSelectedSpeaker( + type === 'supertonic' ? VOICES[0].value : KOKORO_VOICES[0].value + ); + }; + const [inputText, setInputText] = useState(''); const [isPlaying, setIsPlaying] = useState(false); const [readyToGenerate, setReadyToGenerate] = useState(false); @@ -99,7 +178,7 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { iosOptions: ['defaultToSpeaker'], }); - const context = new AudioContext({ sampleRate: 24000 }); + const context = new AudioContext({ sampleRate }); audioContextRef.current = context; context.suspend(); @@ -114,7 +193,7 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { audioContextRef.current = null; gainNodeRef.current = null; }; - }, []); + }, [sampleRate]); useEffect(() => { setReadyToGenerate(!model.isGenerating && model.isReady && !isPlaying); @@ -141,7 +220,7 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { const audioBuffer = createAudioBufferFromVector( audioVec, audioContext, - 24000 + sampleRate ); const source = (sourceRef.current = @@ -167,8 +246,10 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { await model.stream({ text: inputText, - speed: 0.9, - phonemize: true, + speed: 1.0, + ...(selectedTtsModel === 'supertonic' + ? { totalSteps, lang: selectedLang } + : {}), onNext, onEnd, }); @@ -209,14 +290,42 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { setError(null)} /> + + setSelectedSpeaker(m)} /> + {selectedTtsModel === 'supertonic' && ( + <> + setSelectedLang(l)} + /> + + setTotalSteps(s)} + /> + + )} + Enter text to synthesize { + const waveform = await model.forward({ + text, + totalSteps: 8, + lang: 'en', + }); + + const audioBuffer = audioContext.createBuffer(1, waveform.length, 44100); + audioBuffer.getChannelData(0).set(waveform); + + const source = audioContext.createBufferSource(); + source.buffer = audioBuffer; + source.connect(audioContext.destination); + source.start(); +}; +``` + +### Kokoro + ```typescript import { models, useTextToSpeech } from 'react-native-executorch'; import { AudioContext } from 'react-native-audio-api'; @@ -58,9 +91,10 @@ const handleSpeech = async (text: string) => { `useTextToSpeech` takes [`TextToSpeechModelConfig`](../../06-api-reference/interfaces/TextToSpeechModelConfig.md) that consists of: -- `model` of type [`TextToSpeechModelSources`](../../06-api-reference/type-aliases/TextToSpeechModelSources.md) containing the [`durationPredictorSource`](../../06-api-reference/type-aliases/TextToSpeechModelSources.md#durationpredictorsource), [`synthesizerSource`](../../06-api-reference/type-aliases/TextToSpeechModelSources.md#synthesizersource), and [`modelName`](../../06-api-reference/type-aliases/TextToSpeechModelSources.md#modelname). -- [`voiceSource`](../../06-api-reference/interfaces/TextToSpeechModelConfig.md#voicesource) of type [`ResourceSource`](../../06-api-reference/type-aliases/ResourceSource.md) - configuration of specific voice used in TTS. -- [`phonemizerConfig`](../../06-api-reference/interfaces/TextToSpeechModelConfig.md#phonemizerconfig) of type [`TextToSpeechPhonemizerConfig`](../../06-api-reference/interfaces/TextToSpeechPhonemizerConfig.md) - configuration of the phonemizer. +- `model` of type [`TextToSpeechModelSources`](../../06-api-reference/type-aliases/TextToSpeechModelSources.md) — model configuration. +- [`voiceSource`](../../06-api-reference/interfaces/TextToSpeechModelConfig.md#voicesource) of type [`ResourceSource`](../../06-api-reference/type-aliases/ResourceSource.md) — the voice tensor used for synthesis. +- [`phonemizerConfig`](../../06-api-reference/interfaces/TextToSpeechModelConfig.md#phonemizerconfig) of type [`TextToSpeechPhonemizerConfig`](../../06-api-reference/interfaces/TextToSpeechPhonemizerConfig.md) — Kokoro only: phonemizer configuration. Unused by Supertonic. +- [`lang`](../../06-api-reference/interfaces/TextToSpeechModelConfig.md#lang) of type [`TextToSpeechSupertonicLanguage`](../../06-api-reference/type-aliases/TextToSpeechSupertonicLanguage.md) — Supertonic only: default language token (e.g. `'en'`, `'na'`). Unused by Kokoro. `useTextToSpeech`'s second optional argument is an object with: @@ -79,21 +113,28 @@ You need more details? Check the following resources: ## Running the model -The module provides two ways to generate speech using either raw text or pre-generated phonemes: +The module provides two ways to generate speech. The available parameters differ by model family: + +| Parameter | Kokoro | Supertonic | +| ------------ | ------ | ---------- | +| `speed` | ✓ | | +| `phonemize` | ✓ | | +| `totalSteps` | | ✓ | +| `lang` | | ✓ | ### Using Text -1. [**`forward({ text, speed, phonemize })`**](../../06-api-reference/interfaces/TextToSpeechType.md#forward): Generates the complete audio waveform at once. Returns a promise resolving to a `Float32Array`. -2. [**`stream({ speed, phonemize, stopAutomatically, onNext, ... })`**](../../06-api-reference/interfaces/TextToSpeechType.md#stream): An async generator-like functionality (managed via callbacks like `onNext`) that yields chunks of audio as they are computed. +1. [**`forward({ text, speed, phonemize, totalSteps, lang })`**](../../06-api-reference/interfaces/TextToSpeechType.md#forward): Generates the complete audio waveform at once. Returns a promise resolving to a `Float32Array`. +2. [**`stream({ speed, phonemize, totalSteps, lang, stopAutomatically, onNext, ... })`**](../../06-api-reference/interfaces/TextToSpeechType.md#stream): An async generator-like functionality (managed via callbacks like `onNext`) that yields chunks of audio as they are computed. This is ideal for reducing the "time to first audio" for long sentences. You can also dynamically insert text during the generation process using `streamInsert(text)`, force-partition trailing content without an end-of-sentence character via `streamFlush()`, and stop the stream with `streamStop(instant)`. :::tip Recommendation In most cases, the **`stream()`** method is recommended over `forward()`. It significantly reduces latency by allowing audio playback to begin as soon as the first chunk is synthesized, rather than waiting for the entire text to be processed. ::: -Both methods accept a `phonemize` parameter (defaults to `true`). When set to `true`, the input `text` is treated as raw text and converted to phonemes internally. When set to `false`, the input is expected to be a string of IPA phonemes. +Both methods accept a `phonemize` parameter (defaults to `true`). This applies to **Kokoro only** — Supertonic maps text directly through a unicode indexer and does not use phonemization. When set to `true`, the input `text` is treated as raw text and converted to phonemes internally. When set to `false`, the input is expected to be a string of IPA phonemes. -### Using Phonemes +### Using Phonemes (Kokoro only) If you have pre-computed phonemes (e.g., from an external dictionary or a custom G2P model), you can skip the internal phoneme generation step: @@ -106,7 +147,7 @@ Since `forward` and `stream` process the input, they might take a significant am ## Example -### Speech Synthesis +### Raw Synthesis (forward) ```tsx import React from 'react'; @@ -115,16 +156,24 @@ import { models, useTextToSpeech } from 'react-native-executorch'; import { AudioContext } from 'react-native-audio-api'; export default function App() { - const tts = useTextToSpeech(models.text_to_speech.kokoro.en_us.heart()); + // Supertonic 3 — multilingual, any voice works for any language + const tts = useTextToSpeech(models.text_to_speech.supertonic.m1()); + // Kokoro — language-specific voice bundle: + // const tts = useTextToSpeech(models.text_to_speech.kokoro.en_us.heart()); const generateAudio = async () => { const audioData = await tts.forward({ text: 'Hello world! This is a sample text.', + totalSteps: 8, + lang: 'en', + // Kokoro: text only (no totalSteps/lang): + // text: 'Hello world! This is a sample text.', }); - // Playback example - const ctx = new AudioContext({ sampleRate: 24000 }); - const buffer = ctx.createBuffer(1, audioData.length, 24000); + // Playback — sample rate depends on the model + const ctx = new AudioContext({ sampleRate: 44100 }); + // Kokoro: sampleRate: 24000 + const buffer = ctx.createBuffer(1, audioData.length, ctx.sampleRate); buffer.getChannelData(0).set(audioData); const source = ctx.createBufferSource(); @@ -150,18 +199,25 @@ import { models, useTextToSpeech } from 'react-native-executorch'; import { AudioContext } from 'react-native-audio-api'; export default function App() { - const tts = useTextToSpeech(models.text_to_speech.kokoro.en_us.heart()); + // Supertonic 3 + const tts = useTextToSpeech(models.text_to_speech.supertonic.m1()); + // Kokoro: + // const tts = useTextToSpeech(models.text_to_speech.kokoro.en_us.heart()); - const contextRef = useRef(new AudioContext({ sampleRate: 24000 })); + const contextRef = useRef(new AudioContext({ sampleRate: 44100 })); + // Kokoro: sampleRate: 24000 const generateStream = async () => { const ctx = contextRef.current; await tts.stream({ text: "This is a longer text, which is being streamed chunk by chunk. Let's see how it works!", + totalSteps: 8, + lang: 'en', + // Kokoro: use speed: 1.0 instead of totalSteps/lang onNext: async (chunk) => { return new Promise((resolve) => { - const buffer = ctx.createBuffer(1, chunk.length, 24000); + const buffer = ctx.createBuffer(1, chunk.length, ctx.sampleRate); buffer.getChannelData(0).set(chunk); const source = ctx.createBufferSource(); @@ -182,42 +238,9 @@ export default function App() { } ``` -### Synthesis from Phonemes - -If you already have a phoneme string obtained from an external source (e.g. the Python `phonemizer` library, -`espeak-ng`, or any custom phonemizer), you can use `forward` or `stream` with the `phonemize: false` flag to synthesize audio directly, skipping the phoneme generation stage. - -```tsx -import React from 'react'; -import { Button, View } from 'react-native'; -import { models, useTextToSpeech } from 'react-native-executorch'; -export default function App() { - const tts = useTextToSpeech(models.text_to_speech.kokoro.en_us.heart()); - - const synthesizePhonemes = async () => { - // Example phonemes for "Hello" - const audioData = await tts.forward({ - text: 'ɐ mˈæn hˌu dˈʌzᵊnt tɹˈʌst hɪmsˈɛlf, kæn nˈɛvəɹ ɹˈiᵊli tɹˈʌst ˈɛniwˌʌn ˈɛls.', - phonemize: false, - }); - - // ... process or play audioData ... - }; - - return ( - -