diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a1db41bd4..70bd34c5d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,6 +85,7 @@ jobs: pip install -r backend/requirements.txt pip install --no-deps chatterbox-tts pip install --no-deps hume-tada + pip install --no-deps "voxcpm>=2.0.3,<2.1" pip install --no-deps omnivoice - name: Install MLX dependencies (Apple Silicon only) @@ -294,6 +295,7 @@ jobs: pip install -r backend/requirements.txt pip install --no-deps chatterbox-tts pip install --no-deps hume-tada + pip install --no-deps "voxcpm>=2.0.3,<2.1" pip install --no-deps omnivoice - name: Install PyTorch with CUDA 12.8 @@ -457,6 +459,7 @@ jobs: pip install -r backend/requirements.txt pip install --no-deps chatterbox-tts pip install --no-deps hume-tada + pip install --no-deps "voxcpm>=2.0.3,<2.1" pip install --no-deps omnivoice - name: Build ROCm server binary (onedir) diff --git a/Dockerfile b/Dockerfile index 4451dcf62..f7af519e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,6 +63,7 @@ RUN if [ "$PYTORCH_VARIANT" = "rocm" ]; then \ RUN pip install --no-cache-dir --prefix=/install -r requirements.txt RUN pip install --no-cache-dir --prefix=/install --no-deps chatterbox-tts RUN pip install --no-cache-dir --prefix=/install --no-deps hume-tada +RUN pip install --no-cache-dir --prefix=/install --no-deps "voxcpm>=2.0.3,<2.1" RUN pip install --no-cache-dir --prefix=/install --no-deps omnivoice RUN pip install --no-cache-dir --prefix=/install \ git+https://github.com/QwenLM/Qwen3-TTS.git diff --git a/README.md b/README.md index b4db9a4f6..98e789075 100644 --- a/README.md +++ b/README.md @@ -87,16 +87,16 @@ Voicebox is built and maintained with [Capy](https://capy.ai?utm_source=voicebox ## What is Voicebox? -Voicebox is a **local-first AI voice studio** — a free and open-source alternative to **ElevenLabs** and **WisprFlow** in one app. Clone voices from a few seconds of audio, generate speech in 23 languages across 8 TTS engines, dictate into any text field with a global hotkey, and give any MCP-aware AI agent a voice of your choosing. +Voicebox is a **local-first AI voice studio** — a free and open-source alternative to **ElevenLabs** and **WisprFlow** in one app. Clone voices from a few seconds of audio, generate speech in 30 languages across 10 TTS engines, dictate into any text field with a global hotkey, and give any MCP-aware AI agent a voice of your choosing. The two cloud incumbents sit on opposite halves of the voice I/O loop — ElevenLabs on output, WisprFlow on input. Voicebox does both, bridges them with a bundled local LLM for refinement and per-profile personas, and runs the whole thing on your machine. - **Complete privacy** — models, voice data, and captures never leave your machine -- **8 TTS engines** — Qwen3-TTS, Qwen CustomVoice, Qwen VoiceDesign, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, and Kokoro +- **10 TTS engines** — Qwen3-TTS, Qwen CustomVoice, Qwen VoiceDesign, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro, OmniVoice (non-commercial), and VoxCPM2 - **Voice cloning and preset voices** — zero-shot cloning from a reference sample, or 50+ curated preset voices via Kokoro and Qwen CustomVoice -- **23 languages** — from English to Arabic, Japanese, Hindi, Swahili, and more +- **30 languages** — from English to Arabic, Japanese, Hindi, Thai, Vietnamese, and more - **Post-processing effects** — pitch shift, reverb, delay, chorus, compression, and filters -- **Expressive speech** — paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice; voices described in plain language via Qwen VoiceDesign +- **Expressive speech** — paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice and VoxCPM2; voices described in plain language via Qwen VoiceDesign - **Unlimited length** — auto-chunking with crossfade for scripts, articles, and chapters - **Stories editor** — multi-track timeline for conversations, podcasts, and narratives - **Voice input** — global dictation hotkey with push-to-talk and toggle modes, accessibility-verified auto-paste on macOS, in-app mic on every text field, Whisper-based STT @@ -129,7 +129,7 @@ The two cloud incumbents sit on opposite halves of the voice I/O loop — Eleven ### Multi-Engine Voice Cloning -Eight TTS engines with different strengths, switchable per-generation: +Ten TTS engines with different strengths, switchable per-generation: | Engine | Languages | Strengths | | --------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------- | @@ -141,11 +141,13 @@ Eight TTS engines with different strengths, switchable per-generation: | **Chatterbox Turbo** | English | Fast 350M model with paralinguistic emotion/sound tags | | **TADA** (1B / 3B) | 10 | HumeAI speech-language model — 700s+ coherent audio, text-acoustic dual alignment | | **Kokoro** | 8 | 50 curated preset voices, tiny 82M model, fast CPU inference | +| **VoxCPM2** | 30 | 2B multilingual model, 48kHz output, high-fidelity reference-audio cloning; CUDA or CPU fallback | +| **OmniVoice** | 23 exposed | Multilingual cloning and attribute instructions; CC-BY-NC weights, non-commercial use only | ### Emotions & Paralinguistic Tags Only **Chatterbox Turbo** interprets paralinguistic tags like `[laugh]` and -`[sigh]`. Qwen3-TTS, LuxTTS, Chatterbox Multilingual, and HumeAI TADA read them +`[sigh]`. Qwen3-TTS, LuxTTS, Chatterbox Multilingual, HumeAI TADA, and VoxCPM2 read them literally as text. With **Chatterbox Turbo** selected, type `/` in the text input to open the tag @@ -393,7 +395,7 @@ Full API documentation available at `http://127.0.0.1:17493/docs`. | Frontend | React, TypeScript, Tailwind CSS | | State | Zustand, React Query | | Backend | FastAPI (Python) | -| TTS Engines | Qwen3-TTS, Qwen CustomVoice, Qwen VoiceDesign, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro | +| TTS Engines | Qwen3-TTS, Qwen CustomVoice, Qwen VoiceDesign, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro, OmniVoice (non-commercial), VoxCPM2 | | STT | Whisper / Whisper Turbo (PyTorch or MLX) | | Local LLM | Qwen3 (0.6B / 1.7B / 4B), shared runtime with TTS / STT | | MCP Server | FastMCP mounted at `/mcp` (Streamable HTTP) + bundled stdio shim binary | diff --git a/app/src/components/CapturesTab/CapturesTab.tsx b/app/src/components/CapturesTab/CapturesTab.tsx index 401b8a3c7..81f55abaf 100644 --- a/app/src/components/CapturesTab/CapturesTab.tsx +++ b/app/src/components/CapturesTab/CapturesTab.tsx @@ -269,7 +269,7 @@ export function CapturesTab() { // override fall through to whatever the backend picks. const engine = voice.default_engine as | 'qwen' | 'qwen_custom_voice' | 'qwen_voice_design' | 'luxtts' | 'chatterbox' - | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'omnivoice' + | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'omnivoice' | 'voxcpm2' | undefined; return apiClient.generateSpeech({ profile_id: voice.id, diff --git a/app/src/components/Generation/EngineModelSelector.tsx b/app/src/components/Generation/EngineModelSelector.tsx index 9d8b04686..db480282c 100644 --- a/app/src/components/Generation/EngineModelSelector.tsx +++ b/app/src/components/Generation/EngineModelSelector.tsx @@ -29,6 +29,7 @@ const ENGINE_OPTIONS = [ { value: 'tada:3B', label: 'TADA 3B Multilingual', engine: 'tada' }, { value: 'kokoro', label: 'Kokoro 82M', engine: 'kokoro' }, { value: 'omnivoice', label: 'OmniVoice (non-commercial)', engine: 'omnivoice' }, + { value: 'voxcpm2', label: 'VoxCPM2', engine: 'voxcpm2' }, ] as const; const ENGINE_DESCRIPTIONS: Record = { @@ -41,6 +42,7 @@ const ENGINE_DESCRIPTIONS: Record = { tada: 'HumeAI, 700s+ coherent audio', kokoro: '82M params, CPU realtime, 8 langs', omnivoice: '23 languages, voice design. CC-BY-NC weights: non-commercial use only', + voxcpm2: '2B params, 48kHz, multilingual cloning', }; /** Engines that only support English and should force language to 'en' on select. */ @@ -54,6 +56,7 @@ const CLONING_ENGINES = new Set([ 'chatterbox_turbo', 'tada', 'omnivoice', + 'voxcpm2', ]); /** Engines that synthesise a voice from a written description. */ diff --git a/app/src/components/Generation/FloatingGenerateBox.tsx b/app/src/components/Generation/FloatingGenerateBox.tsx index b04673c60..e3fc903dc 100644 --- a/app/src/components/Generation/FloatingGenerateBox.tsx +++ b/app/src/components/Generation/FloatingGenerateBox.tsx @@ -154,7 +154,8 @@ export function FloatingGenerateBox({ | 'kokoro' | 'qwen_custom_voice' | 'qwen_voice_design' - | 'omnivoice'; + | 'omnivoice' + | 'voxcpm2'; useEffect(() => { if (selectedProfile?.language) { form.setValue('language', selectedProfile.language as LanguageCode); diff --git a/app/src/components/ServerSettings/ModelManagement.tsx b/app/src/components/ServerSettings/ModelManagement.tsx index 5f14157bd..4b953460d 100644 --- a/app/src/components/ServerSettings/ModelManagement.tsx +++ b/app/src/components/ServerSettings/ModelManagement.tsx @@ -71,6 +71,8 @@ const MODEL_DESCRIPTIONS: Record = { 'Kokoro 82M by hexgrad. Tiny 82M-parameter TTS that runs at CPU realtime. Supports 8 languages with pre-built voice styles. Apache 2.0 licensed.', omnivoice: 'OmniVoice by k2-fsa. Zero-shot voice cloning via a diffusion language model, plus attribute-based voice design (gender, age, pitch, accent). The model covers 600+ languages; Voicebox exposes the 23 in its own language list. 24kHz output. NON-COMMERCIAL: the weights are CC-BY-NC (code is Apache 2.0; the bundled Higgs Audio V2 codec is under the Boson community licence), so do not use output commercially.', + voxcpm2: + 'VoxCPM2 by OpenBMB. A 2B-parameter multilingual TTS model with 48kHz output and high-fidelity reference-audio voice cloning.', 'qwen-custom-voice-1.7B': 'Qwen3-TTS CustomVoice 1.7B by Alibaba. 9 premium preset voices with instruct-based style control for tone, emotion, and prosody. Supports 10 languages.', 'qwen-custom-voice-0.6B': @@ -420,7 +422,8 @@ export function ModelManagement() { m.model_name.startsWith('chatterbox') || m.model_name.startsWith('tada') || m.model_name.startsWith('kokoro') || - m.model_name.startsWith('omnivoice'), + m.model_name.startsWith('omnivoice') || + m.model_name.startsWith('voxcpm2'), ) ?? []; const whisperModels = modelStatus?.models.filter((m) => m.model_name.startsWith('whisper')) ?? []; const llmModels = modelStatus?.models.filter((m) => m.model_name.startsWith('qwen3-')) ?? []; diff --git a/app/src/components/VoiceProfiles/ProfileForm.tsx b/app/src/components/VoiceProfiles/ProfileForm.tsx index 4099caffa..87808d070 100644 --- a/app/src/components/VoiceProfiles/ProfileForm.tsx +++ b/app/src/components/VoiceProfiles/ProfileForm.tsx @@ -76,6 +76,7 @@ const DEFAULT_ENGINE_OPTIONS = [ { value: 'tada', label: 'TADA' }, { value: 'kokoro', label: 'Kokoro 82M' }, { value: 'omnivoice', label: 'OmniVoice (non-commercial)' }, + { value: 'voxcpm2', label: 'VoxCPM2' }, ] as const; function makeProfileSchema(t: (key: string) => string) { diff --git a/app/src/i18n/locales/en/translation.json b/app/src/i18n/locales/en/translation.json index 8ab4dec94..7cfbf6b6c 100644 --- a/app/src/i18n/locales/en/translation.json +++ b/app/src/i18n/locales/en/translation.json @@ -890,7 +890,7 @@ "body": "A few seconds of reference audio is enough. Multi-sample support for higher quality when you want it." }, "engines": { - "title": "Seven engines, 23 languages.", + "title": "Ten engines, 30 languages.", "body": "Pick the tradeoff that fits — quality, speed, or multilingual coverage." }, "agentReady": { diff --git a/app/src/i18n/locales/es/translation.json b/app/src/i18n/locales/es/translation.json index 286590842..876605522 100644 --- a/app/src/i18n/locales/es/translation.json +++ b/app/src/i18n/locales/es/translation.json @@ -889,7 +889,7 @@ "body": "Bastan unos segundos de audio de referencia. Compatible con varias muestras para mayor calidad cuando lo necesites." }, "engines": { - "title": "Siete motores, 23 idiomas.", + "title": "Diez motores, 30 idiomas.", "body": "Elige el equilibrio que encaje: calidad, velocidad o cobertura multilingüe." }, "agentReady": { diff --git a/app/src/i18n/locales/fr/translation.json b/app/src/i18n/locales/fr/translation.json index 7717d2c85..9d2272e89 100644 --- a/app/src/i18n/locales/fr/translation.json +++ b/app/src/i18n/locales/fr/translation.json @@ -889,7 +889,7 @@ "body": "Quelques secondes d'audio de référence suffisent. Prise en charge multi-échantillons pour une qualité supérieure quand vous le souhaitez." }, "engines": { - "title": "Sept moteurs, 23 langues.", + "title": "Dix moteurs, 30 langues.", "body": "Choisissez le compromis qui vous convient — qualité, vitesse ou couverture multilingue." }, "agentReady": { diff --git a/app/src/i18n/locales/it/translation.json b/app/src/i18n/locales/it/translation.json index fdd806aea..4355c1388 100644 --- a/app/src/i18n/locales/it/translation.json +++ b/app/src/i18n/locales/it/translation.json @@ -889,7 +889,7 @@ "body": "Bastano pochi secondi di audio di riferimento. Supporto multi-campione per una qualità superiore quando ne hai bisogno." }, "engines": { - "title": "Sette motori, 23 lingue.", + "title": "Dieci motori, 30 lingue.", "body": "Scegli il compromesso più adatto alle tue esigenze tra qualità, velocità o copertura multilingue." }, "agentReady": { diff --git a/app/src/i18n/locales/ja/translation.json b/app/src/i18n/locales/ja/translation.json index 782a4b307..7501560b6 100644 --- a/app/src/i18n/locales/ja/translation.json +++ b/app/src/i18n/locales/ja/translation.json @@ -889,7 +889,7 @@ "body": "数秒のリファレンス音声があれば十分です。より高い品質を求めるときは複数サンプルにも対応します。" }, "engines": { - "title": "7 つのエンジン、23 言語。", + "title": "10 のエンジン、30 言語。", "body": "品質、速度、多言語対応 — 用途に合ったトレードオフを選べます。" }, "agentReady": { diff --git a/app/src/i18n/locales/ko/translation.json b/app/src/i18n/locales/ko/translation.json index 45165e08c..04c168d7c 100644 --- a/app/src/i18n/locales/ko/translation.json +++ b/app/src/i18n/locales/ko/translation.json @@ -889,7 +889,7 @@ "body": "몇 초의 참조 오디오면 충분합니다. 원할 경우 여러 샘플로 더 높은 품질 지원." }, "engines": { - "title": "7개 엔진, 23개 언어.", + "title": "10개 엔진, 30개 언어.", "body": "품질, 속도, 다국어 지원 중 원하는 트레이드오프를 선택하세요." }, "agentReady": { diff --git a/app/src/i18n/locales/pt-BR/translation.json b/app/src/i18n/locales/pt-BR/translation.json index df53072c7..48a5328d6 100644 --- a/app/src/i18n/locales/pt-BR/translation.json +++ b/app/src/i18n/locales/pt-BR/translation.json @@ -889,7 +889,7 @@ "body": "Alguns segundos de áudio de referência já bastam. Suporte a múltiplas amostras para maior qualidade quando você quiser." }, "engines": { - "title": "Sete motores, 23 idiomas.", + "title": "Dez motores, 30 idiomas.", "body": "Escolha o equilíbrio ideal — qualidade, velocidade ou cobertura multilíngue." }, "agentReady": { diff --git a/app/src/i18n/locales/zh-CN/translation.json b/app/src/i18n/locales/zh-CN/translation.json index 83ae9f440..c218c36a4 100644 --- a/app/src/i18n/locales/zh-CN/translation.json +++ b/app/src/i18n/locales/zh-CN/translation.json @@ -889,7 +889,7 @@ "body": "几秒钟的参考音频就足够了。需要更高质量时,支持多样本克隆。" }, "engines": { - "title": "七种引擎,23 种语言。", + "title": "十种引擎,30 种语言。", "body": "选择最合适的取舍——质量、速度,或多语言覆盖。" }, "agentReady": { diff --git a/app/src/i18n/locales/zh-TW/translation.json b/app/src/i18n/locales/zh-TW/translation.json index 81986a87f..5566b0b69 100644 --- a/app/src/i18n/locales/zh-TW/translation.json +++ b/app/src/i18n/locales/zh-TW/translation.json @@ -889,7 +889,7 @@ "body": "幾秒鐘的參考音訊就夠了。需要更高品質時也支援多樣本。" }, "engines": { - "title": "七種引擎、23 種語言。", + "title": "十種引擎、30 種語言。", "body": "選擇最符合需求的取捨——品質、速度,或多語言覆蓋。" }, "agentReady": { diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index 2de56a154..db92485aa 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -80,7 +80,8 @@ export interface GenerationRequest { | 'chatterbox_turbo' | 'tada' | 'kokoro' - | 'omnivoice'; + | 'omnivoice' + | 'voxcpm2'; instruct?: string; /** When true and the profile has a personality prompt, input text is rewritten in-character before TTS. */ personality?: boolean; diff --git a/app/src/lib/constants/engines.ts b/app/src/lib/constants/engines.ts index 6cda56b01..bcc9a9669 100644 --- a/app/src/lib/constants/engines.ts +++ b/app/src/lib/constants/engines.ts @@ -9,12 +9,14 @@ * VoiceDesign take free-form prose ("speak in an angry tone"; VoiceDesign * layers it on top of the profile's stored design prompt), while OmniVoice * takes a closed vocabulary of attributes ("female, low pitch, british - * accent") and raises on anything outside it. + * accent") and raises on anything outside it. VoxCPM2 maps free-form prose + * to a parenthesized delivery prefix in the backend. */ export const INSTRUCT_ENGINES: ReadonlySet = new Set([ 'qwen_custom_voice', 'qwen_voice_design', 'omnivoice', + 'voxcpm2', ]); /** Whether the given engine should show and forward the instruct field. */ diff --git a/app/src/lib/constants/languages.ts b/app/src/lib/constants/languages.ts index c075892ec..a666fcdc1 100644 --- a/app/src/lib/constants/languages.ts +++ b/app/src/lib/constants/languages.ts @@ -6,6 +6,7 @@ * Chatterbox Multilingual supports 23 languages. * Chatterbox Turbo is English-only. * Kokoro supports 8 languages. + * VoxCPM2 supports 30 languages. */ /** All languages that any engine supports. */ @@ -20,10 +21,14 @@ export const ALL_LANGUAGES = { fr: 'French', he: 'Hebrew', hi: 'Hindi', + id: 'Indonesian', it: 'Italian', ja: 'Japanese', + km: 'Khmer', ko: 'Korean', + lo: 'Lao', ms: 'Malay', + my: 'Burmese', nl: 'Dutch', no: 'Norwegian', pl: 'Polish', @@ -31,7 +36,10 @@ export const ALL_LANGUAGES = { ru: 'Russian', sv: 'Swedish', sw: 'Swahili', + th: 'Thai', + tl: 'Tagalog', tr: 'Turkish', + vi: 'Vietnamese', zh: 'Chinese', } as const; @@ -71,8 +79,7 @@ export const ENGINE_LANGUAGES: Record = { kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'], qwen_custom_voice: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'], qwen_voice_design: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'], - // OmniVoice covers 600+ languages. Listed here are the ones ALL_LANGUAGES - // already knows -- widening that list is a separate change. + // Keep OmniVoice's validated 23-language subset; VoxCPM2 adds seven more below. omnivoice: [ 'ar', 'da', @@ -98,6 +105,38 @@ export const ENGINE_LANGUAGES: Record = { 'tr', 'zh', ], + voxcpm2: [ + 'ar', + 'da', + 'de', + 'el', + 'en', + 'es', + 'fi', + 'fr', + 'he', + 'hi', + 'id', + 'it', + 'ja', + 'km', + 'ko', + 'lo', + 'ms', + 'my', + 'nl', + 'no', + 'pl', + 'pt', + 'ru', + 'sv', + 'sw', + 'th', + 'tl', + 'tr', + 'vi', + 'zh', + ], } as const; /** Helper: get language options for a given engine. */ diff --git a/app/src/lib/hooks/useGenerationForm.ts b/app/src/lib/hooks/useGenerationForm.ts index 551a2d06a..a1c0f4219 100644 --- a/app/src/lib/hooks/useGenerationForm.ts +++ b/app/src/lib/hooks/useGenerationForm.ts @@ -30,6 +30,7 @@ const generationSchema = z.object({ 'tada', 'kokoro', 'omnivoice', + 'voxcpm2', ]) .optional(), personality: z.boolean().optional(), @@ -107,9 +108,11 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { ? 'qwen-voice-design-1.7B' : engine === 'omnivoice' ? 'omnivoice' - : engine === 'qwen_custom_voice' - ? `qwen-custom-voice-${data.modelSize}` - : `qwen-tts-${data.modelSize}`; + : engine === 'voxcpm2' + ? 'voxcpm2' + : engine === 'qwen_custom_voice' + ? `qwen-custom-voice-${data.modelSize}` + : `qwen-tts-${data.modelSize}`; const displayName = engine === 'luxtts' ? 'LuxTTS' @@ -127,13 +130,15 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { ? 'Qwen VoiceDesign 1.7B' : engine === 'omnivoice' ? 'OmniVoice' - : engine === 'qwen_custom_voice' - ? data.modelSize === '1.7B' - ? 'Qwen CustomVoice 1.7B' - : 'Qwen CustomVoice 0.6B' - : data.modelSize === '1.7B' - ? 'Qwen TTS 1.7B' - : 'Qwen TTS 0.6B'; + : engine === 'voxcpm2' + ? 'VoxCPM2' + : engine === 'qwen_custom_voice' + ? data.modelSize === '1.7B' + ? 'Qwen CustomVoice 1.7B' + : 'Qwen CustomVoice 0.6B' + : data.modelSize === '1.7B' + ? 'Qwen TTS 1.7B' + : 'Qwen TTS 0.6B'; // Check if model needs downloading try { diff --git a/app/src/lib/utils/format.ts b/app/src/lib/utils/format.ts index 6719c20c5..ed114c16c 100644 --- a/app/src/lib/utils/format.ts +++ b/app/src/lib/utils/format.ts @@ -64,6 +64,7 @@ const ENGINE_DISPLAY_NAMES: Record = { chatterbox_turbo: 'Chatterbox Turbo', tada: 'TADA', kokoro: 'Kokoro 82M', + voxcpm2: 'VoxCPM2', }; export function formatEngineName(engine?: string, modelSize?: string): string { diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index fb80ed0e7..7f0c3d5be 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -239,6 +239,7 @@ def is_loaded(self) -> bool: "tada": "TADA", "kokoro": "Kokoro", "omnivoice": "OmniVoice", + "voxcpm2": "VoxCPM2", } LLM_ENGINES = { @@ -430,6 +431,19 @@ def _get_non_qwen_tts_configs() -> list[ModelConfig]: size_mb=350, languages=["en", "es", "fr", "hi", "it", "pt", "ja", "zh"], ), + ModelConfig( + model_name="voxcpm2", + display_name="VoxCPM2", + engine="voxcpm2", + hf_repo_id="openbmb/VoxCPM2", + size_mb=4800, + supports_instruct=True, + languages=[ + "zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it", + "he", "ar", "da", "el", "fi", "hi", "id", "km", "lo", "ms", + "my", "nl", "no", "pl", "sv", "sw", "th", "tl", "tr", "vi", + ], + ), ModelConfig( model_name="omnivoice", # Non-commercial: the OmniVoice weights are CC-BY-NC (the code is @@ -441,9 +455,8 @@ def _get_non_qwen_tts_configs() -> list[ModelConfig]: hf_repo_id="k2-fsa/OmniVoice", size_mb=3300, supports_instruct=True, - # OmniVoice covers 600+ languages; these are the ones already in - # Voicebox's language enum. Widening that enum is a separate change - # that touches the API contract and the frontend. + # Retain OmniVoice's validated 23-language subset. VoxCPM2 extends + # the API's language enum separately. languages=[ "zh", "en", @@ -846,6 +859,10 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend: from .qwen_custom_voice_backend import QwenCustomVoiceBackend backend = QwenCustomVoiceBackend() + elif engine == "voxcpm2": + from .voxcpm_backend import VoxCPMBackend + + backend = VoxCPMBackend() elif engine == "qwen_voice_design": from .qwen_voice_design_backend import QwenVoiceDesignBackend diff --git a/backend/backends/voxcpm_backend.py b/backend/backends/voxcpm_backend.py new file mode 100644 index 000000000..b86aa49b0 --- /dev/null +++ b/backend/backends/voxcpm_backend.py @@ -0,0 +1,187 @@ +"""VoxCPM2 TTS backend implementation. + +Wraps the native ``voxcpm`` package for local reference-audio cloning. The +backend deliberately disables VoxCPM's optional denoiser so Voicebox can keep +the dependency surface focused on synthesis; reference audio is already +validated and normalized by the profile pipeline. +""" + +import asyncio +import inspect +import logging +from pathlib import Path + +import numpy as np + +from .base import ( + combine_voice_prompts as _combine_voice_prompts, + empty_device_cache, + get_torch_device, + is_model_cached, + manual_seed, + model_load_progress, +) + +logger = logging.getLogger(__name__) + +VOXCPM_HF_REPO = "openbmb/VoxCPM2" +VOXCPM_MODEL_FILES = ("config.json", "model.safetensors", "audiovae.pth") + + +class VoxCPMBackend: + """Native VoxCPM2 backend for cloned voice profiles.""" + + def __init__(self): + self.model = None + self.model_size = "default" + self._device: str | None = None + self._model_load_lock = asyncio.Lock() + + def _get_device(self) -> str: + # Current VoxCPM2 runs on MPS, but a smoke test showed it substantially + # slower than CPU on Apple Silicon and upstream has had MPS-specific + # AudioVAE failures. Match Voicebox's Chatterbox/TADA policy: use CUDA + # where available and a predictable CPU fallback on macOS. + return get_torch_device(force_cpu_on_mac=True) + + @property + def device(self) -> str: + if self._device is None: + self._device = self._get_device() + return self._device + + def is_loaded(self) -> bool: + return self.model is not None + + def _get_model_path(self, model_size: str = "default") -> str: + return VOXCPM_HF_REPO + + def _is_model_cached(self, model_size: str = "default") -> bool: + return is_model_cached(VOXCPM_HF_REPO, required_files=list(VOXCPM_MODEL_FILES)) + + async def load_model(self, model_size: str = "default") -> None: + """Download and load VoxCPM2 lazily on the selected device.""" + if self.model is not None: + return + async with self._model_load_lock: + if self.model is not None: + return + await asyncio.to_thread(self._load_model_sync) + + async def load_model_async(self, model_size: str = "default") -> None: + """Compatibility alias used by a few model-management call sites.""" + await self.load_model(model_size) + + def _load_model_sync(self) -> None: + model_name = "voxcpm2" + is_cached = self._is_model_cached() + + with model_load_progress(model_name, is_cached): + from voxcpm import VoxCPM + + device = self.device + logger.info("Loading VoxCPM2 on %s...", device) + self.model = VoxCPM.from_pretrained( + VOXCPM_HF_REPO, + load_denoiser=False, + optimize=device == "cuda", + device=device, + ) + + logger.info("VoxCPM2 loaded successfully on %s", self.device) + + def unload_model(self) -> None: + """Unload VoxCPM2 and release accelerator cache.""" + if self.model is None: + return + device = self._device + del self.model + self.model = None + self._device = None + empty_device_cache(device) + logger.info("VoxCPM2 unloaded") + + async def create_voice_prompt( + self, + audio_path: str, + reference_text: str, + use_cache: bool = True, + ) -> tuple[dict, bool]: + """Store reference paths; VoxCPM2 builds its prompt at generation time.""" + del use_cache + return { + "ref_audio": str(audio_path), + "ref_text": str(reference_text or "").strip(), + }, False + + async def combine_voice_prompts( + self, + audio_paths: list[str], + reference_texts: list[str], + ) -> tuple[np.ndarray, str]: + # Voicebox's profile service persists combined prompts at 24 kHz. Keep + # that convention; VoxCPM2 resamples references to its 16 kHz encoder + # rate internally. + return await _combine_voice_prompts(audio_paths, reference_texts, sample_rate=24000) + + async def generate( + self, + text: str, + voice_prompt: dict, + language: str = "en", + seed: int | None = None, + instruct: str | None = None, + ) -> tuple[np.ndarray, int]: + """Generate 48 kHz speech with reference-audio voice cloning.""" + del language + await self.load_model() + + ref_audio = voice_prompt.get("ref_audio") + ref_text = str(voice_prompt.get("ref_text") or "").strip() + if ref_audio and not Path(ref_audio).exists(): + raise FileNotFoundError(f"Reference audio not found: {ref_audio}") + + def _generate_sync(): + if seed is not None: + manual_seed(seed, self.device) + + kwargs = {} + if ref_audio: + # With delivery instructions, use controllable cloning so the + # parenthesized style prefix can take effect. Without an + # instruction, Voicebox's accurate reference transcript enables + # VoxCPM2's highest-fidelity continuation/cloning mode. + kwargs["reference_wav_path"] = str(ref_audio) + if ref_text and not (instruct and instruct.strip()): + kwargs["prompt_wav_path"] = str(ref_audio) + kwargs["prompt_text"] = ref_text + + generation_text = text + if instruct and instruct.strip(): + generation_text = f"({instruct.strip()}){text}" + + generation_kwargs = { + "text": generation_text, + "retry_badcase": True, + **kwargs, + } + # PyPI 2.0.3 predates the explicit seed parameter now available on + # VoxCPM main. Voicebox still seeds torch above, and forwards the + # argument automatically once a newer VoxCPM release exposes it. + try: + generate_parameters = inspect.signature( + getattr(self.model, "_generate", self.model.generate) + ).parameters + except (TypeError, ValueError): + generate_parameters = {} + if seed is not None and "seed" in generate_parameters: + generation_kwargs["seed"] = seed + + wav = self.model.generate(**generation_kwargs) + audio = np.asarray(wav, dtype=np.float32).squeeze() + if audio.ndim != 1 or audio.size == 0: + raise ValueError("VoxCPM2 returned empty audio") + sample_rate = int(getattr(getattr(self.model, "tts_model", None), "sample_rate", 48000)) + return audio, sample_rate + + return await asyncio.to_thread(_generate_sync) diff --git a/backend/build_binary.py b/backend/build_binary.py index 5e88fd05c..40af116a5 100644 --- a/backend/build_binary.py +++ b/backend/build_binary.py @@ -121,6 +121,8 @@ def build_server(cuda=False, rocm=False): "--hidden-import", "backend.backends.qwen_custom_voice_backend", "--hidden-import", + "backend.backends.voxcpm_backend", + "--hidden-import", "backend.backends.qwen_voice_design_backend", "--hidden-import", "backend.utils.audio", @@ -188,6 +190,8 @@ def build_server(cuda=False, rocm=False): "--hidden-import", "qwen_tts", "--hidden-import", + "voxcpm", + "--hidden-import", "qwen_tts.inference", "--hidden-import", "qwen_tts.inference.qwen3_tts_model", @@ -200,6 +204,8 @@ def build_server(cuda=False, rocm=False): "--copy-metadata", "qwen-tts", "--copy-metadata", + "voxcpm", + "--copy-metadata", "requests", "--copy-metadata", "transformers", @@ -217,6 +223,10 @@ def build_server(cuda=False, rocm=False): # modeling_qwen3_tts.py — needs physical .py source files bundled "--collect-all", "qwen_tts", + # VoxCPM2's AudioVAE uses @torch.jit.script, which requires the + # physical Python source files in frozen builds. + "--collect-all", + "voxcpm", # Fix for pkg_resources and jaraco namespace packages "--hidden-import", "pkg_resources.extern", @@ -772,6 +782,8 @@ def build_shim(): "--exclude-module", "kokoro", "--exclude-module", + "voxcpm", + "--exclude-module", "misaki", "--exclude-module", "spacy", @@ -835,4 +847,3 @@ def build_shim(): build_shim() else: build_server(cuda=cli_args.cuda, rocm=cli_args.rocm) - diff --git a/backend/models.py b/backend/models.py index 409a0c964..9d104a9bf 100644 --- a/backend/models.py +++ b/backend/models.py @@ -18,7 +18,7 @@ class VoiceProfileCreate(BaseModel): name: str = Field(..., min_length=1, max_length=100) description: Optional[str] = Field(None, max_length=500) language: str = Field( - default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$" + default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|id|km|lo|ms|my|nl|no|pl|sv|sw|th|tl|tr|vi)$" ) voice_type: Optional[str] = Field(default="cloned", pattern="^(cloned|preset|designed)$") preset_engine: Optional[str] = Field(None, max_length=50) @@ -81,11 +81,11 @@ class GenerationRequest(BaseModel): profile_id: str text: str = Field(..., min_length=1, max_length=50000) - language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$") + language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|id|km|lo|ms|my|nl|no|pl|sv|sw|th|tl|tr|vi)$") seed: Optional[int] = Field(None, ge=0) model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$") instruct: Optional[str] = Field(None, max_length=500) - engine: Optional[str] = Field(default=None, pattern="^(qwen|qwen_custom_voice|qwen_voice_design|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|omnivoice)$") + engine: Optional[str] = Field(default=None, pattern="^(qwen|qwen_custom_voice|qwen_voice_design|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|omnivoice|voxcpm2)$") personality: bool = Field( default=False, description="When true and the profile has a personality prompt, the input text is rewritten in-character before TTS.", @@ -352,7 +352,7 @@ class MCPClientBindingResponse(BaseModel): profile_id: Optional[str] = None default_engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|qwen_voice_design|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|omnivoice)$", + pattern="^(qwen|qwen_custom_voice|qwen_voice_design|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|omnivoice|voxcpm2)$", ) default_personality: bool = False last_seen_at: Optional[datetime] = None @@ -371,7 +371,7 @@ class MCPClientBindingUpsert(BaseModel): profile_id: Optional[str] = None default_engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|qwen_voice_design|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|omnivoice)$", + pattern="^(qwen|qwen_custom_voice|qwen_voice_design|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|omnivoice|voxcpm2)$", ) default_personality: bool = False @@ -390,7 +390,7 @@ class SpeakRequest(BaseModel): ) engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|qwen_voice_design|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|omnivoice)$", + pattern="^(qwen|qwen_custom_voice|qwen_voice_design|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|omnivoice|voxcpm2)$", ) personality: Optional[bool] = Field( None, @@ -398,7 +398,7 @@ class SpeakRequest(BaseModel): ) language: Optional[str] = Field( None, - pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$", + pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|id|km|lo|ms|my|nl|no|pl|sv|sw|th|tl|tr|vi)$", ) diff --git a/backend/requirements.txt b/backend/requirements.txt index f686eb345..6c1c55c24 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -19,6 +19,13 @@ huggingface_hub>=0.20.0 # >=0.1.1 for Qwen3TTSModel.generate_voice_design (the qwen_voice_design engine) qwen-tts>=0.1.1 +# VoxCPM2 runtime dependencies. The voxcpm package itself is installed with +# --no-deps by setup scripts because its full metadata pulls Gradio, training, +# ModelScope, and denoising packages that Voicebox does not use. +einops>=0.8.0 +safetensors>=0.4.0 +tqdm>=4.60.0 + # LuxTTS (voice cloning engine) # piper-phonemize needs custom index (no PyPI wheels) --find-links https://k2-fsa.github.io/icefall/piper_phonemize.html @@ -43,7 +50,7 @@ pyloudnorm # descript-audio-codec is NOT installed — it pulls onnx/tensorboard # via descript-audiotools. A lightweight shim in utils/dac_shim.py # provides the only class TADA uses: Snake1d.) -torchaudio +torchaudio>=2.5.0 # OmniVoice sub-dependencies (omnivoice itself is installed --no-deps # in the setup script because it requires transformers>=5.3, while several diff --git a/backend/services/history.py b/backend/services/history.py index 809a556bf..b69d2d47a 100644 --- a/backend/services/history.py +++ b/backend/services/history.py @@ -117,7 +117,7 @@ async def create_generation( instruct: Natural language instruction used (if any) generation_id: Pre-assigned ID (for async generation flow) status: Generation status (generating, completed, failed) - engine: TTS engine used (qwen, luxtts, chatterbox, chatterbox_turbo) + engine: TTS engine used (qwen, luxtts, chatterbox, chatterbox_turbo, voxcpm2) model_size: Model size variant (1.7B, 0.6B) — only relevant for qwen source: Origin marker stored on the row. ``"manual"`` for regular /generate calls; ``"personality_speak"`` for rows created diff --git a/backend/services/profiles.py b/backend/services/profiles.py index 22bdf0092..684dd37d1 100644 --- a/backend/services/profiles.py +++ b/backend/services/profiles.py @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) -CLONING_ENGINES = {"qwen", "luxtts", "chatterbox", "chatterbox_turbo", "tada", "omnivoice"} +CLONING_ENGINES = {"qwen", "luxtts", "chatterbox", "chatterbox_turbo", "tada", "omnivoice", "voxcpm2"} # Engines that synthesise a voice from a natural-language description # (voice_type "designed") rather than from samples or a preset id. diff --git a/backend/tests/E2E_MODEL_TEST_DESIGN.md b/backend/tests/E2E_MODEL_TEST_DESIGN.md index 0e6b49fb2..bc6d17f2f 100644 --- a/backend/tests/E2E_MODEL_TEST_DESIGN.md +++ b/backend/tests/E2E_MODEL_TEST_DESIGN.md @@ -4,7 +4,7 @@ A single script, runnable on macOS and Windows, that exercises every TTS model against the **frozen PyInstaller binary** (not the dev server), captures per-model pass/fail and error messages, and exits non-zero if any model fails. Generation is strictly sequential — one model loaded at a time. -## Test matrix (10 runs) +## Test matrix (12 runs) Derived from `backend/backends/__init__.py:185-316`. Each row maps to one `POST /generate` call. @@ -20,8 +20,10 @@ Derived from `backend/backends/__init__.py:185-316`. Each row maps to one `POST | 8 | `tada` | `1B` | cloned | tada-1b, English only | | 9 | `tada` | `3B` | cloned | tada-3b-ml, multilingual | | 10| `kokoro` | — | preset | `preset_voice_id="af_heart"` | +| 11| `voxcpm2` | — | cloned | shared reference WAV + transcript | +| 12| `qwen_voice_design` | — | designed | single 1.7B checkpoint, stored design prompt | -Cloned engines (1, 2, 5, 6, 7, 8, 9) share **one** profile created once with the reference WAV. Preset profiles are created separately, one for kokoro and one for qwen_custom_voice. +Cloned engines (1, 2, 5, 6, 7, 8, 9, 11) share **one** profile created once with the reference WAV. Preset profiles are created separately, one for kokoro and one for qwen_custom_voice. VoiceDesign uses a separate designed profile. OmniVoice is not yet included in this frozen-binary matrix. Language for every run: `en` (covers every engine's supported set). diff --git a/backend/tests/fixtures/README.md b/backend/tests/fixtures/README.md index 341b98acb..32b3b363b 100644 --- a/backend/tests/fixtures/README.md +++ b/backend/tests/fixtures/README.md @@ -5,7 +5,7 @@ Place two files here before running `test_all_models_e2e.py`: - `reference_voice.wav` — a clean speech sample, mono, 16–24 kHz, ~5–15 seconds. - `reference_voice.txt` — the **exact** transcription of the WAV (single line, no trailing newline required). -These are used to create a cloned voice profile for every cloning-capable engine (qwen, luxtts, chatterbox, chatterbox_turbo, tada). Keep them out of version control if they contain personal audio — this directory is not gitignored by default, so add them to `.gitignore` locally if needed. +These are used to create a cloned voice profile for every cloning-capable engine (qwen, luxtts, chatterbox, chatterbox_turbo, tada, voxcpm2). Keep them out of version control if they contain personal audio — this directory is not gitignored by default, so add them to `.gitignore` locally if needed. You can point the test at different files with: diff --git a/backend/tests/test_all_models_e2e.py b/backend/tests/test_all_models_e2e.py index 1d9b69bdf..c91f3d515 100644 --- a/backend/tests/test_all_models_e2e.py +++ b/backend/tests/test_all_models_e2e.py @@ -63,6 +63,7 @@ class MatrixRow: MatrixRow("tada 1B", "tada", "1B", "cloned", "tada-1b"), MatrixRow("tada 3B", "tada", "3B", "cloned", "tada-3b-ml"), MatrixRow("kokoro", "kokoro", None, "preset_kokoro", "kokoro"), + MatrixRow("voxcpm2", "voxcpm2", None, "cloned", "voxcpm2"), ] TEXT = "The quick brown fox jumps over the lazy dog." diff --git a/backend/tests/test_generation_engine_resolution.py b/backend/tests/test_generation_engine_resolution.py index 7cd3dc344..74dc0bf84 100644 --- a/backend/tests/test_generation_engine_resolution.py +++ b/backend/tests/test_generation_engine_resolution.py @@ -25,3 +25,22 @@ def test_explicit_engine_is_preserved(): def test_invalid_explicit_engine_is_rejected(): with pytest.raises(ValidationError): _request(engine="invalid") + + +@pytest.mark.parametrize("engine", ["voxcpm2", "omnivoice", "qwen_voice_design"]) +def test_omitted_engine_uses_saved_profile_preference(engine): + from types import SimpleNamespace + + from backend.routes.generations import _resolve_generation_engine + + profile = SimpleNamespace(default_engine=engine, voice_type="cloned", preset_engine=None) + assert _resolve_generation_engine(_request(), profile) == engine + + +@pytest.mark.parametrize("engine", ["voxcpm2", "omnivoice"]) +def test_cloned_profiles_accept_both_multilingual_engines(engine): + from types import SimpleNamespace + + from backend.services.profiles import validate_profile_engine + + validate_profile_engine(SimpleNamespace(voice_type="cloned"), engine) diff --git a/backend/tests/test_models_load_by_name.py b/backend/tests/test_models_load_by_name.py index 1e1f50a32..41c224254 100644 --- a/backend/tests/test_models_load_by_name.py +++ b/backend/tests/test_models_load_by_name.py @@ -78,7 +78,7 @@ def test_load_by_model_name_targets_that_model(client, loaded): @pytest.mark.parametrize( "model_name", - ["qwen-tts-0.6B", "chatterbox-tts", "whisper-turbo", "qwen3-0.6b"], + ["qwen-tts-0.6B", "chatterbox-tts", "whisper-turbo", "qwen3-0.6b", "voxcpm2", "omnivoice", "qwen-voice-design-1.7B"], ) def test_load_dispatches_across_engine_families(client, loaded, model_name): """TTS, STT, and LLM entries all resolve through the same registry.""" diff --git a/backend/tests/test_openai_compat.py b/backend/tests/test_openai_compat.py index 847a529c4..e65b49b1c 100644 --- a/backend/tests/test_openai_compat.py +++ b/backend/tests/test_openai_compat.py @@ -146,9 +146,10 @@ async def test_speech_wav_default_with_openai_alias_uses_profile_engine(client, assert calls["voice_prompt_engine"] == "qwen" -async def test_speech_model_is_a_voicebox_engine(client, app): - await client.audio.speech.create(model="chatterbox", voice="Morgan", input="Hi") - assert app.state.calls["load"] == ("chatterbox", "default") +@pytest.mark.parametrize("engine", ["chatterbox", "voxcpm2", "omnivoice"]) +async def test_speech_model_is_a_voicebox_engine(client, app, engine): + await client.audio.speech.create(model=engine, voice="Morgan", input="Hi") + assert app.state.calls["load"] == (engine, "default") async def test_speech_model_variant_selects_size(client, app): diff --git a/backend/tests/test_voxcpm_backend.py b/backend/tests/test_voxcpm_backend.py new file mode 100644 index 000000000..d4f1b51bd --- /dev/null +++ b/backend/tests/test_voxcpm_backend.py @@ -0,0 +1,139 @@ +"""Unit tests for the native VoxCPM2 backend without loading model weights.""" + +import asyncio +from types import SimpleNamespace + +import numpy as np +import pytest + +from backend.backends.voxcpm_backend import VoxCPMBackend + + +@pytest.mark.asyncio +async def test_create_voice_prompt_preserves_reference_audio_and_text(tmp_path): + backend = VoxCPMBackend() + audio_path = tmp_path / "reference.wav" + + prompt, cached = await backend.create_voice_prompt(str(audio_path), " hello ") + + assert prompt == {"ref_audio": str(audio_path), "ref_text": "hello"} + assert cached is False + + +@pytest.mark.asyncio +async def test_generate_forwards_ultimate_clone_prompt_and_returns_48khz_audio(tmp_path): + backend = VoxCPMBackend() + audio_path = tmp_path / "reference.wav" + audio_path.write_bytes(b"reference") + backend._device = "cpu" + captured = {} + + class FakeModel: + tts_model = SimpleNamespace(sample_rate=48000) + + def _generate(self, text, seed=None, **kwargs): + pass + + def generate(self, **kwargs): + captured.update(kwargs) + return np.zeros(32, dtype=np.float32) + + backend.model = FakeModel() + + audio, sample_rate = await backend.generate( + "hello", + {"ref_audio": str(audio_path), "ref_text": "reference text"}, + seed=7, + instruct="warm and clear", + ) + + assert audio.dtype == np.float32 + assert audio.shape == (32,) + assert sample_rate == 48000 + assert captured == { + "text": "(warm and clear)hello", + "seed": 7, + "retry_badcase": True, + "reference_wav_path": str(audio_path), + } + + +@pytest.mark.asyncio +async def test_generate_uses_transcript_guided_cloning_without_instruction(tmp_path): + backend = VoxCPMBackend() + audio_path = tmp_path / "reference.wav" + audio_path.write_bytes(b"reference") + captured = {} + + class LegacyModel: + tts_model = SimpleNamespace(sample_rate=48000) + + def _generate(self, text, **kwargs): + pass + + def generate(self, **kwargs): + captured.update(kwargs) + return np.ones(16, dtype=np.float32) + + backend.model = LegacyModel() + backend._device = "cpu" + + await backend.generate("hello", {"ref_audio": str(audio_path), "ref_text": "reference text"}, seed=9) + + assert captured == { + "text": "hello", + "retry_badcase": True, + "reference_wav_path": str(audio_path), + "prompt_wav_path": str(audio_path), + "prompt_text": "reference text", + } + + +def test_model_registry_contains_voxcpm2(): + from backend.backends import get_model_config, get_tts_backend_for_engine + + config = get_model_config("voxcpm2") + assert config is not None + assert config.engine == "voxcpm2" + assert config.hf_repo_id == "openbmb/VoxCPM2" + assert config.supports_instruct is True + assert len(config.languages) == 30 + assert isinstance(get_tts_backend_for_engine("voxcpm2"), VoxCPMBackend) + + +def test_voxcpm2_request_accepts_languages_added_by_the_engine(): + from backend.models import GenerationRequest, VoiceProfileCreate + + request = GenerationRequest(profile_id="profile", text="สวัสดี", language="th", engine="voxcpm2") + profile = VoiceProfileCreate(name="Thai voice", language="th") + + assert request.language == "th" + assert profile.language == "th" + + +@pytest.mark.parametrize("engine", ["voxcpm2", "omnivoice", "qwen_voice_design"]) +def test_request_contracts_preserve_all_new_engines(engine): + from backend.models import GenerationRequest, MCPClientBindingUpsert, SpeakRequest + + assert GenerationRequest(profile_id="p", text="hello", engine=engine).engine == engine + assert SpeakRequest(text="hello", engine=engine).engine == engine + assert MCPClientBindingUpsert(client_id="test", default_engine=engine).default_engine == engine + + +def test_voxcpm2_respects_force_cpu_override(monkeypatch): + monkeypatch.setenv("VOICEBOX_FORCE_CPU", "1") + assert VoxCPMBackend().device == "cpu" + + +@pytest.mark.asyncio +async def test_concurrent_model_loads_only_initialize_once(monkeypatch): + backend = VoxCPMBackend() + calls = [] + + def fake_load(): + calls.append(True) + backend.model = object() + + monkeypatch.setattr(backend, "_load_model_sync", fake_load) + await asyncio.gather(backend.load_model(), backend.load_model()) + assert calls == [True] diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index ff15c087e..1c00658bd 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -23,14 +23,17 @@ The backend exposes: -- **`TTSBackend` Protocol** with seven concrete engine implementations: +- **`TTSBackend` Protocol** with ten concrete engine implementations: - Qwen3-TTS (PyTorch or MLX depending on platform) - Qwen CustomVoice (predefined speakers with instruct) + - Qwen VoiceDesign (voices from written descriptions) - LuxTTS (fast, CPU-friendly) - Chatterbox Multilingual (23 languages) - Chatterbox Turbo (English, paralinguistic tags) - TADA (1B English, 3B multilingual via HumeAI) - Kokoro 82M (pre-built voices, CPU realtime) + - OmniVoice (23 exposed languages, non-commercial weights) + - VoxCPM2 (30-language local cloning) - **`STTBackend` Protocol** for Whisper (PyTorch or MLX-Whisper) - **Profiles / History / Stories** services for persistence and timeline editing @@ -50,6 +53,9 @@ The backend exposes: | TADA | `backend/backends/hume_backend.py` | HumeAI TADA — 1B English + 3B Multilingual | | Kokoro | `backend/backends/kokoro_backend.py` | Kokoro 82M — CPU realtime, pre-built voices | | Qwen CustomVoice | `backend/backends/qwen_custom_voice_backend.py` | Qwen CustomVoice — predefined speakers with instruct | +| Qwen VoiceDesign | `backend/backends/qwen_voice_design_backend.py` | Qwen VoiceDesign — voices from written descriptions | +| OmniVoice | `backend/backends/omnivoice_backend.py` | OmniVoice — non-commercial multilingual cloning | +| VoxCPM2 | `backend/backends/voxcpm_backend.py` | VoxCPM2 — 30-language local cloning | | Platform detect | `backend/platform_detect.py` | Apple Silicon → MLX, else → PyTorch | | API types | `backend/models.py` | Pydantic request/response models | | HF progress | `backend/utils/hf_progress.py` | HFProgressTracker (tqdm patching for download progress) | @@ -69,7 +75,7 @@ The backend exposes: ``` POST /generate 1. Look up voice profile from DB - 2. Resolve engine from request (qwen | qwen_custom_voice | luxtts | chatterbox | chatterbox_turbo | tada | kokoro) + 2. Resolve engine from request or profile preference (qwen | qwen_custom_voice | qwen_voice_design | luxtts | chatterbox | chatterbox_turbo | tada | kokoro | omnivoice | voxcpm2) 3. Get backend: get_tts_backend_for_engine(engine) # thread-safe singleton per engine 4. Check model cache → if missing, trigger background download, return HTTP 202 5. Load model (lazy): tts_backend.load_model(model_size) @@ -189,7 +195,7 @@ Shipped 2026-04-25 (PR #544). Voicebox went from a voice-cloning studio to a ful | Model | PR / Branch | Reason | |-------|-------------|--------| | **CosyVoice2/3** | PR #311 | Output quality too poor. Heavy deps, no PyPI, needed 5+ shims. PR should be closed. | -| **VoxCPM 1.5 / VoxCPM2** | `voicebox-new-models` research (2026-04-18) | **Backlogged.** See detailed analysis below. | +| **VoxCPM 1.5 / VoxCPM2** | `feat/voxcpm2-backend` | **Active community integration.** VoxCPM2 now has a native Voicebox backend draft using CUDA with macOS CPU fallback; see detailed analysis below. | #### VoxCPM — Evaluation Notes (2026-04-18) @@ -203,18 +209,15 @@ Shipped 2026-04-25 (PR #544). Voicebox went from a voice-cloning studio to a ful - Zero-shot cloning + style control via parenthetical prefixes in text (`(slightly faster, cheerful tone)...`) - Relatively high-quality output per demos -**Why we backlogged it:** -- **Effectively CUDA-only.** README states `CUDA ≥ 12.0` as hard requirement. Source code's `from_pretrained(device=None|"auto")` claims "preferring CUDA, then MPS, then CPU," but in practice: - - **MPS (Apple Silicon) broken upstream** — OpenBMB/VoxCPM issues #232 (`NotImplementedError: Output channels > 65536 not supported at the MPS device`) and #248 (`IndexError` on M3 Mac) are both open with no resolution. - - **CPU unsupported in the Python package** — issue #256 shows `voxcpm --device cpu` rejected with `unrecognized arguments`. The only CPU path is the third-party **VoxCPM.cpp** GGML engine, which is a separate ecosystem project, not `pip install voxcpm`. - - **macOS source install fails** — issue #233 open with no resolution. -- Would require CUDA-only gating in UI (new `requires_cuda` flag on `ModelConfig`, lock icon + "Requires NVIDIA GPU" in `ModelManagement.tsx` / `EngineModelSelector.tsx`) plus a hard error at `load_model()` as safety net. Doable but adds first-class platform gating that doesn't exist for any other engine today. -- Voicebox's user base skews Apple Silicon (MLX is a primary backend). Shipping a CUDA-only model sets a precedent worth a separate scoping discussion (see issues #419 engine sprawl, #420 platform tiers, PR #465). +**Earlier backlog assessment:** +- The original 2026-04-18 assessment treated VoxCPM2 as effectively CUDA-only because CPU/MPS reports failed during model warm-up and generation. +- Upstream subsequently added explicit device selection, safe MPS dtype handling, and a broadcastable SDPA mask for the CPU attention path. +- Local validation on Apple Silicon with PyTorch 2.8.0 confirmed CPU loading, ordinary generation, reference-audio cloning, and a short MPS generation all complete successfully. MPS was substantially slower in the smoke test, so the integration follows existing Chatterbox/TADA precedent: CUDA where available and CPU on macOS. -**What would change the decision:** -- Upstream fixes MPS crashes (watch issues #232, #248). -- We define an "experimental / CUDA-only" engine tier as part of issue #419 / PR #465, and decide it's acceptable to ship engines that are hidden on non-NVIDIA platforms. -- VoxCPM.cpp matures into a viable CPU path we can wrap (currently separate project, C++/GGML, unclear ergonomics). +**Remaining validation before shipping:** +- Exercise the frozen PyInstaller binary with a clean model cache. +- Validate CUDA generation on Windows/Linux hardware. +- Keep macOS on CPU until MPS performance and stability are consistently better. **Integration shape if we revive it:** Zero-shot cloning maps naturally to the Chatterbox-style backend (store `ref_audio` + `ref_text` paths in the voice prompt dict, process at generate time). Est. ~250 lines for `voxcpm_backend.py` + one `ModelConfig` entry + engine registration in `backends/__init__.py`. Frontend UI gating is the bigger lift. @@ -260,7 +263,7 @@ Shipped 2026-04-25 (PR #544). Voicebox went from a voice-cloning studio to a ful - **Thread-safe backend registry** (`_tts_backends` dict + `_tts_backends_lock`) with double-checked locking - **Per-engine backend instances** — each engine gets its own singleton, loaded lazily -- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro'` +- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'qwen_custom_voice' | 'qwen_voice_design' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'omnivoice' | 'voxcpm2'` - **Per-engine language filtering** — `ENGINE_LANGUAGES` map in frontend, backend regex accepts all languages - **Per-engine voice prompts** — `create_voice_prompt_for_profile()` dispatches to the correct backend - **Profile type system** — preset vs cloned profiles, UI grays out incompatible engines and auto-switches on selection @@ -578,7 +581,7 @@ Notable: | **HumeAI TADA 1B/3B** | Zero-shot | 5x faster than LLM-TTS | 24 kHz | EN (1B), 10 (3B) | Medium | Partial — prosody | PyTorch | **Shipped** (PR #296) | | **Kokoro-82M** | Preset voices | CPU realtime | 24 kHz | 8 | Tiny (82M) | None | All | **Shipped** (PR #325) | | ~~**CosyVoice2-0.5B**~~ | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | **Yes** | — | **Abandoned** (PR #311) — poor output quality | -| ~~**VoxCPM2**~~ | Zero-shot | ~0.15 RTF streaming | 48 kHz | 30 | Medium | Partial — parenthetical style | **CUDA-only in practice** | **Backlogged** (2026-04-18) — see notes above | +| **VoxCPM2** | Zero-shot | ~0.15 RTF streaming on CUDA | 48 kHz | 30 | Medium | Partial — parenthetical style | CUDA + CPU fallback | **Integration in progress** — see notes above | | **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | **Yes** — word-level inline | All | Candidate — license TBD | | **Fish Audio S2** | — | — | — | — | — | — | — | Candidate (#385) | | **XTTS-v2** | 6s zero-shot | Mid-GPU | 24 kHz | 17+ | Medium | Partial — style transfer from ref | All | Candidate — CPML license likely blocker | @@ -687,7 +690,7 @@ The generation form now uses a flat model dropdown with engine-based routing. Pe ### 7. Engine Sprawl — NEW -Eight TTS engines shipped, more candidates queued. Issue #419 asks for a first-class vs experimental distinction. Related: issue #420 asks for formalized platform support tiers. Combined, these would let us ship more engines more confidently with clearer expectations for users. +Ten TTS engines are represented by the active integration set, with more candidates queued. Issue #419 asks for a first-class vs experimental distinction. Related: issue #420 asks for formalized platform support tiers. Combined, these would let us ship more engines more confidently with clearer expectations for users. --- diff --git a/docs/content/docs/developer/model-management.mdx b/docs/content/docs/developer/model-management.mdx index 6572484b1..591c13d15 100644 --- a/docs/content/docs/developer/model-management.mdx +++ b/docs/content/docs/developer/model-management.mdx @@ -7,7 +7,7 @@ description: "How model downloading, loading, and status tracking works across a Voicebox manages two categories of models: -**TTS Models** — Eight engines covering zero-shot cloning, preset voices and voice design. Each engine may have one or more size variants. +**TTS Models** — Ten engines covering zero-shot cloning, preset voices and voice design. Each engine may have one or more size variants. **ASR Models** — Whisper for transcription. Five sizes, plus MLX-Whisper on Apple Silicon for ~8× faster transcription. @@ -28,9 +28,13 @@ Every model is described by a `ModelConfig` entry in `backend/backends/__init__. | **TADA 1B** | `tada` | `HumeAI/tada-1b` | 4 GB | ~4 GB | English | | **TADA 3B Multilingual** | `tada` | `HumeAI/tada-3b-ml` | 8 GB | ~8 GB | 10 | | **Kokoro 82M** | `kokoro` | `hexgrad/Kokoro-82M` | 350 MB | ~150 MB | 8 | +| **VoxCPM2** | `voxcpm2` | `openbmb/VoxCPM2` | 4.8 GB | ~5 GB | 30 | +| **OmniVoice (non-commercial)** | `omnivoice` | `k2-fsa/OmniVoice` | 3.3 GB | — | 23 exposed | On Apple Silicon, Qwen TTS uses MLX-optimized repos from `mlx-community` instead of the PyTorch repos. The backend picks automatically via `get_backend_type()`. CustomVoice and VoiceDesign are PyTorch-only — upstream publishes no MLX conversion. +VoxCPM2 uses CPU on macOS and CUDA when available on other platforms. MPS is deliberately disabled for this backend. + Qwen VoiceDesign ships a single 1.7B checkpoint; there is no 0.6B variant. It is the only engine that consumes `voice_type: "designed"` profiles, where the voice identity is the profile's `design_prompt` rather than reference audio or a preset id. ## Available Whisper Models diff --git a/docs/content/docs/developer/setup.mdx b/docs/content/docs/developer/setup.mdx index d4a777a52..f00f506c8 100644 --- a/docs/content/docs/developer/setup.mdx +++ b/docs/content/docs/developer/setup.mdx @@ -183,7 +183,7 @@ Models are automatically downloaded from HuggingFace Hub on first use, with live See [Model Management](/developer/model-management) for the full list. - First-time usage will be slower due to model downloads, but subsequent runs will use cached models. + First-time usage will be slower due to model downloads, but subsequent runs will use cached models. VoxCPM2 requires roughly 4.8 GB and outputs 48kHz audio. ## Generate OpenAPI Client @@ -235,9 +235,12 @@ pip install -r requirements-mlx.txt # Chatterbox pins numpy<1.26 / torch==2.6 which break on Python 3.12+ pip install --no-deps chatterbox-tts -# HumeAI TADA pins torch>=2.7,<2.8 which conflicts with our torch>=2.1 +# HumeAI TADA pins a narrow torch range; keep Voicebox's shared torch build pip install --no-deps hume-tada +# Voicebox uses VoxCPM2 synthesis without its optional UI/training/denoiser stack +pip install --no-deps "voxcpm>=2.0.3,<2.1" + # Install Qwen3-TTS from source pip install git+https://github.com/QwenLM/Qwen3-TTS.git diff --git a/docs/content/docs/developer/tts-generation.mdx b/docs/content/docs/developer/tts-generation.mdx index 4e2b9d0c6..e4a4d425b 100644 --- a/docs/content/docs/developer/tts-generation.mdx +++ b/docs/content/docs/developer/tts-generation.mdx @@ -142,7 +142,7 @@ The request path from frontend to audio file: "max_chunk_chars": 800 } ``` - The `engine` field is validated against the regex `^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$`. + The `engine` field accepts `qwen`, `qwen_custom_voice`, `qwen_voice_design`, `luxtts`, `chatterbox`, `chatterbox_turbo`, `tada`, `kokoro`, `omnivoice`, or `voxcpm2`. When omitted, the profile's default engine is used. 2. **Route** — `routes/generate.py` validates input and delegates. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index e0c4f5d64..0be1be928 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -5,7 +5,7 @@ description: "Voicebox is the open-source, local-first AI voice studio — a fre Voicebox is the **open-source, local-first AI voice studio** — a free alternative to ElevenLabs and WisprFlow in one app. Clone voices, generate -speech across 8 TTS engines, dictate into any app with a global hotkey, +speech across 10 TTS engines, dictate into any app with a global hotkey, compose multi-voice projects, and let any MCP-aware agent speak in a voice you own. Everything runs on your hardware. @@ -15,9 +15,9 @@ you own. Everything runs on your hardware. - **Captures tab** — paired audio + transcript archive, retranscribe / refine / play-as-voice - **Voice personalities** — per-profile compose button + persona-rewrite toggle, powered by a local LLM - **Agents speak back** — any MCP-aware agent can call Voicebox to speak in one of your cloned voices -- **8 TTS engines** — Qwen3-TTS, Qwen CustomVoice, Qwen VoiceDesign, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro +- **10 TTS engines** — Qwen3-TTS, Qwen CustomVoice, Qwen VoiceDesign, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro, OmniVoice (non-commercial), VoxCPM2 - **Cloning and preset voices** — zero-shot cloning or 50+ curated preset voices -- **23 languages** — from English to Arabic, Japanese, Hindi, Swahili +- **30 languages** — from English to Arabic, Japanese, Hindi, Thai, Vietnamese - **Post-processing effects** — pitch shift, reverb, delay, chorus, compression, filters - **Expressive speech** — paralinguistic tags (`[laugh]`, `[sigh]`) and natural-language delivery control - **Unlimited length** — auto-chunking with crossfade for long scripts diff --git a/docs/content/docs/overview/creating-voice-profiles.mdx b/docs/content/docs/overview/creating-voice-profiles.mdx index 2813b5841..901fdfeb2 100644 --- a/docs/content/docs/overview/creating-voice-profiles.mdx +++ b/docs/content/docs/overview/creating-voice-profiles.mdx @@ -27,7 +27,7 @@ Use this when you want to replicate a specific person's voice from a recording. 10-30 seconds of clear speech, minimal background noise. See [Voice Cloning](/overview/voice-cloning) for the engine catalog. - **Profiles** → **+ New Profile** → choose a cloning engine (Qwen3-TTS, Chatterbox Multilingual, Chatterbox Turbo, LuxTTS, or TADA) + **Profiles** → **+ New Profile** → choose a cloning engine (Qwen3-TTS, Chatterbox Multilingual, Chatterbox Turbo, LuxTTS, TADA, or VoxCPM2) Drag in an audio file, or record directly with the in-app recorder diff --git a/docs/content/docs/overview/introduction.mdx b/docs/content/docs/overview/introduction.mdx index 0a1f53a18..88b7541b1 100644 --- a/docs/content/docs/overview/introduction.mdx +++ b/docs/content/docs/overview/introduction.mdx @@ -27,7 +27,7 @@ shared between input and output. into the focused field on macOS and Windows (see [Dictation](/overview/dictation)) - **Captures tab** — paired audio + transcript archive, retranscribe, refine, play-as-voice, promote-to-sample (see [Captures](/overview/captures)) -- **Voice cloning** — 5 cloning engines covering 23 languages. Zero-shot +- **Voice cloning** — 7 cloning engines covering 30 languages. Zero-shot cloning from a reference sample (see [Voice Cloning](/overview/voice-cloning)) - **Preset voices** — 50+ curated voices via Kokoro and Qwen CustomVoice for when you don't want to clone (see [Preset Voices](/overview/preset-voices)) @@ -46,7 +46,7 @@ shared between input and output. ## TTS Engines -Eight engines with different strengths, switchable per-generation: +Ten engines with different strengths, switchable per-generation: | Engine | Profile Type | Languages | Strengths | |--------|--------------|-----------|-----------| @@ -58,6 +58,8 @@ Eight engines with different strengths, switchable per-generation: | **Chatterbox Turbo** | Cloned | English | Fast 350M model with paralinguistic emotion/sound tags | | **TADA** (1B / 3B) | Cloned | 10 | HumeAI speech-language model — 700s+ coherent audio | | **Kokoro** | Preset (50 voices) | 9 | 82M parameters, CPU realtime, lowest VRAM of any engine | +| **VoxCPM2** | Cloned | 30 | 2B multilingual model, 48kHz output, transcript-guided cloning | +| **OmniVoice** | Cloned | 23 exposed | Multilingual cloning and attribute instructions; non-commercial weights | ## STT and local LLM @@ -101,7 +103,7 @@ No cloud fallback, no bring-your-own-API-key. Local is the product. | Frontend | React, TypeScript, Tailwind CSS | | State | Zustand, React Query | | Backend | FastAPI (Python) | -| TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro | +| TTS Engines | Qwen3-TTS, Qwen CustomVoice, Qwen VoiceDesign, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro, OmniVoice (non-commercial), VoxCPM2 | | STT | Whisper / Whisper Turbo (PyTorch or MLX) | | Local LLM | Qwen3 0.6B / 1.7B / 4B (MLX or PyTorch) | | Effects | Pedalboard (Spotify) | diff --git a/docs/content/docs/overview/mcp-server.mdx b/docs/content/docs/overview/mcp-server.mdx index 4acdeee05..3d36c7645 100644 --- a/docs/content/docs/overview/mcp-server.mdx +++ b/docs/content/docs/overview/mcp-server.mdx @@ -116,7 +116,7 @@ for the shim to connect. voicebox_speak({ text: "Deploy complete.", profile?: "Morgan", // name or id; falls back to per-client binding, then default - engine?: "qwen", // qwen | qwen_custom_voice | luxtts | chatterbox | chatterbox_turbo | tada | kokoro + engine?: "qwen", // qwen | qwen_custom_voice | qwen_voice_design | luxtts | chatterbox | chatterbox_turbo | tada | kokoro | omnivoice | voxcpm2 personality?: true, // rewrite via the profile's personality LLM before TTS; default comes from the per-client binding language?: "en", }) diff --git a/docs/content/docs/overview/remote-mode.mdx b/docs/content/docs/overview/remote-mode.mdx index 12abcfc63..7b3f9274a 100644 --- a/docs/content/docs/overview/remote-mode.mdx +++ b/docs/content/docs/overview/remote-mode.mdx @@ -35,6 +35,7 @@ In Remote Mode, the Voicebox desktop app (running on your local machine) communi # Engines with incompatible transitive pins — install with --no-deps pip install --no-deps chatterbox-tts pip install --no-deps hume-tada + pip install --no-deps "voxcpm>=2.0.3,<2.1" # Qwen3-TTS from source pip install git+https://github.com/QwenLM/Qwen3-TTS.git diff --git a/docs/content/docs/overview/troubleshooting.mdx b/docs/content/docs/overview/troubleshooting.mdx index f0e4ff4a4..fe8eb1d51 100644 --- a/docs/content/docs/overview/troubleshooting.mdx +++ b/docs/content/docs/overview/troubleshooting.mdx @@ -338,6 +338,7 @@ This is expected behavior. The first generation downloads the selected TTS engin pip install -r requirements.txt pip install --no-deps chatterbox-tts pip install --no-deps hume-tada + pip install --no-deps "voxcpm>=2.0.3,<2.1" pip install git+https://github.com/QwenLM/Qwen3-TTS.git ``` diff --git a/docs/content/docs/overview/voice-cloning.mdx b/docs/content/docs/overview/voice-cloning.mdx index 400e114f5..ae6fc87b0 100644 --- a/docs/content/docs/overview/voice-cloning.mdx +++ b/docs/content/docs/overview/voice-cloning.mdx @@ -7,7 +7,7 @@ description: "Clone any voice from a few seconds of reference audio" Voicebox can replicate a specific person's voice from a short audio sample — known as **zero-shot voice cloning**. You provide 10-30 seconds of clear speech, the model extracts a voice embedding, and from then on you can generate any text in that voice. -Five engines in 0.4 support cloning: +Six engines support cloning: | Engine | Languages | Strengths | | --------------------------- | --------- | -------------------------------------------------------------------------- | @@ -16,6 +16,7 @@ Five engines in 0.4 support cloning: | **Chatterbox Turbo** | English | Fast 350M model with paralinguistic emotion tags (`[laugh]`, `[sigh]`) | | **LuxTTS** | English | Lightweight (~1 GB VRAM), 48 kHz output, 150x realtime on CPU | | **TADA** (1B / 3B) | 10 | Speech-language model with 700s+ coherent long-form generation | +| **VoxCPM2** | 30 | 2B multilingual model, 48 kHz output, transcript-guided high-fidelity cloning | Don't want to record audio? Use a curated voice from Kokoro or Qwen CustomVoice instead — see [Preset Voices](/overview/preset-voices). @@ -50,6 +51,7 @@ Different engines suit different use cases. The profile grid greys out unsupport | Expressive English with `[laugh]` `[sigh]` tags | **Chatterbox Turbo** | | CPU-only or GPU-light setup, English | **LuxTTS** | | Long-form generation (audiobooks, full chapters) | **TADA 3B** | +| Broad multilingual cloning with 48 kHz output | **VoxCPM2** | ## Best Practices @@ -89,6 +91,7 @@ Adding multiple samples from the same speaker can improve quality: - **Chatterbox Turbo** — English - **LuxTTS** — English - **TADA 3B** — 10 multilingual; **TADA 1B** — English +- **VoxCPM2** — 30 languages; CUDA acceleration with CPU fallback For complete language tables and engine-specific notes, see the [TTS Engines developer guide](/developer/tts-engines). diff --git a/justfile b/justfile index 4e12685a8..7e8ad04ce 100644 --- a/justfile +++ b/justfile @@ -125,6 +125,8 @@ setup-python: # OmniVoice requires transformers>=5.3; we vendor the one class it # needs from 5.x (see backend/vendor/higgs_audio_v2_tokenizer). "{{ python }}" -m pip install --no-deps omnivoice + # VoxCPM's optional UI/training/denoiser dependency tree is not used by Voicebox. + "{{ python }}" -m pip install --no-deps 'voxcpm>=2.0.3,<2.1' # Apple Silicon: install MLX backend if [ "$(uname -m)" = "arm64" ] && [ "$(uname)" = "Darwin" ]; then echo "Detected Apple Silicon — installing MLX dependencies..." @@ -211,6 +213,7 @@ setup-python: Invoke-Pip install --no-deps chatterbox-tts; \ Invoke-Pip install --no-deps hume-tada; \ Invoke-Pip install --no-deps omnivoice; \ + Invoke-Pip install --no-deps "voxcpm>=2.0.3,<2.1"; \ Invoke-Pip install git+https://github.com/QwenLM/Qwen3-TTS.git; \ Invoke-Pip install pyinstaller ruff pytest pytest-asyncio openai -q; \ Write-Host "Python environment ready." diff --git a/scripts/engine-capabilities.test.ts b/scripts/engine-capabilities.test.ts new file mode 100644 index 000000000..39e0dd5e4 --- /dev/null +++ b/scripts/engine-capabilities.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { engineSupportsInstruct } from '../app/src/lib/constants/engines'; +import { getLanguageOptionsForEngine } from '../app/src/lib/constants/languages'; + +test('instruction controls preserve upstream engines alongside VoxCPM2', () => { + for (const engine of ['qwen_custom_voice', 'qwen_voice_design', 'omnivoice', 'voxcpm2']) { + assert.equal(engineSupportsInstruct(engine), true, engine); + } + for (const engine of ['qwen', 'kokoro', undefined]) { + assert.equal(engineSupportsInstruct(engine), false, engine); + } +}); + +test('language menus retain separate engine capabilities', () => { + const voxcpm = getLanguageOptionsForEngine('voxcpm2'); + const omni = getLanguageOptionsForEngine('omnivoice'); + const design = getLanguageOptionsForEngine('qwen_voice_design'); + + assert.equal(voxcpm.length, 30); + assert.equal(new Set(voxcpm.map(({ value }) => value)).size, 30); + assert.ok(voxcpm.every(({ label }) => label)); + assert.ok(voxcpm.some(({ value }) => value === 'th')); + assert.equal(omni.length, 23); + assert.equal( + omni.some(({ value }) => value === 'th'), + false, + ); + assert.equal(design.length, 10); +});