update

data/speakers/Bob_ft10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91015b82a99c40034048090228b6d647ab99fd7b86e8babd6a7c3a9236e8d800
+size 4508

modules/ChatTTS/ChatTTS/core.py

@@ -17,7 +17,7 @@ from .infer.api import refine_text, infer_code
 
 from huggingface_hub import snapshot_download
 
-logging.basicConfig(level=logging.ERROR)
+logging.basicConfig(level=logging.INFO)
 
 
 class Chat:

modules/SynthesizeSegments.py

@@ -1,4 +1,5 @@
 import copy
+import re
 from box import Box
 from pydub import AudioSegment
 from typing import List, Union
@@ -160,7 +161,21 @@ class SynthesizeSegments:
         for i in range(0, len(bucket), self.batch_size):
             batch = bucket[i : i + self.batch_size]
             param_arr = [self.segment_to_generate_params(segment) for segment in batch]
-            texts = [params.text + self.eos for params in param_arr]
+
+            def append_eos(text: str):
+                text = text.strip()
+                eos_arr = ["[uv_break]", "[v_break]", "[lbreak]", "[llbreak]"]
+                has_eos = False
+                for eos in eos_arr:
+                    if eos in text:
+                        has_eos = True
+                        break
+                if not has_eos:
+                    text += self.eos
+                return text
+
+            # 这里会添加 end_of_text 到 text 之后
+            texts = [append_eos(params.text) for params in param_arr]
 
             params = param_arr[0]
             audio_datas = generate_audio.generate_audio_batch(
@@ -182,6 +197,7 @@ class SynthesizeSegments:
 
                 audio_segment = audio_data_to_segment(audio_data, sr)
                 audio_segment = apply_prosody(audio_segment, rate, volume, pitch)
+                # compare by Box object
                 original_index = src_segments.index(segment)
                 audio_segments[original_index] = audio_segment
 
@@ -226,13 +242,30 @@ class SynthesizeSegments:
 
             sentences = spliter.parse(text)
             for sentence in sentences:
-                ret_segments.append(
-                    SSMLSegment(
-                        text=sentence,
-                        attrs=segment.attrs.copy(),
-                        params=copy.copy(segment.params),
-                    )
+                seg = SSMLSegment(
+                    text=sentence,
+                    attrs=segment.attrs.copy(),
+                    params=copy.copy(segment.params),
                 )
+                ret_segments.append(seg)
+                setattr(seg, "_idx", len(ret_segments) - 1)
+
+        def is_none_speak_segment(segment: SSMLSegment):
+            text = segment.text.strip()
+            regexp = r"\[[^\]]+?\]"
+            text = re.sub(regexp, "", text)
+            text = text.strip()
+            if not text:
+                return True
+            return False
+
+        # 将 none_speak 合并到前一个 speak segment
+        for i in range(1, len(ret_segments)):
+            if is_none_speak_segment(ret_segments[i]):
+                ret_segments[i - 1].text += ret_segments[i].text
+                ret_segments[i].text = ""
+        # 移除空的 segment
+        ret_segments = [seg for seg in ret_segments if seg.text.strip()]
 
         return ret_segments
 

modules/api/app_config.py

@@ -1,6 +1,6 @@
 app_description = """
-ChatTTS-Forge 是一个功能强大的文本转语音生成工具，支持通过类 SSML 语法生成丰富的音频长文本，并提供全面的 API 服务，适用于各种场景。<br/>
-ChatTTS-Forge is a powerful text-to-speech generation tool that supports generating rich audio long texts through class SSML syntax
+🍦 ChatTTS-Forge 是一个围绕 TTS 生成模型 ChatTTS 开发的项目，实现了 API Server 和 基于 Gradio 的 WebUI。<br/>
+🍦 ChatTTS-Forge is a project developed around the TTS generation model ChatTTS, implementing an API Server and a Gradio-based WebUI.
 
 项目地址: [https://github.com/lenML/ChatTTS-Forge](https://github.com/lenML/ChatTTS-Forge)
 

modules/api/impl/google_api.py

@@ -1,38 +1,25 @@
-import base64
-from typing import Literal
+from typing import Union
 from fastapi import HTTPException
 
-import io
-import soundfile as sf
 from pydantic import BaseModel
 
 
-from modules.Enhancer.ResembleEnhance import (
-    apply_audio_enhance,
-    apply_audio_enhance_full,
-)
 from modules.api.Api import APIManager
-from modules.synthesize_audio import synthesize_audio
-from modules.utils import audio
-from modules.utils.audio import apply_prosody_to_audio_data
-from modules.normalization import text_normalize
+from modules.api.impl.handler.SSMLHandler import SSMLHandler
+from modules.api.impl.handler.TTSHandler import TTSHandler
+from modules.api.impl.model.audio_model import AdjustConfig, AudioFormat
+from modules.api.impl.model.chattts_model import ChatTTSConfig, InferConfig
+from modules.api.impl.model.enhancer_model import EnhancerConfig
 
-from modules import generate_audio as generate
-from modules.speaker import speaker_mgr
+from modules.speaker import Speaker, speaker_mgr
 
 
-from modules.ssml_parser.SSMLParser import create_ssml_parser
-from modules.SynthesizeSegments import (
-    SynthesizeSegments,
-    combine_audio_segments,
-)
-
 from modules.api import utils as api_utils
 
 
 class SynthesisInput(BaseModel):
-    text: str = ""
-    ssml: str = ""
+    text: Union[str, None] = None
+    ssml: Union[str, None] = None
 
 
 class VoiceSelectionParams(BaseModel):
@@ -50,24 +37,15 @@ class VoiceSelectionParams(BaseModel):
 
 
 class AudioConfig(BaseModel):
-    audioEncoding: api_utils.AudioFormat = "mp3"
+    audioEncoding: AudioFormat = AudioFormat.mp3
     speakingRate: float = 1
     pitch: float = 0
     volumeGainDb: float = 0
     sampleRateHertz: int = 24000
-    batchSize: int = 1
+    batchSize: int = 4
     spliterThreshold: int = 100
 
 
-class EnhancerConfig(BaseModel):
-    enabled: bool = False
-    model: str = "resemble-enhance"
-    nfe: int = 32
-    solver: Literal["midpoint", "rk4", "euler"] = "midpoint"
-    lambd: float = 0.5
-    tau: float = 0.5
-
-
 class GoogleTextSynthesizeRequest(BaseModel):
     input: SynthesisInput
     voice: VoiceSelectionParams
@@ -92,7 +70,11 @@ async def google_text_synthesize(request: GoogleTextSynthesizeRequest):
     voice_name = voice.name
     infer_seed = voice.seed or 42
     eos = voice.eos or "[uv_break]"
-    audio_format = audioConfig.audioEncoding or "mp3"
+    audio_format = audioConfig.audioEncoding
+
+    if not isinstance(audio_format, AudioFormat) and isinstance(audio_format, str):
+        audio_format = AudioFormat(audio_format)
+
     speaking_rate = audioConfig.speakingRate or 1
     pitch = audioConfig.pitch or 0
     volume_gain_db = audioConfig.volumeGainDb or 0
@@ -101,6 +83,7 @@ async def google_text_synthesize(request: GoogleTextSynthesizeRequest):
 
     spliter_threshold = audioConfig.spliterThreshold or 100
 
+    # TODO
     sample_rate = audioConfig.sampleRateHertz or 24000
 
     params = api_utils.calc_spk_style(spk=voice.name, style=voice.style)
@@ -111,92 +94,68 @@ async def google_text_synthesize(request: GoogleTextSynthesizeRequest):
             status_code=422, detail="The specified voice name is not supported."
         )
 
-    if audio_format != "mp3" and audio_format != "wav":
+    if not isinstance(params.get("spk"), Speaker):
         raise HTTPException(
-            status_code=422, detail="Invalid audio encoding format specified."
+            status_code=422, detail="The specified voice name is not supported."
         )
 
-    if enhancerConfig.enabled:
-        # TODO enhancer params checker
-        pass
-
+    speaker = params.get("spk")
+    tts_config = ChatTTSConfig(
+        style=params.get("style", ""),
+        temperature=voice.temperature,
+        top_k=voice.topK,
+        top_p=voice.topP,
+    )
+    infer_config = InferConfig(
+        batch_size=batch_size,
+        spliter_threshold=spliter_threshold,
+        eos=eos,
+        seed=infer_seed,
+    )
+    adjust_config = AdjustConfig(
+        speaking_rate=speaking_rate,
+        pitch=pitch,
+        volume_gain_db=volume_gain_db,
+    )
+    enhancer_config = enhancerConfig
+
+    mime_type = f"audio/{audio_format.value}"
+    if audio_format == AudioFormat.mp3:
+        mime_type = "audio/mpeg"
     try:
         if input.text:
-            # 处理文本合成逻辑
-            text = text_normalize(input.text, is_end=True)
-            sample_rate, audio_data = synthesize_audio(
-                text,
-                temperature=(
-                    voice.temperature
-                    if voice.temperature
-                    else params.get("temperature", 0.3)
-                ),
-                top_P=voice.topP if voice.topP else params.get("top_p", 0.7),
-                top_K=voice.topK if voice.topK else params.get("top_k", 20),
-                spk=params.get("spk", -1),
-                infer_seed=infer_seed,
-                prompt1=params.get("prompt1", ""),
-                prompt2=params.get("prompt2", ""),
-                prefix=params.get("prefix", ""),
-                batch_size=batch_size,
-                spliter_threshold=spliter_threshold,
-                end_of_sentence=eos,
+            text_content = input.text
+
+            handler = TTSHandler(
+                text_content=text_content,
+                spk=speaker,
+                tts_config=tts_config,
+                infer_config=infer_config,
+                adjust_config=adjust_config,
+                enhancer_config=enhancer_config,
             )
 
-        elif input.ssml:
-            parser = create_ssml_parser()
-            segments = parser.parse(input.ssml)
-            for seg in segments:
-                seg["text"] = text_normalize(seg["text"], is_end=True)
-
-            if len(segments) == 0:
-                raise HTTPException(
-                    status_code=422, detail="The SSML text is empty or parsing failed."
-                )
-
-            synthesize = SynthesizeSegments(
-                batch_size=batch_size, eos=eos, spliter_thr=spliter_threshold
-            )
-            audio_segments = synthesize.synthesize_segments(segments)
-            combined_audio = combine_audio_segments(audio_segments)
+            base64_string = handler.enqueue_to_base64(format=audio_format)
+            return {"audioContent": f"data:{mime_type};base64,{base64_string}"}
 
-            sample_rate, audio_data = audio.pydub_to_np(combined_audio)
-        else:
-            raise HTTPException(
-                status_code=422, detail="Either text or SSML input must be provided."
-            )
+        elif input.ssml:
+            ssml_content = input.ssml
 
-        if enhancerConfig.enabled:
-            audio_data, sample_rate = apply_audio_enhance_full(
-                audio_data=audio_data,
-                sr=sample_rate,
-                nfe=enhancerConfig.nfe,
-                solver=enhancerConfig.solver,
-                lambd=enhancerConfig.lambd,
-                tau=enhancerConfig.tau,
+            handler = SSMLHandler(
+                ssml_content=ssml_content,
+                infer_config=infer_config,
+                adjust_config=adjust_config,
+                enhancer_config=enhancer_config,
             )
 
-        audio_data = apply_prosody_to_audio_data(
-            audio_data,
-            rate=speaking_rate,
-            pitch=pitch,
-            volume=volume_gain_db,
-            sr=sample_rate,
-        )
-
-        buffer = io.BytesIO()
-        sf.write(buffer, audio_data, sample_rate, format="wav")
-        buffer.seek(0)
+            base64_string = handler.enqueue_to_base64(format=audio_format)
 
-        if audio_format == "mp3":
-            buffer = api_utils.wav_to_mp3(buffer)
+            return {"audioContent": f"data:{mime_type};base64,{base64_string}"}
 
-        base64_encoded = base64.b64encode(buffer.read())
-        base64_string = base64_encoded.decode("utf-8")
-
-        return {
-            "audioContent": f"data:audio/{audio_format.lower()};base64,{base64_string}"
-        }
+        else:
+            raise HTTPException(
+                status_code=422, detail="Invalid input text or ssml specified."
+            )
 
     except Exception as e:
         import logging

modules/api/impl/handler/AudioHandler.py

@@ -0,0 +1,37 @@
+import base64
+import io
+import numpy as np
+import soundfile as sf
+
+from modules.api.impl.model.audio_model import AudioFormat
+from modules.api import utils as api_utils
+
+
+class AudioHandler:
+    def enqueue(self) -> tuple[np.ndarray, int]:
+        raise NotImplementedError
+
+    def enqueue_to_buffer(self, format: AudioFormat) -> io.BytesIO:
+        audio_data, sample_rate = self.enqueue()
+
+        buffer = io.BytesIO()
+        sf.write(buffer, audio_data, sample_rate, format="wav")
+        buffer.seek(0)
+
+        if format == AudioFormat.mp3:
+            buffer = api_utils.wav_to_mp3(buffer)
+
+        return buffer
+
+    def enqueue_to_bytes(self, format: AudioFormat) -> bytes:
+        buffer = self.enqueue_to_buffer(format=format)
+        binary = buffer.read()
+        return binary
+
+    def enqueue_to_base64(self, format: AudioFormat) -> str:
+        binary = self.enqueue_to_bytes(format=format)
+
+        base64_encoded = base64.b64encode(binary)
+        base64_string = base64_encoded.decode("utf-8")
+
+        return base64_string

modules/api/impl/handler/SSMLHandler.py

@@ -0,0 +1,94 @@
+from fastapi import HTTPException
+import numpy as np
+
+from modules.Enhancer.ResembleEnhance import apply_audio_enhance_full
+from modules.SynthesizeSegments import SynthesizeSegments, combine_audio_segments
+from modules.api.impl.handler.AudioHandler import AudioHandler
+from modules.api.impl.model.audio_model import AdjustConfig
+from modules.api.impl.model.chattts_model import InferConfig
+from modules.api.impl.model.enhancer_model import EnhancerConfig
+from modules.normalization import text_normalize
+from modules.ssml_parser.SSMLParser import create_ssml_parser
+from modules.utils import audio
+
+
+class SSMLHandler(AudioHandler):
+    def __init__(
+        self,
+        ssml_content: str,
+        infer_config: InferConfig,
+        adjust_config: AdjustConfig,
+        enhancer_config: EnhancerConfig,
+    ) -> None:
+        assert isinstance(ssml_content, str), "ssml_content must be a string."
+        assert isinstance(
+            infer_config, InferConfig
+        ), "infer_config must be an InferConfig object."
+        assert isinstance(
+            adjust_config, AdjustConfig
+        ), "adjest_config should be AdjustConfig"
+        assert isinstance(
+            enhancer_config, EnhancerConfig
+        ), "enhancer_config must be an EnhancerConfig object."
+
+        self.ssml_content = ssml_content
+        self.infer_config = infer_config
+        self.adjest_config = adjust_config
+        self.enhancer_config = enhancer_config
+
+        self.validate()
+
+    def validate(self):
+        # TODO params checker
+        pass
+
+    def enqueue(self) -> tuple[np.ndarray, int]:
+        ssml_content = self.ssml_content
+        infer_config = self.infer_config
+        adjust_config = self.adjest_config
+        enhancer_config = self.enhancer_config
+
+        parser = create_ssml_parser()
+        segments = parser.parse(ssml_content)
+        for seg in segments:
+            seg["text"] = text_normalize(seg["text"], is_end=True)
+
+        if len(segments) == 0:
+            raise HTTPException(
+                status_code=422, detail="The SSML text is empty or parsing failed."
+            )
+
+        synthesize = SynthesizeSegments(
+            batch_size=infer_config.batch_size,
+            eos=infer_config.eos,
+            spliter_thr=infer_config.spliter_threshold,
+        )
+        audio_segments = synthesize.synthesize_segments(segments)
+        combined_audio = combine_audio_segments(audio_segments)
+
+        sample_rate, audio_data = audio.pydub_to_np(combined_audio)
+
+        if enhancer_config.enabled:
+            nfe = enhancer_config.nfe
+            solver = enhancer_config.solver
+            lambd = enhancer_config.lambd
+            tau = enhancer_config.tau
+
+            audio_data, sample_rate = apply_audio_enhance_full(
+                audio_data=audio_data,
+                sr=sample_rate,
+                nfe=nfe,
+                solver=solver,
+                lambd=lambd,
+                tau=tau,
+            )
+
+        audio_data = audio.apply_prosody_to_audio_data(
+            audio_data=audio_data,
+            rate=adjust_config.speed_rate,
+            pitch=adjust_config.pitch,
+            volume=adjust_config.volume_gain_db,
+            sr=sample_rate,
+        )
+
+        return audio_data, sample_rate

modules/api/impl/handler/TTSHandler.py

@@ -0,0 +1,97 @@
+import numpy as np
+from modules.Enhancer.ResembleEnhance import apply_audio_enhance_full
+from modules.api.impl.handler.AudioHandler import AudioHandler
+from modules.api.impl.model.audio_model import AdjustConfig
+from modules.api.impl.model.chattts_model import ChatTTSConfig, InferConfig
+from modules.api.impl.model.enhancer_model import EnhancerConfig
+from modules.normalization import text_normalize
+from modules.speaker import Speaker
+from modules.synthesize_audio import synthesize_audio
+
+from modules.utils.audio import apply_prosody_to_audio_data
+
+
+class TTSHandler(AudioHandler):
+    def __init__(
+        self,
+        text_content: str,
+        spk: Speaker,
+        tts_config: ChatTTSConfig,
+        infer_config: InferConfig,
+        adjust_config: AdjustConfig,
+        enhancer_config: EnhancerConfig,
+    ):
+        assert isinstance(text_content, str), "text_content should be str"
+        assert isinstance(spk, Speaker), "spk should be Speaker"
+        assert isinstance(
+            tts_config, ChatTTSConfig
+        ), "tts_config should be ChatTTSConfig"
+        assert isinstance(
+            infer_config, InferConfig
+        ), "infer_config should be InferConfig"
+        assert isinstance(
+            adjust_config, AdjustConfig
+        ), "adjest_config should be AdjustConfig"
+        assert isinstance(
+            enhancer_config, EnhancerConfig
+        ), "enhancer_config should be EnhancerConfig"
+
+        self.text_content = text_content
+        self.spk = spk
+        self.tts_config = tts_config
+        self.infer_config = infer_config
+        self.adjest_config = adjust_config
+        self.enhancer_config = enhancer_config
+
+        self.validate()
+
+    def validate(self):
+        # TODO params checker
+        pass
+
+    def enqueue(self) -> tuple[np.ndarray, int]:
+        text = text_normalize(self.text_content)
+        tts_config = self.tts_config
+        infer_config = self.infer_config
+        adjust_config = self.adjest_config
+        enhancer_config = self.enhancer_config
+
+        sample_rate, audio_data = synthesize_audio(
+            text,
+            spk=self.spk,
+            temperature=tts_config.temperature,
+            top_P=tts_config.top_p,
+            top_K=tts_config.top_k,
+            prompt1=tts_config.prompt1,
+            prompt2=tts_config.prompt2,
+            prefix=tts_config.prefix,
+            infer_seed=infer_config.seed,
+            batch_size=infer_config.batch_size,
+            spliter_threshold=infer_config.spliter_threshold,
+            end_of_sentence=infer_config.eos,
+        )
+
+        if enhancer_config.enabled:
+            nfe = enhancer_config.nfe
+            solver = enhancer_config.solver
+            lambd = enhancer_config.lambd
+            tau = enhancer_config.tau
+
+            audio_data, sample_rate = apply_audio_enhance_full(
+                audio_data=audio_data,
+                sr=sample_rate,
+                nfe=nfe,
+                solver=solver,
+                lambd=lambd,
+                tau=tau,
+            )
+
+        audio_data = apply_prosody_to_audio_data(
+            audio_data=audio_data,
+            rate=adjust_config.speed_rate,
+            pitch=adjust_config.pitch,
+            volume=adjust_config.volume_gain_db,
+            sr=sample_rate,
+        )
+
+        return audio_data, sample_rate

modules/api/impl/model/audio_model.py

@@ -0,0 +1,14 @@
+from enum import Enum
+
+from pydantic import BaseModel
+
+
+class AudioFormat(str, Enum):
+    mp3 = "mp3"
+    wav = "wav"
+
+
+class AdjustConfig(BaseModel):
+    pitch: float = 0
+    speed_rate: float = 1
+    volume_gain_db: float = 0

modules/api/impl/model/chattts_model.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+
+
+class ChatTTSConfig(BaseModel):
+    style: str = ""
+    temperature: float = 0.3
+    top_p: float = 0.7
+    top_k: int = 20
+    prompt1: str = ""
+    prompt2: str = ""
+    prefix: str = ""
+
+
+class InferConfig(BaseModel):
+    batch_size: int = 4
+    spliter_threshold: int = 100
+    # end_of_sentence
+    eos: str = "[uv_break]"
+    seed: int = 42

modules/api/impl/model/enhancer_model.py

@@ -0,0 +1,11 @@
+from typing import Literal
+from pydantic import BaseModel
+
+
+class EnhancerConfig(BaseModel):
+    enabled: bool = False
+    model: str = "resemble-enhance"
+    nfe: int = 32
+    solver: Literal["midpoint", "rk4", "euler"] = "midpoint"
+    lambd: float = 0.5
+    tau: float = 0.5

modules/api/impl/openai_api.py

@@ -1,42 +1,38 @@
 from fastapi import File, Form, HTTPException, Body, UploadFile
-from fastapi.responses import StreamingResponse
 
-import io
 from numpy import clip
-import soundfile as sf
 from pydantic import BaseModel, Field
-from fastapi.responses import FileResponse
-
+from fastapi.responses import StreamingResponse
 
-from modules.synthesize_audio import synthesize_audio
-from modules.normalization import text_normalize
 
-from modules import generate_audio as generate
+from modules.api.impl.handler.TTSHandler import TTSHandler
+from modules.api.impl.model.audio_model import AdjustConfig, AudioFormat
+from modules.api.impl.model.chattts_model import ChatTTSConfig, InferConfig
+from modules.api.impl.model.enhancer_model import EnhancerConfig
 
 
-from typing import List, Literal, Optional, Union
-import pyrubberband as pyrb
+from typing import List, Optional
 
 from modules.api import utils as api_utils
 from modules.api.Api import APIManager
 
-from modules.speaker import speaker_mgr
+from modules.speaker import Speaker, speaker_mgr
 from modules.data import styles_mgr
 
-import numpy as np
-
 
 class AudioSpeechRequest(BaseModel):
     input: str  # 需要合成的文本
     model: str = "chattts-4w"
     voice: str = "female2"
-    response_format: Literal["mp3", "wav"] = "mp3"
+    response_format: AudioFormat = "mp3"
     speed: float = Field(1, ge=0.1, le=10, description="Speed of the audio")
     seed: int = 42
+
     temperature: float = 0.3
+    top_k: int = 20
+    top_p: float = 0.7
+
     style: str = ""
-    # 是否开启batch合成，小于等于1表示不适用batch
-    # 开启batch合成会自动分割句子
     batch_size: int = Field(1, ge=1, le=20, description="Batch size")
     spliter_threshold: float = Field(
         100, ge=10, le=1024, description="Threshold for sentence spliter"
@@ -44,6 +40,9 @@ class AudioSpeechRequest(BaseModel):
     # end of sentence
     eos: str = "[uv_break]"
 
+    enhance: bool = False
+    denoise: bool = False
+
 
 async def openai_speech_api(
     request: AudioSpeechRequest = Body(
@@ -55,7 +54,14 @@ async def openai_speech_api(
     voice = request.voice
     style = request.style
     eos = request.eos
+    seed = request.seed
+
     response_format = request.response_format
+    if not isinstance(response_format, AudioFormat) and isinstance(
+        response_format, str
+    ):
+        response_format = AudioFormat(response_format)
+
     batch_size = request.batch_size
     spliter_threshold = request.spliter_threshold
     speed = request.speed
@@ -71,49 +77,45 @@ async def openai_speech_api(
     except:
         raise HTTPException(status_code=400, detail="Invalid style.")
 
-    try:
-        # Normalize the text
-        text = text_normalize(input_text, is_end=True)
-
-        # Calculate speaker and style based on input voice
-        params = api_utils.calc_spk_style(spk=voice, style=style)
-
-        spk = params.get("spk", -1)
-        seed = params.get("seed", request.seed or 42)
-        temperature = params.get("temperature", request.temperature or 0.3)
-        prompt1 = params.get("prompt1", "")
-        prompt2 = params.get("prompt2", "")
-        prefix = params.get("prefix", "")
-
-        # Generate audio
-        sample_rate, audio_data = synthesize_audio(
-            text,
-            temperature=temperature,
-            top_P=0.7,
-            top_K=20,
-            spk=spk,
-            infer_seed=seed,
-            batch_size=batch_size,
-            spliter_threshold=spliter_threshold,
-            prompt1=prompt1,
-            prompt2=prompt2,
-            prefix=prefix,
-            end_of_sentence=eos,
-        )
+    ctx_params = api_utils.calc_spk_style(spk=voice, style=style)
 
-        if speed != 1:
-            audio_data = pyrb.time_stretch(audio_data, sample_rate, speed)
+    speaker = ctx_params.get("spk")
+    if not isinstance(speaker, Speaker):
+        raise HTTPException(status_code=400, detail="Invalid voice.")
 
-        # Convert audio data to wav format
-        buffer = io.BytesIO()
-        sf.write(buffer, audio_data, sample_rate, format="wav")
-        buffer.seek(0)
+    tts_config = ChatTTSConfig(
+        style=style,
+        temperature=request.temperature,
+        top_k=request.top_k,
+        top_p=request.top_p,
+    )
+    infer_config = InferConfig(
+        batch_size=batch_size,
+        spliter_threshold=spliter_threshold,
+        eos=eos,
+        seed=seed,
+    )
+    adjust_config = AdjustConfig(speaking_rate=speed)
+    enhancer_config = EnhancerConfig(
+        enabled=request.enhance or request.denoise or False,
+        lambd=0.9 if request.denoise else 0.1,
+    )
+    try:
+        handler = TTSHandler(
+            text_content=input_text,
+            spk=speaker,
+            tts_config=tts_config,
+            infer_config=infer_config,
+            adjust_config=adjust_config,
+            enhancer_config=enhancer_config,
+        )
 
-        if response_format == "mp3":
-            # Convert wav to mp3
-            buffer = api_utils.wav_to_mp3(buffer)
+        buffer = handler.enqueue_to_buffer(response_format)
 
-        return StreamingResponse(buffer, media_type="audio/mp3")
+        mime_type = f"audio/{response_format.value}"
+        if response_format == AudioFormat.mp3:
+            mime_type = "audio/mpeg"
+        return StreamingResponse(buffer, media_type=mime_type)
 
     except Exception as e:
         import logging
@@ -150,7 +152,6 @@ class TranscriptionsVerboseResponse(BaseModel):
 def setup(app: APIManager):
     app.post(
         "/v1/audio/speech",
-        response_class=FileResponse,
         description="""
 openai api document: 
 [https://platform.openai.com/docs/guides/text-to-speech](https://platform.openai.com/docs/guides/text-to-speech)

modules/api/impl/refiner_api.py

@@ -31,6 +31,7 @@ async def refiner_prompt_post(request: RefineTextRequest):
         text = request.text
         if request.normalize:
             text = text_normalize(request.text)
+        # TODO 其实这里可以做 spliter 和 batch 处理
         refined_text = refiner.refine_text(
             text=text,
             prompt=request.prompt,

modules/api/impl/ssml_api.py

@@ -1,27 +1,22 @@
 from fastapi import HTTPException, Body
 from fastapi.responses import StreamingResponse
 
-import io
 from pydantic import BaseModel
 from fastapi.responses import FileResponse
 
 
-from modules.normalization import text_normalize
-from modules.ssml_parser.SSMLParser import create_ssml_parser
-from modules.SynthesizeSegments import (
-    SynthesizeSegments,
-    combine_audio_segments,
-)
+from modules.api.impl.handler.SSMLHandler import SSMLHandler
+from modules.api.impl.model.audio_model import AdjustConfig, AudioFormat
+from modules.api.impl.model.chattts_model import InferConfig
+from modules.api.impl.model.enhancer_model import EnhancerConfig
 
 
-from modules.api import utils as api_utils
-
 from modules.api.Api import APIManager
 
 
 class SSMLRequest(BaseModel):
     ssml: str
-    format: str = "mp3"
+    format: AudioFormat = "mp3"
 
     # NOTE: 🤔 也许这个值应该配置成系统变量？ 传进来有点奇怪
     batch_size: int = 4
@@ -31,6 +26,9 @@ class SSMLRequest(BaseModel):
 
     spliter_thr: int = 100
 
+    enhancer: EnhancerConfig = EnhancerConfig()
+    adjuster: AdjustConfig = AdjustConfig()
+
 
 async def synthesize_ssml_api(
     request: SSMLRequest = Body(
@@ -43,6 +41,8 @@ async def synthesize_ssml_api(
         batch_size = request.batch_size
         eos = request.eos
         spliter_thr = request.spliter_thr
+        enhancer = request.enhancer
+        adjuster = request.adjuster
 
         if batch_size < 1:
             raise HTTPException(
@@ -62,22 +62,27 @@ async def synthesize_ssml_api(
                 status_code=400, detail="Format must be 'mp3' or 'wav'."
             )
 
-        parser = create_ssml_parser()
-        segments = parser.parse(ssml)
-        for seg in segments:
-            seg["text"] = text_normalize(seg["text"], is_end=True)
-
-        synthesize = SynthesizeSegments(
-            batch_size=batch_size, eos=eos, spliter_thr=spliter_thr
+        infer_config = InferConfig(
+            batch_size=batch_size,
+            spliter_threshold=spliter_thr,
+            eos=eos,
         )
-        audio_segments = synthesize.synthesize_segments(segments)
-        combined_audio = combine_audio_segments(audio_segments)
-        buffer = io.BytesIO()
-        combined_audio.export(buffer, format="wav")
-        buffer.seek(0)
-        if format == "mp3":
-            buffer = api_utils.wav_to_mp3(buffer)
-        return StreamingResponse(buffer, media_type=f"audio/{format}")
+        adjust_config = adjuster
+        enhancer_config = enhancer
+
+        handler = SSMLHandler(
+            ssml_content=ssml,
+            infer_config=infer_config,
+            adjust_config=adjust_config,
+            enhancer_config=enhancer_config,
+        )
+
+        buffer = handler.enqueue_to_buffer(format=request.format)
+
+        mime_type = f"audio/{format}"
+        if format == AudioFormat.mp3:
+            mime_type = "audio/mpeg"
+        return StreamingResponse(buffer, media_type=mime_type)
 
     except Exception as e:
         import logging

modules/api/impl/tts_api.py

@@ -1,17 +1,18 @@
 from fastapi import Depends, HTTPException, Query
 from fastapi.responses import StreamingResponse
 
-import io
 from pydantic import BaseModel
-import soundfile as sf
 from fastapi.responses import FileResponse
 
 
-from modules.normalization import text_normalize
+from modules.api.impl.handler.TTSHandler import TTSHandler
+from modules.api.impl.model.audio_model import AdjustConfig, AudioFormat
+from modules.api.impl.model.chattts_model import ChatTTSConfig, InferConfig
+from modules.api.impl.model.enhancer_model import EnhancerConfig
 
 from modules.api import utils as api_utils
 from modules.api.Api import APIManager
-from modules.synthesize_audio import synthesize_audio
+from modules.speaker import Speaker
 
 
 class TTSParams(BaseModel):
@@ -23,10 +24,10 @@ class TTSParams(BaseModel):
     temperature: float = Query(
         0.3, description="Temperature for sampling (may be overridden by style or spk)"
     )
-    top_P: float = Query(
+    top_p: float = Query(
         0.5, description="Top P for sampling (may be overridden by style or spk)"
     )
-    top_K: int = Query(
+    top_k: int = Query(
         20, description="Top K for sampling (may be overridden by style or spk)"
     )
     seed: int = Query(
@@ -38,7 +39,14 @@ class TTSParams(BaseModel):
     prefix: str = Query("", description="Text prefix for inference")
     bs: str = Query("8", description="Batch size for inference")
     thr: str = Query("100", description="Threshold for sentence spliter")
-    eos: str = Query("", description="End of sentence str")
+    eos: str = Query("[uv_break]", description="End of sentence str")
+
+    enhance: bool = Query(False, description="Enable enhancer")
+    denoise: bool = Query(False, description="Enable denoiser")
+
+    speed: float = Query(1.0, description="Speed of the audio")
+    pitch: float = Query(0, description="Pitch of the audio")
+    volume_gain: float = Query(0, description="Volume gain of the audio")
 
 
 async def synthesize_tts(params: TTSParams = Depends()):
@@ -55,18 +63,18 @@ async def synthesize_tts(params: TTSParams = Depends()):
                 status_code=422, detail="Temperature must be between 0 and 1"
             )
 
-        # Validate top_P
-        if not (0 <= params.top_P <= 1):
-            raise HTTPException(status_code=422, detail="top_P must be between 0 and 1")
+        # Validate top_p
+        if not (0 <= params.top_p <= 1):
+            raise HTTPException(status_code=422, detail="top_p must be between 0 and 1")
 
-        # Validate top_K
-        if params.top_K <= 0:
+        # Validate top_k
+        if params.top_k <= 0:
             raise HTTPException(
-                status_code=422, detail="top_K must be a positive integer"
+                status_code=422, detail="top_k must be a positive integer"
             )
-        if params.top_K > 100:
+        if params.top_k > 100:
             raise HTTPException(
-                status_code=422, detail="top_K must be less than or equal to 100"
+                status_code=422, detail="top_k must be less than or equal to 100"
             )
 
         # Validate format
@@ -76,11 +84,13 @@ async def synthesize_tts(params: TTSParams = Depends()):
                 detail="Invalid format. Supported formats are mp3 and wav",
             )
 
-        text = text_normalize(params.text, is_end=False)
-
         calc_params = api_utils.calc_spk_style(spk=params.spk, style=params.style)
 
         spk = calc_params.get("spk", params.spk)
+        if not isinstance(spk, Speaker):
+            raise HTTPException(status_code=422, detail="Invalid speaker")
+
+        style = calc_params.get("style", params.style)
         seed = params.seed or calc_params.get("seed", params.seed)
         temperature = params.temperature or calc_params.get(
             "temperature", params.temperature
@@ -93,29 +103,46 @@ async def synthesize_tts(params: TTSParams = Depends()):
         batch_size = int(params.bs)
         threshold = int(params.thr)
 
-        sample_rate, audio_data = synthesize_audio(
-            text,
+        tts_config = ChatTTSConfig(
+            style=style,
             temperature=temperature,
-            top_P=params.top_P,
-            top_K=params.top_K,
-            spk=spk,
-            infer_seed=seed,
+            top_k=params.top_k,
+            top_p=params.top_p,
+            prefix=prefix,
             prompt1=prompt1,
             prompt2=prompt2,
-            prefix=prefix,
+        )
+        infer_config = InferConfig(
             batch_size=batch_size,
             spliter_threshold=threshold,
-            end_of_sentence=eos,
+            eos=eos,
+            seed=seed,
+        )
+        adjust_config = AdjustConfig(
+            pitch=params.pitch,
+            speed_rate=params.speed,
+            volume_gain_db=params.volume_gain,
+        )
+        enhancer_config = EnhancerConfig(
+            enabled=params.enhance or params.denoise or False,
+            lambd=0.9 if params.denoise else 0.1,
         )
 
-        buffer = io.BytesIO()
-        sf.write(buffer, audio_data, sample_rate, format="wav")
-        buffer.seek(0)
+        handler = TTSHandler(
+            text_content=params.text,
+            spk=spk,
+            tts_config=tts_config,
+            infer_config=infer_config,
+            adjust_config=adjust_config,
+            enhancer_config=enhancer_config,
+        )
 
-        if format == "mp3":
-            buffer = api_utils.wav_to_mp3(buffer)
+        buffer = handler.enqueue_to_buffer(format=AudioFormat(params.format))
 
-        return StreamingResponse(buffer, media_type="audio/wav")
+        media_type = f"audio/{params.format}"
+        if params.format == "mp3":
+            media_type = "audio/mpeg"
+        return StreamingResponse(buffer, media_type=media_type)
 
     except Exception as e:
         import logging

modules/api/impl/xtts_v2_api.py

@@ -30,8 +30,19 @@ class XTTS_V2_Settings:
         self.top_k = 20
         self.enable_text_splitting = True
 
+        # 下面是额外配置 xtts_v2 中不包含的，但是本系统需要的
+        self.batch_size = 4
+        self.eos = "[uv_break]"
+        self.infer_seed = 42
+        self.use_decoder = True
+        self.prompt1 = ""
+        self.prompt2 = ""
+        self.prefix = ""
+        self.spliter_threshold = 100
+
 
 class TTSSettingsRequest(BaseModel):
+    # 这个 stream_chunk 现在当作 spliter_threshold 用
     stream_chunk_size: int
     temperature: float
     speed: float
@@ -41,6 +52,15 @@ class TTSSettingsRequest(BaseModel):
     top_k: int
     enable_text_splitting: bool
 
+    batch_size: int = None
+    eos: str = None
+    infer_seed: int = None
+    use_decoder: bool = None
+    prompt1: str = None
+    prompt2: str = None
+    prefix: str = None
+    spliter_threshold: int = None
+
 
 class SynthesisRequest(BaseModel):
     text: str
@@ -79,17 +99,22 @@ def setup(app: APIManager):
 
         text = text_normalize(text, is_end=True)
         sample_rate, audio_data = synthesize_audio(
-            text=text,
-            temperature=XTTSV2.temperature,
+            # TODO: 这两个参数现在用不着...但是其实gpt是可以用的
             # length_penalty=XTTSV2.length_penalty,
             # repetition_penalty=XTTSV2.repetition_penalty,
+            text=text,
+            temperature=XTTSV2.temperature,
             top_P=XTTSV2.top_p,
             top_K=XTTSV2.top_k,
             spk=spk,
-            spliter_threshold=XTTSV2.stream_chunk_size,
-            # TODO 支持设置 batch_size
-            batch_size=4,
-            end_of_sentence="[uv_break]",
+            spliter_threshold=XTTSV2.spliter_threshold,
+            batch_size=XTTSV2.batch_size,
+            end_of_sentence=XTTSV2.eos,
+            infer_seed=XTTSV2.infer_seed,
+            use_decoder=XTTSV2.use_decoder,
+            prompt1=XTTSV2.prompt1,
+            prompt2=XTTSV2.prompt2,
+            prefix=XTTSV2.prefix,
         )
 
         if XTTSV2.speed:
@@ -145,6 +170,8 @@ def setup(app: APIManager):
                 )
 
             XTTSV2.stream_chunk_size = request.stream_chunk_size
+            XTTSV2.spliter_threshold = request.stream_chunk_size
+
             XTTSV2.temperature = request.temperature
             XTTSV2.speed = request.speed
             XTTSV2.length_penalty = request.length_penalty
@@ -152,6 +179,25 @@ def setup(app: APIManager):
             XTTSV2.top_p = request.top_p
             XTTSV2.top_k = request.top_k
             XTTSV2.enable_text_splitting = request.enable_text_splitting
+
+            # TODO: checker
+            if request.batch_size:
+                XTTSV2.batch_size = request.batch_size
+            if request.eos:
+                XTTSV2.eos = request.eos
+            if request.infer_seed:
+                XTTSV2.infer_seed = request.infer_seed
+            if request.use_decoder:
+                XTTSV2.use_decoder = request.use_decoder
+            if request.prompt1:
+                XTTSV2.prompt1 = request.prompt1
+            if request.prompt2:
+                XTTSV2.prompt2 = request.prompt2
+            if request.prefix:
+                XTTSV2.prefix = request.prefix
+            if request.spliter_threshold:
+                XTTSV2.spliter_threshold = request.spliter_threshold
+
             return {"message": "Settings successfully applied"}
         except Exception as e:
             if isinstance(e, HTTPException):

modules/api/utils.py

@@ -1,9 +1,8 @@
 from pydantic import BaseModel
 from typing import Any, Union
 
-import torch
 
-from modules.speaker import Speaker, speaker_mgr
+from modules.speaker import speaker_mgr
 
 
 from modules.data import styles_mgr
@@ -13,18 +12,10 @@ from pydub import AudioSegment
 from modules.ssml import merge_prompt
 
 
-from enum import Enum
-
-
 class ParamsTypeError(Exception):
     pass
 
 
-class AudioFormat(str, Enum):
-    mp3 = "mp3"
-    wav = "wav"
-
-
 class BaseResponse(BaseModel):
     message: str
     data: Any
@@ -35,7 +26,7 @@ def success_response(data: Any, message: str = "ok") -> BaseResponse:
 
 
 def wav_to_mp3(wav_data, bitrate="48k"):
-    audio = AudioSegment.from_wav(
+    audio: AudioSegment = AudioSegment.from_wav(
         wav_data,
     )
     return audio.export(format="mp3", bitrate=bitrate)

modules/devices/devices.py

@@ -127,6 +127,12 @@ def reset_device():
     global dtype_gpt
     global dtype_decoder
 
+    if "all" in config.runtime_env_vars.use_cpu and not config.runtime_env_vars.no_half:
+        logger.warning(
+            "Cannot use half precision with CPU, using full precision instead"
+        )
+        config.runtime_env_vars.no_half = True
+
     if not config.runtime_env_vars.no_half:
         dtype = torch.float16
         dtype_dvae = torch.float16
@@ -144,7 +150,7 @@ def reset_device():
 
         logger.info("Using full precision: torch.float32")
 
-    if config.runtime_env_vars.use_cpu == "all":
+    if "all" in config.runtime_env_vars.use_cpu:
         device = cpu
     else:
         device = get_optimal_device()

modules/finetune/train_speaker.py

@@ -45,9 +45,10 @@ def train_speaker_embeddings(
             )
             for speaker in dataset.speakers
         }
-        for speaker_embed in speaker_embeds.values():
-            std, mean = chat.pretrain_models["spk_stat"].chunk(2)
-            speaker_embed.data = speaker_embed.data * std + mean
+
+    for speaker_embed in speaker_embeds.values():
+        std, mean = chat.pretrain_models["spk_stat"].chunk(2)
+        speaker_embed.data = speaker_embed.data * std + mean
 
     SPEAKER_TOKEN_ID = tokenizer.convert_tokens_to_ids("[spk_emb]")
     AUDIO_EOS_TOKEN_ID = 0
@@ -166,13 +167,13 @@ def train_speaker_embeddings(
                 audio_logits.flatten(0, 2), labels[:, text_len:].flatten(0, 2)
             )
             loss = audio_loss
-            if train_text:
-                text_logits = gpt.head_text(text_hidden_states)
-                text_loss = loss_fn(
-                    text_logits.flatten(0, 1), labels[:, 1:text_len, 0].flatten(0, 1)
-                )
-                loss += text_loss
-                logger.meters["text_loss"].update(text_loss.item(), n=batch_size)
+
+            text_logits = gpt.head_text(text_hidden_states)
+            text_loss = loss_fn(
+                text_logits.flatten(0, 1), labels[:, 1:text_len, 0].flatten(0, 1)
+            )
+            loss += text_loss
+            logger.meters["text_loss"].update(text_loss.item(), n=batch_size)
 
             gpt_gen_mel_specs = decoder_decoder(
                 audio_hidden_states[:, :-1].transpose(1, 2)
@@ -181,7 +182,12 @@ def train_speaker_embeddings(
             loss += 0.01 * mse_loss
 
             optimizer.zero_grad()
-            loss.backward()
+
+            if train_text:
+                # just for test
+                text_loss.backward()
+            else:
+                loss.backward()
             torch.nn.utils.clip_grad_norm_(speaker_embeds.values(), 1.0)
             optimizer.step()
             logger.meters["loss"].update(loss.item(), n=batch_size)
@@ -203,6 +209,7 @@ if __name__ == "__main__":
     from modules.speaker import Speaker
 
     config.runtime_env_vars.no_half = True
+    config.runtime_env_vars.use_cpu = []
     devices.reset_device()
 
     parser = argparse.ArgumentParser()

modules/prompts/news_oral_prompt.txt

@@ -0,0 +1,14 @@
+# 任务要求
+任务： 新闻稿口播化
+
+你需要将一个新闻稿改写为口语化的口播文本
+同时，适当的添加一些 附语言 标签为文本增加多样性
+
+目前可以使用的附语言标签如下:
+- `[laugh]`: 表示笑声
+- `[uv_break]`: 表示无声停顿
+- `[v_break]`: 表示有声停顿，如“嗯”、“啊”等
+- `[lbreak]`: 表示一个长停顿一般表示段落结束
+
+# 输入
+{{USER_INPUT}}

modules/prompts/podcast_prompt.txt

@@ -0,0 +1 @@
+TODO

modules/ssml_parser/SSMLParser.py

@@ -1,13 +1,10 @@
 from lxml import etree
 
 
-from typing import Any, List, Dict, Union
+from typing import List, Union
 import logging
 
-from modules.data import styles_mgr
-from modules.speaker import speaker_mgr
 from box import Box
-import copy
 
 
 class SSMLContext(Box):

modules/webui/speaker/speaker_editor.py

@@ -25,7 +25,7 @@ def speaker_editor_ui():
         spk: Speaker = Speaker.from_file(spk_file)
         spk.name = name
         spk.gender = gender
-        spk.desc = desc
+        spk.describe = desc
 
         with tempfile.NamedTemporaryFile(delete=False, suffix=".pt") as tmp_file:
             torch.save(spk, tmp_file)

modules/webui/speaker/speaker_merger.py

@@ -38,12 +38,8 @@ def merge_spk(
     tensor_c = spk_to_tensor(spk_c)
     tensor_d = spk_to_tensor(spk_d)
 
-    assert (
-        tensor_a is not None
-        or tensor_b is not None
-        or tensor_c is not None
-        or tensor_d is not None
-    ), "At least one speaker should be selected"
+    if tensor_a is None and tensor_b is None and tensor_c is None and tensor_d is None:
+        raise gr.Error("At least one speaker should be selected")
 
     merge_tensor = torch.zeros_like(
         tensor_a