added demo

README.md

@@ -1,14 +1,14 @@
 ---
-title: Indic Parler Tts
-emoji: 👀
-colorFrom: gray
-colorTo: pink
+title: Parler-TTS Streaming
+emoji: 📝
+colorFrom: red
+colorTo: indigo
 sdk: gradio
-sdk_version: 5.7.1
+sdk_version: 4.31.5
 app_file: app.py
 pinned: false
 license: apache-2.0
-short_description: A demo of Indic Parler-TTS
+short_description: High-fidelity Text-To-Speech
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,425 @@
+import io
+import os
+import math
+from queue import Queue
+from threading import Thread
+from typing import Optional
+
+import numpy as np
+import spaces
+import gradio as gr
+import torch
+
+from parler_tts import ParlerTTSForConditionalGeneration, ParlerTTSStreamer
+from pydub import AudioSegment
+from transformers import AutoTokenizer, AutoFeatureExtractor, set_seed
+
+device = "cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
+torch_dtype = torch.bfloat16 if device != "cpu" else torch.float32
+
+repo_id = "ai4bharat/indic-parler-tts-pretrained"
+jenny_repo_id = "ai4bharat/indic-parler-tts"
+
+model = ParlerTTSForConditionalGeneration.from_pretrained(
+    repo_id, attn_implementation="eager", torch_dtype=torch_dtype, low_cpu_mem_usage=True
+).to(device)
+jenny_model = ParlerTTSForConditionalGeneration.from_pretrained(
+    jenny_repo_id, attn_implementation="eager", torch_dtype=torch_dtype, low_cpu_mem_usage=True
+).to(device)
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id)
+description_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large")
+feature_extractor = AutoFeatureExtractor.from_pretrained(repo_id)
+
+SAMPLE_RATE = feature_extractor.sampling_rate
+SEED = 42
+
+default_text = "Please surprise me and speak in whatever voice you enjoy."
+examples = [
+    [
+        "मुले बागेत खेळत आहेत आणि पक्षी किलबिलाट करत आहेत.",
+        "Sunita speaks slowly in a calm, moderate-pitched voice, delivering the news with a neutral tone. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "ಉದ್ಯಾನದಲ್ಲಿ ಮಕ್ಕಳ ಆಟವಾಡುತ್ತಿದ್ದಾರೆ ಮತ್ತು ಪಕ್ಷಿಗಳು ಚಿಲಿಪಿಲಿ ಮಾಡುತ್ತಿವೆ.",
+        "Suresh speaks slowly in a low-pitched, calm voice, with a neutral tone, perfect for narration. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "বাচ্চারা বাগানে খেলছে আর পাখি কিচিরমিচির করছে।",
+        "Aditi speaks at a moderate pace and pitch, with a clear, neutral tone and no emotional emphasis. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "పిల్లలు తోటలో ఆడుకుంటున్నారు, పక్షుల కిలకిలరావాలు.",
+        "Prakash speaks slowly in a low-pitched, calm voice, with a neutral tone, perfect for narration. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "పిల్లలు తోటలో ఆడుకుంటున్నారు, పక్షుల కిలకిలరావాలు.",
+        "Prakash speaks slowly in a low-pitched, calm voice, with a neutral tone, perfect for narration. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "This is the best time of my life, Bartley,' she said happily",
+        "A male speaker with a low-pitched voice speaks with a British accent at a fast pace in a small, confined space with very clear audio and an animated tone.",
+        3.0
+    ],
+    [
+        "Montrose also, after having experienced still more variety of good and bad fortune, threw down his arms, and retired out of the kingdom.",
+        "A female speaker with a slightly low-pitched, quite monotone voice speaks with an American accent at a slightly faster-than-average pace in a confined space with very clear audio.",
+        3.0
+    ],
+    [
+        "बगीचे में बच्चे खेल रहे हैं और पक्षी चहचहा रहे हैं।",
+        "Rohit speaks with a slightly high-pitched voice delivering his words at a slightly slow pace in a small, confined space with a touch of background noise and a quite monotone tone.",
+        3.0
+    ],
+    [
+        "കുട്ടികൾ പൂന്തോട്ടത്തിൽ കളിക്കുന്നു, പക്ഷികൾ ചിലയ്ക്കുന്നു.",
+        "Anjali speaks with a low-pitched voice delivering her words at a fast pace and an animated tone, in a very spacious environment, accompanied by noticeable background noise.",
+        3.0
+    ],
+    [
+        "குழந்தைகள் தோட்டத்தில் விளையாடுகிறார்கள், பறவைகள் கிண்டல் செய்கின்றன.",
+        "Jaya speaks with a slightly low-pitched, quite monotone voice at a slightly faster-than-average pace in a confined space with very clear audio.",
+        3.0
+    ]
+]
+
+
+jenny_examples = [
+    [
+        "मुले बागेत खेळत आहेत आणि पक्षी किलबिलाट करत आहेत.",
+        "Sunita speaks slowly in a calm, moderate-pitched voice, delivering the news with a neutral tone. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "ಉದ್ಯಾನದಲ್ಲಿ ಮಕ್ಕಳ ಆಟವಾಡುತ್ತಿದ್ದಾರೆ ಮತ್ತು ಪಕ್ಷಿಗಳು ಚಿಲಿಪಿಲಿ ಮಾಡುತ್ತಿವೆ.",
+        "Suresh speaks slowly in a low-pitched, calm voice, with a neutral tone, perfect for narration. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "বাচ্চারা বাগানে খেলছে আর পাখি কিচিরমিচির করছে।",
+        "Aditi speaks at a moderate pace and pitch, with a clear, neutral tone and no emotional emphasis. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "పిల్లలు తోటలో ఆడుకుంటున్నారు, పక్షుల కిలకిలరావాలు.",
+        "Prakash speaks slowly in a low-pitched, calm voice, with a neutral tone, perfect for narration. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "పిల్లలు తోటలో ఆడుకుంటున్నారు, పక్షుల కిలకిలరావాలు.",
+        "Prakash speaks slowly in a low-pitched, calm voice, with a neutral tone, perfect for narration. The recording is very high quality with no background noise.",
+        3.0
+    ],
+    [
+        "This is the best time of my life, Bartley,' she said happily",
+        "A male speaker with a low-pitched voice speaks with a British accent at a fast pace in a small, confined space with very clear audio and an animated tone.",
+        3.0
+    ],
+    [
+        "Montrose also, after having experienced still more variety of good and bad fortune, threw down his arms, and retired out of the kingdom.",
+        "A female speaker with a slightly low-pitched, quite monotone voice speaks with an American accent at a slightly faster-than-average pace in a confined space with very clear audio.",
+        3.0
+    ],
+    [
+        "बगीचे में बच्चे खेल रहे हैं और पक्षी चहचहा रहे हैं।",
+        "Rohit speaks with a slightly high-pitched voice delivering his words at a slightly slow pace in a small, confined space with a touch of background noise and a quite monotone tone.",
+        3.0
+    ],
+    [
+        "കുട്ടികൾ പൂന്തോട്ടത്തിൽ കളിക്കുന്നു, പക്ഷികൾ ചിലയ്ക്കുന്നു.",
+        "Anjali speaks with a low-pitched voice delivering her words at a fast pace and an animated tone, in a very spacious environment, accompanied by noticeable background noise.",
+        3.0
+    ],
+    [
+        "குழந்தைகள் தோட்டத்தில் விளையாடுகிறார்கள், பறவைகள் கிண்டல் செய்கின்றன.",
+        "Jaya speaks with a slightly low-pitched, quite monotone voice at a slightly faster-than-average pace in a confined space with very clear audio.",
+        3.0
+    ]
+]
+
+
+def numpy_to_mp3(audio_array, sampling_rate):
+    # Normalize audio_array if it's floating-point
+    if np.issubdtype(audio_array.dtype, np.floating):
+        max_val = np.max(np.abs(audio_array))
+        audio_array = (audio_array / max_val) * 32767  # Normalize to 16-bit range
+        audio_array = audio_array.astype(np.int16)
+
+    # Create an audio segment from the numpy array
+    audio_segment = AudioSegment(
+        audio_array.tobytes(),
+        frame_rate=sampling_rate,
+        sample_width=audio_array.dtype.itemsize,
+        channels=1
+    )
+
+    # Export the audio segment to MP3 bytes - use a high bitrate to maximise quality
+    mp3_io = io.BytesIO()
+    audio_segment.export(mp3_io, format="mp3", bitrate="320k")
+
+    # Get the MP3 bytes
+    mp3_bytes = mp3_io.getvalue()
+    mp3_io.close()
+
+    return mp3_bytes
+
+sampling_rate = model.audio_encoder.config.sampling_rate
+frame_rate = model.audio_encoder.config.frame_rate
+
+# @spaces.GPU
+# def generate_base(text, description, play_steps_in_s=2.0):
+#     play_steps = int(frame_rate * play_steps_in_s)
+#     streamer = ParlerTTSStreamer(model, device=device, play_steps=play_steps)
+
+#     inputs = description_tokenizer(description, return_tensors="pt").to(device)
+#     prompt = tokenizer(text, return_tensors="pt").to(device)
+
+#     generation_kwargs = dict(
+#         input_ids=inputs.input_ids,
+#         prompt_input_ids=prompt.input_ids,
+#         streamer=streamer,
+#         do_sample=True,
+#         temperature=1.0,
+#         min_new_tokens=10,
+#     )
+
+#     set_seed(SEED)
+#     thread = Thread(target=model.generate, kwargs=generation_kwargs)
+#     thread.start()
+
+#     for new_audio in streamer:
+#         print(f"Sample of length: {round(new_audio.shape[0] / sampling_rate, 2)} seconds")
+#         yield numpy_to_mp3(new_audio, sampling_rate=sampling_rate)
+
+@spaces.GPU
+def generate_base(text, description, play_steps_in_s=2.0):
+    # Initialize variables
+    play_steps = int(frame_rate * play_steps_in_s)
+    chunk_size = 10  # Process 10 words at a time
+    
+    # Tokenize the full text and description
+    inputs = description_tokenizer(description, return_tensors="pt").to(device)
+    
+    # Split text into chunks of approximately 10 words
+    words = text.split()
+    chunks = [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
+    
+    all_audio = []
+    
+    # Process each chunk
+    for chunk in chunks:
+        # Tokenize the chunk
+        prompt = tokenizer(chunk, return_tensors="pt").to(device)
+        
+        # Generate audio for the chunk
+        generation = model.generate(
+            input_ids=inputs.input_ids,
+            attention_mask=inputs.attention_mask,
+            prompt_input_ids=prompt.input_ids,
+            prompt_attention_mask=prompt.attention_mask,
+            do_sample=True,
+            # temperature=1.0,
+            # min_new_tokens=10,
+            return_dict_in_generate=True
+        )
+            
+        # Extract audio from generation
+        if hasattr(generation, 'sequences') and hasattr(generation, 'audios_length'):
+            audio = generation.sequences[0, :generation.audios_length[0]]
+            audio_np = audio.to(torch.float32).cpu().numpy().squeeze()
+            if len(audio_np.shape) > 1:
+                audio_np = audio_np.flatten()
+            all_audio.append(audio_np)
+    
+    # Combine all audio chunks
+    combined_audio = np.concatenate(all_audio)
+    
+    # Convert to expected format and yield
+    print(f"Sample of length: {round(combined_audio.shape[0] / sampling_rate, 2)} seconds")
+    yield numpy_to_mp3(combined_audio, sampling_rate=sampling_rate)
+
+# @spaces.GPU
+# def generate_jenny(text, description, play_steps_in_s=2.0):
+#     play_steps = int(frame_rate * play_steps_in_s)
+#     streamer = ParlerTTSStreamer(jenny_model, device=device, play_steps=play_steps)
+
+#     inputs = description_tokenizer(description, return_tensors="pt").to(device)
+#     prompt = tokenizer(text, return_tensors="pt").to(device)
+
+#     generation_kwargs = dict(
+#         input_ids=inputs.input_ids,
+#         prompt_input_ids=prompt.input_ids,
+#         streamer=streamer,
+#         do_sample=True,
+#         temperature=1.0,
+#         min_new_tokens=10,
+#     )
+
+#     set_seed(SEED)
+#     thread = Thread(target=jenny_model.generate, kwargs=generation_kwargs)
+#     thread.start()
+
+#     for new_audio in streamer:
+#         print(f"Sample of length: {round(new_audio.shape[0] / sampling_rate, 2)} seconds")
+#         yield sampling_rate, new_audio
+
+@spaces.GPU
+def generate_jenny(text, description, play_steps_in_s=2.0):
+    # Initialize variables
+    play_steps = int(frame_rate * play_steps_in_s)
+    chunk_size = 10  # Process 10 words at a time
+    
+    # Tokenize the full text and description
+    inputs = description_tokenizer(description, return_tensors="pt").to(device)
+    
+    # Split text into chunks of approximately 10 words
+    words = text.split()
+    chunks = [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
+    
+    all_audio = []
+    
+    # Process each chunk
+    for chunk in chunks:
+        # Tokenize the chunk
+        prompt = tokenizer(chunk, return_tensors="pt").to(device)
+
+        # Generate audio for the chunk
+        generation = jenny_model.generate(
+            input_ids=inputs.input_ids,
+            attention_mask=inputs.attention_mask,
+            prompt_input_ids=prompt.input_ids,
+            prompt_attention_mask=prompt.attention_mask,
+            do_sample=True,
+            # temperature=1.0,
+            # min_new_tokens=10,
+            return_dict_in_generate=True
+        )
+
+        # Extract audio from generation
+        if hasattr(generation, 'sequences') and hasattr(generation, 'audios_length'):
+            audio = generation.sequences[0, :generation.audios_length[0]]
+            audio_np = audio.to(torch.float32).cpu().numpy().squeeze()
+            if len(audio_np.shape) > 1:
+                audio_np = audio_np.flatten()
+            all_audio.append(audio_np)
+    
+    # Combine all audio chunks
+    combined_audio = np.concatenate(all_audio)
+    
+    # Convert to expected format and yield
+    print(f"Sample of length: {round(combined_audio.shape[0] / sampling_rate, 2)} seconds")
+    yield numpy_to_mp3(combined_audio, sampling_rate=sampling_rate)
+
+
+css = """
+        #share-btn-container {
+            display: flex;
+            padding-left: 0.5rem !important;
+            padding-right: 0.5rem !important;
+            background-color: #000000;
+            justify-content: center;
+            align-items: center;
+            border-radius: 9999px !important; 
+            width: 13rem;
+            margin-top: 10px;
+            margin-left: auto;
+            flex: unset !important;
+        }
+        #share-btn {
+            all: initial;
+            color: #ffffff;
+            font-weight: 600;
+            cursor: pointer;
+            font-family: 'IBM Plex Sans', sans-serif;
+            margin-left: 0.5rem !important;
+            padding-top: 0.25rem !important;
+            padding-bottom: 0.25rem !important;
+            right:0;
+        }
+        #share-btn * {
+            all: unset !important;
+        }
+        #share-btn-container div:nth-child(-n+2){
+            width: auto !important;
+            min-height: 0px !important;
+        }
+        #share-btn-container .wrap {
+            display: none !important;
+        }
+"""
+with gr.Blocks(css=css) as block:
+    gr.HTML(
+        """
+            <div style="text-align: center; max-width: 700px; margin: 0 auto;">
+              <div
+                style="
+                  display: inline-flex; align-items: center; gap: 0.8rem; font-size: 1.75rem;
+                "
+              >
+                <h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;">
+                  Parler-TTS 🗣️
+                </h1>
+              </div>
+            </div>
+        """
+    )
+    gr.HTML(
+        f"""
+        <p><a href="https://github.com/huggingface/IndicParlerTTS">IndicParlerTTS</a> is a training and inference library for high-quality text-to-speech (TTS) models. This demonstration highlights the flexibility of the IndicParlerTTS model, which generates natural, expressive speech for over 22 Indian languages, using a simple text prompt to control features like speaker style, tone, pitch, pace, and more.</p>
+
+        <p>Tips for effective usage:
+        <ul>
+            <li>Use detailed captions to describe the speaker and desired characteristics (e.g., "Aditi speaks in a slightly expressive tone, with clear audio quality and a moderate pace.").</li>
+            <li>For best results, reference specific named speakers provided in the model card on the <a href="https://huggingface.co/IndicParlerTTS">model page</a>.</li>
+            <li>Include terms like <b>"very clear audio"</b> or <b>"slightly noisy audio"</b> to control the audio quality and background ambiance.</li>
+            <li>Punctuation can be used to shape prosody (e.g., commas add pauses for natural phrasing).</li>
+            <li>If unsure about what caption to use, you can start with: <b>"The speaker speaks naturally. The recording is very high quality with no background noise."</b></li>
+        </ul>
+        </p>
+        """
+    )
+
+    with gr.Tab("Finetuned"):
+        with gr.Row():
+            with gr.Column():
+                input_text = gr.Textbox(label="Input Text", lines=2, value=jenny_examples[0][0], elem_id="input_text")
+                description = gr.Textbox(label="Description", lines=2, value=jenny_examples[0][1], elem_id="input_description")
+                play_seconds = gr.Slider(3.0, 7.0, value=jenny_examples[0][2], step=2, label="Streaming interval in seconds", info="Lower = shorter chunks, lower latency, more codec steps")
+                run_button = gr.Button("Generate Audio", variant="primary")
+            with gr.Column():
+                audio_out = gr.Audio(label="Parler-TTS generation", format="mp3", elem_id="audio_out", streaming=True, autoplay=True)
+
+        inputs = [input_text, description, play_seconds]
+        outputs = [audio_out]
+        gr.Examples(examples=jenny_examples, fn=generate_jenny, inputs=inputs, outputs=outputs, cache_examples=False)
+        run_button.click(fn=generate_jenny, inputs=inputs, outputs=outputs, queue=True)
+
+    with gr.Tab("Pretrained"):
+        with gr.Row():
+            with gr.Column():
+                input_text = gr.Textbox(label="Input Text", lines=2, value=default_text, elem_id="input_text")
+                description = gr.Textbox(label="Description", lines=2, value="", elem_id="input_description")
+                play_seconds = gr.Slider(3.0, 7.0, value=3.0, step=2, label="Streaming interval in seconds", info="Lower = shorter chunks, lower latency, more codec steps")
+                run_button = gr.Button("Generate Audio", variant="primary")
+            with gr.Column():
+                audio_out = gr.Audio(label="Parler-TTS generation", format="mp3", elem_id="audio_out", streaming=True, autoplay=True)
+
+        inputs = [input_text, description, play_seconds]
+        outputs = [audio_out]
+        gr.Examples(examples=examples, fn=generate_base, inputs=inputs, outputs=outputs, cache_examples=False)
+        run_button.click(fn=generate_base, inputs=inputs, outputs=outputs, queue=True)
+
+
+    gr.HTML(
+        """
+        If you'd like to learn more about how the model was trained or explore fine-tuning it yourself, visit the <a href="https://github.com/huggingface/parler-tts">Parler-TTS</a> repository on GitHub. The Parler-TTS codebase and associated checkpoints are licensed under the <a href="https://github.com/huggingface/parler-tts/blob/main/LICENSE">Apache 2.0 license</a>.</p>
+                """
+    )
+
+block.queue()
+block.launch(share=True)

requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/huggingface/parler-tts.git
+accelerate