app.py

import gradio as gr
import torch as T
import torch.nn as nn
import torch.nn.functional as F
import torchaudio
import matplotlib.pyplot as plt
from utils import load_ckpt, print_colored
from tokenizer import make_tokenizer
from model import get_hertz_dev_config
from typing import Tuple
import numpy as np
import os

# Global variables for model and tokenizer
global_generator = None
global_tokenizer = None
default_audio_path = "testingtesting.wav"  # Your default audio file

def init_model(use_pure_audio_ablation: bool = False) -> Tuple[nn.Module, object]:
    """Initialize the model and tokenizer"""
    global global_generator, global_tokenizer
    
    if global_generator is not None and global_tokenizer is not None:
        return global_generator, global_tokenizer
        
    device = 'cuda' if T.cuda.is_available() else 'cpu'
    T.cuda.set_device(0) if device == 'cuda' else None
    
    print_colored("Initializing model and tokenizer...", "blue")
    global_tokenizer = make_tokenizer(device)
    model_config = get_hertz_dev_config(is_split=False, use_pure_audio_ablation=use_pure_audio_ablation)
    
    global_generator = model_config()
    global_generator = global_generator.eval().to(T.bfloat16).to(device)
    print_colored("Model initialization complete!", "green")
    
    return global_generator, global_tokenizer

def process_audio(audio_path: str, sr: int) -> T.Tensor:
    """Load and preprocess audio file"""
    audio_tensor, sr = torchaudio.load(audio_path)
    

    if audio_tensor.shape[0] == 2:
        audio_tensor = audio_tensor.mean(dim=0).unsqueeze(0)
    
    if sr != 16000:
        resampler = torchaudio.transforms.Resample(orig_freq=sr, new_freq=16000)
        audio_tensor = resampler(audio_tensor)
    
    max_samples = 16000 * 60 * 5  # 5 minutes
    if audio_tensor.shape[1] > max_samples:
        audio_tensor = audio_tensor[:, :max_samples]
    
    return audio_tensor.unsqueeze(0)

def generate_completion(
    audio_file,
    prompt_len_seconds: float = 3.0,
    num_completions: int = 5,
    generation_seconds: float = 20.0,
    token_temp: float = 0.8,
    categorical_temp: float = 0.5,
    gaussian_temp: float = 0.1,
    progress=gr.Progress(track_tqdm=True)
) -> list:
    """Generate audio completions from the input audio"""
    device = 'cuda' if T.cuda.is_available() else 'cpu'
    
    # Use existing model and tokenizer
    generator, audio_tokenizer = global_generator, global_tokenizer
    
    progress(0, desc="Processing input audio...")
    # Process input audio
    prompt_audio = process_audio(audio_file, sr=16000)
    prompt_len = int(prompt_len_seconds * 8)
    
    progress(0.2, desc="Encoding prompt...")
    # Encode prompt
    with T.autocast(device_type='cuda', dtype=T.bfloat16):
        encoded_prompt_audio = audio_tokenizer.latent_from_data(prompt_audio.to(device))
    
    completions = []
    for i in range(num_completions):
        progress((i + 1) / num_completions, desc=f"Generating completion {i+1}/{num_completions}")
        
        # Generate completion
        encoded_prompt = encoded_prompt_audio[:, :prompt_len]
        with T.autocast(device_type='cuda', dtype=T.bfloat16):
            completed_audio_batch = generator.completion(
                encoded_prompt,
                temps=(token_temp, (categorical_temp, gaussian_temp)),
                use_cache=True,
                gen_len=int(generation_seconds * 8)
            )
            
            decoded_completion = audio_tokenizer.data_from_latent(completed_audio_batch.bfloat16())
        
        # Process audio for output
        audio_tensor = decoded_completion.cpu().squeeze()
        if audio_tensor.ndim == 1:
            audio_tensor = audio_tensor.unsqueeze(0)
        audio_tensor = audio_tensor.float()
        
        if audio_tensor.abs().max() > 1:
            audio_tensor = audio_tensor / audio_tensor.abs().max()
        
        # Trim to include only the generated portion
        output_audio = audio_tensor[:, max(prompt_len*2000 - 16000, 0):]
        completions.append((16000, output_audio.numpy().T))
    
    progress(1.0, desc="Generation complete!")
    return completions

def create_interface():
    # Initialize model at startup
    init_model()
    
    with gr.Blocks(title="Audio Completion Generator") as app:
        gr.Markdown("""
        # Audio Completion Generator
        Upload an audio file (or use the default) and generate AI completions based on the prompt.
        """)
        
        with gr.Row():
            with gr.Column():
                # Load the default audio if it exists
                default_value = default_audio_path if os.path.exists(default_audio_path) else None
                
                audio_input = gr.Audio(
                    label="Input Audio",
                    type="filepath",
                    sources=["microphone", "upload"],
                    value=default_value
                )
                
                with gr.Row():
                    prompt_len = gr.Slider(
                        minimum=1,
                        maximum=10,
                        value=3,
                        step=0.5,
                        label="Prompt Length (seconds)"
                    )
                    default_num_completions = 5
                    num_completions = gr.Slider(
                        minimum=1,
                        maximum=10,
                        value=default_num_completions,
                        step=1,
                        label="Number of Completions"
                    )
                    gen_length = gr.Slider(
                        minimum=5,
                        maximum=60,
                        value=20,
                        step=5,
                        label="Generation Length (seconds)"
                    )
                
                with gr.Row():
                    token_temp = gr.Slider(
                        minimum=0.1,
                        maximum=1.0,
                        value=0.8,
                        step=0.1,
                        label="Token Temperature"
                    )
                    cat_temp = gr.Slider(
                        minimum=0.1,
                        maximum=1.0,
                        value=0.5,
                        step=0.1,
                        label="Categorical Temperature"
                    )
                    gauss_temp = gr.Slider(
                        minimum=0.1,
                        maximum=1.0,
                        value=0.1,
                        step=0.1,
                        label="Gaussian Temperature"
                    )
                
                generate_btn = gr.Button("Generate Completions")
                status_text = gr.Markdown("Ready")
            
            with gr.Column():
                output_audios = []
                for i in range(10):  # Create 10 audio components
                    output_audios.append(gr.Audio(
                        label=f"Generated Completion {i+1}",
                        type="numpy",
                        visible=False
                    ))

        def update_visibility(num):
            return [gr.update(visible=(i < num)) for i in range(10)]

        def generate_with_status(*args):
            status_text.value = "Processing input audio..."
            completions = generate_completion(*args)
            status_text.value = "Generation complete!"
            
            # Prepare outputs for all audio components
            outputs = []
            for i in range(10):
                if i < len(completions):
                    outputs.append(completions[i])
                else:
                    outputs.append(None)
            return outputs

        # Set initial visibility on load
        app.load(
            fn=update_visibility,
            inputs=[num_completions],
            outputs=output_audios
        )

        # Update visibility when slider changes
        num_completions.change(
            fn=update_visibility,
            inputs=[num_completions],
            outputs=output_audios
        )
        
        generate_btn.click(
            fn=generate_with_status,
            inputs=[
                audio_input,
                prompt_len,
                num_completions,
                gen_length,
                token_temp,
                cat_temp,
                gauss_temp
            ],
            outputs=output_audios
        )
    
    return app

if __name__ == "__main__":
    app = create_interface()
    app.launch(share=True)