models.py

import torch
from diffusers import DDIMScheduler, CosineDPMSolverMultistepScheduler
from diffusers.schedulers.scheduling_dpmsolver_sde import BrownianTreeNoiseSampler
from diffusers import AudioLDM2Pipeline, StableAudioPipeline
from transformers import RobertaTokenizer, RobertaTokenizerFast, VitsTokenizer
from diffusers.models.unets.unet_2d_condition import UNet2DConditionOutput
from diffusers.models.embeddings import get_1d_rotary_pos_embed
from typing import Any, Dict, List, Optional, Tuple, Union
import gradio as gr


class PipelineWrapper(torch.nn.Module):
    def __init__(self, model_id: str,
                 device: torch.device,
                 double_precision: bool = False,
                 token: Optional[str] = None, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        self.model_id = model_id
        self.device = device
        self.double_precision = double_precision
        self.token = token

    def get_sigma(self, timestep: int) -> float:
        sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.model.scheduler.alphas_cumprod - 1)
        return sqrt_recipm1_alphas_cumprod[timestep]

    def load_scheduler(self) -> None:
        pass

    def get_fn_STFT(self) -> torch.nn.Module:
        pass

    def get_sr(self) -> int:
        return 16000

    def vae_encode(self, x: torch.Tensor) -> torch.Tensor:
        pass

    def vae_decode(self, x: torch.Tensor) -> torch.Tensor:
        pass

    def decode_to_mel(self, x: torch.Tensor) -> torch.Tensor:
        pass

    def setup_extra_inputs(self, *args, **kwargs) -> None:
        pass

    def encode_text(self, prompts: List[str], **kwargs
                    ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]:
        pass

    def get_variance(self, timestep: torch.Tensor, prev_timestep: torch.Tensor) -> torch.Tensor:
        pass

    def get_alpha_prod_t_prev(self, prev_timestep: torch.Tensor) -> torch.Tensor:
        pass

    def get_noise_shape(self, x0: torch.Tensor, num_steps: int) -> Tuple[int, ...]:
        variance_noise_shape = (num_steps,
                                self.model.unet.config.in_channels,
                                x0.shape[-2],
                                x0.shape[-1])
        return variance_noise_shape

    def sample_xts_from_x0(self, x0: torch.Tensor, num_inference_steps: int = 50) -> torch.Tensor:
        """
        Samples from P(x_1:T|x_0)
        """
        alpha_bar = self.model.scheduler.alphas_cumprod
        sqrt_one_minus_alpha_bar = (1-alpha_bar) ** 0.5

        variance_noise_shape = self.get_noise_shape(x0, num_inference_steps + 1)
        timesteps = self.model.scheduler.timesteps.to(self.device)
        t_to_idx = {int(v): k for k, v in enumerate(timesteps)}
        xts = torch.zeros(variance_noise_shape).to(x0.device)
        xts[0] = x0
        for t in reversed(timesteps):
            idx = num_inference_steps - t_to_idx[int(t)]
            xts[idx] = x0 * (alpha_bar[t] ** 0.5) + torch.randn_like(x0) * sqrt_one_minus_alpha_bar[t]

        return xts

    def get_zs_from_xts(self, xt: torch.Tensor, xtm1: torch.Tensor, noise_pred: torch.Tensor,
                        t: torch.Tensor, eta: float = 0, numerical_fix: bool = True, **kwargs
                        ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
        # pred of x0
        alpha_bar = self.model.scheduler.alphas_cumprod
        if self.model.scheduler.config.prediction_type == 'epsilon':
            pred_original_sample = (xt - (1 - alpha_bar[t]) ** 0.5 * noise_pred) / alpha_bar[t] ** 0.5
        elif self.model.scheduler.config.prediction_type == 'v_prediction':
            pred_original_sample = (alpha_bar[t] ** 0.5) * xt - ((1 - alpha_bar[t]) ** 0.5) * noise_pred

        # direction to xt
        prev_timestep = t - self.model.scheduler.config.num_train_timesteps // \
            self.model.scheduler.num_inference_steps

        alpha_prod_t_prev = self.get_alpha_prod_t_prev(prev_timestep)
        variance = self.get_variance(t, prev_timestep)

        if self.model.scheduler.config.prediction_type == 'epsilon':
            radom_noise_pred = noise_pred
        elif self.model.scheduler.config.prediction_type == 'v_prediction':
            radom_noise_pred = (alpha_bar[t] ** 0.5) * noise_pred + ((1 - alpha_bar[t]) ** 0.5) * xt

        pred_sample_direction = (1 - alpha_prod_t_prev - eta * variance) ** (0.5) * radom_noise_pred

        mu_xt = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction

        z = (xtm1 - mu_xt) / (eta * variance ** 0.5)

        # correction to avoid error accumulation
        if numerical_fix:
            xtm1 = mu_xt + (eta * variance ** 0.5)*z

        return z, xtm1, None

    def reverse_step_with_custom_noise(self, model_output: torch.Tensor, timestep: torch.Tensor, sample: torch.Tensor,
                                       variance_noise: Optional[torch.Tensor] = None, eta: float = 0, **kwargs
                                       ) -> torch.Tensor:
        # 1. get previous step value (=t-1)
        prev_timestep = timestep - self.model.scheduler.config.num_train_timesteps // \
            self.model.scheduler.num_inference_steps
        # 2. compute alphas, betas
        alpha_prod_t = self.model.scheduler.alphas_cumprod[timestep]
        alpha_prod_t_prev = self.get_alpha_prod_t_prev(prev_timestep)
        beta_prod_t = 1 - alpha_prod_t
        # 3. compute predicted original sample from predicted noise also called
        # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
        if self.model.scheduler.config.prediction_type == 'epsilon':
            pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5)
        elif self.model.scheduler.config.prediction_type == 'v_prediction':
            pred_original_sample = (alpha_prod_t ** 0.5) * sample - (beta_prod_t ** 0.5) * model_output

        # 5. compute variance: "sigma_t(η)" -> see formula (16)
        # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1)
        # variance = self.scheduler._get_variance(timestep, prev_timestep)
        variance = self.get_variance(timestep, prev_timestep)
        # std_dev_t = eta * variance ** (0.5)
        # Take care of asymetric reverse process (asyrp)
        if self.model.scheduler.config.prediction_type == 'epsilon':
            model_output_direction = model_output
        elif self.model.scheduler.config.prediction_type == 'v_prediction':
            model_output_direction = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample
        # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
        # pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * model_output_direction
        pred_sample_direction = (1 - alpha_prod_t_prev - eta * variance) ** (0.5) * model_output_direction
        # 7. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
        prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction
        # 8. Add noice if eta > 0
        if eta > 0:
            if variance_noise is None:
                variance_noise = torch.randn(model_output.shape, device=self.device)
            sigma_z = eta * variance ** (0.5) * variance_noise
            prev_sample = prev_sample + sigma_z

        return prev_sample

    def unet_forward(self,
                     sample: torch.FloatTensor,
                     timestep: Union[torch.Tensor, float, int],
                     encoder_hidden_states: torch.Tensor,
                     class_labels: Optional[torch.Tensor] = None,
                     timestep_cond: Optional[torch.Tensor] = None,
                     attention_mask: Optional[torch.Tensor] = None,
                     cross_attention_kwargs: Optional[Dict[str, Any]] = None,
                     added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
                     down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
                     mid_block_additional_residual: Optional[torch.Tensor] = None,
                     encoder_attention_mask: Optional[torch.Tensor] = None,
                     replace_h_space: Optional[torch.Tensor] = None,
                     replace_skip_conns: Optional[Dict[int, torch.Tensor]] = None,
                     return_dict: bool = True,
                     zero_out_resconns: Optional[Union[int, List]] = None) -> Tuple:
        pass


class AudioLDM2Wrapper(PipelineWrapper):
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        if self.double_precision:
            self.model = AudioLDM2Pipeline.from_pretrained(self.model_id, torch_dtype=torch.float64, token=self.token
                                                           ).to(self.device)
        else:
            try:
                self.model = AudioLDM2Pipeline.from_pretrained(self.model_id, local_files_only=True, token=self.token
                                                               ).to(self.device)
            except FileNotFoundError:
                self.model = AudioLDM2Pipeline.from_pretrained(self.model_id, local_files_only=False, token=self.token
                                                               ).to(self.device)

    def load_scheduler(self) -> None:
        self.model.scheduler = DDIMScheduler.from_pretrained(self.model_id, subfolder="scheduler")

    def get_fn_STFT(self) -> torch.nn.Module:
        from audioldm.audio import TacotronSTFT
        return TacotronSTFT(
            filter_length=1024,
            hop_length=160,
            win_length=1024,
            n_mel_channels=64,
            sampling_rate=16000,
            mel_fmin=0,
            mel_fmax=8000,
        )

    def vae_encode(self, x: torch.Tensor) -> torch.Tensor:
        # self.model.vae.disable_tiling()
        if x.shape[2] % 4:
            x = torch.nn.functional.pad(x, (0, 0, 4 - (x.shape[2] % 4), 0))
        return (self.model.vae.encode(x).latent_dist.mode() * self.model.vae.config.scaling_factor).float()
        # return (self.encode_no_tiling(x).latent_dist.mode() * self.model.vae.config.scaling_factor).float()

    def vae_decode(self, x: torch.Tensor) -> torch.Tensor:
        return self.model.vae.decode(1 / self.model.vae.config.scaling_factor * x).sample

    def decode_to_mel(self, x: torch.Tensor) -> torch.Tensor:
        if self.double_precision:
            tmp = self.model.mel_spectrogram_to_waveform(x[:, 0].detach().double()).detach()
        tmp = self.model.mel_spectrogram_to_waveform(x[:, 0].detach().float()).detach()
        if len(tmp.shape) == 1:
            tmp = tmp.unsqueeze(0)
        return tmp

    def encode_text(self, prompts: List[str], negative: bool = False, 
                    save_compute: bool = False, cond_length: int = 0, **kwargs
                    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        tokenizers = [self.model.tokenizer, self.model.tokenizer_2]
        text_encoders = [self.model.text_encoder, self.model.text_encoder_2]
        prompt_embeds_list = []
        attention_mask_list = []

        for tokenizer, text_encoder in zip(tokenizers, text_encoders):
            text_inputs = tokenizer(
                prompts,
                padding="max_length" if (save_compute and negative) or isinstance(tokenizer, (RobertaTokenizer, RobertaTokenizerFast))
                else True,
                max_length=tokenizer.model_max_length
                if (not save_compute) or ((not negative) or isinstance(tokenizer, (RobertaTokenizer, RobertaTokenizerFast, VitsTokenizer)))
                else cond_length,
                truncation=True,
                return_tensors="pt",
            )

            text_input_ids = text_inputs.input_ids
            attention_mask = text_inputs.attention_mask
            untruncated_ids = tokenizer(prompts, padding="longest", return_tensors="pt").input_ids

            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] \
                    and not torch.equal(text_input_ids, untruncated_ids):
                removed_text = tokenizer.batch_decode(
                    untruncated_ids[:, tokenizer.model_max_length - 1: -1])
                print(f"The following part of your input was truncated because {text_encoder.config.model_type} can "
                      f"only handle sequences up to {tokenizer.model_max_length} tokens: {removed_text}"
                      )

            text_input_ids = text_input_ids.to(self.device)
            attention_mask = attention_mask.to(self.device)

            with torch.no_grad():
                if text_encoder.config.model_type == "clap":
                    prompt_embeds = text_encoder.get_text_features(
                        text_input_ids,
                        attention_mask=attention_mask,
                    )
                    # append the seq-len dim: (bs, hidden_size) -> (bs, seq_len, hidden_size)
                    prompt_embeds = prompt_embeds[:, None, :]
                    # make sure that we attend to this single hidden-state
                    attention_mask = attention_mask.new_ones((len(prompts), 1))
                else:
                    prompt_embeds = text_encoder(
                        text_input_ids,
                        attention_mask=attention_mask,
                    )
                    prompt_embeds = prompt_embeds[0]

            prompt_embeds_list.append(prompt_embeds)
            attention_mask_list.append(attention_mask)

        # print(f'prompt[0].shape: {prompt_embeds_list[0].shape}')
        # print(f'prompt[1].shape: {prompt_embeds_list[1].shape}')
        # print(f'attn[0].shape: {attention_mask_list[0].shape}')
        # print(f'attn[1].shape: {attention_mask_list[1].shape}')

        projection_output = self.model.projection_model(
            hidden_states=prompt_embeds_list[0],
            hidden_states_1=prompt_embeds_list[1],
            attention_mask=attention_mask_list[0],
            attention_mask_1=attention_mask_list[1],
        )
        projected_prompt_embeds = projection_output.hidden_states
        projected_attention_mask = projection_output.attention_mask

        generated_prompt_embeds = self.model.generate_language_model(
            projected_prompt_embeds,
            attention_mask=projected_attention_mask,
            max_new_tokens=None,
        )

        prompt_embeds = prompt_embeds.to(dtype=self.model.text_encoder_2.dtype, device=self.device)
        attention_mask = (
            attention_mask.to(device=self.device)
            if attention_mask is not None
            else torch.ones(prompt_embeds.shape[:2], dtype=torch.long, device=self.device)
        )
        generated_prompt_embeds = generated_prompt_embeds.to(dtype=self.model.language_model.dtype, device=self.device)

        return generated_prompt_embeds, prompt_embeds, attention_mask

    def get_variance(self, timestep: torch.Tensor, prev_timestep: torch.Tensor) -> torch.Tensor:
        alpha_prod_t = self.model.scheduler.alphas_cumprod[timestep]
        alpha_prod_t_prev = self.get_alpha_prod_t_prev(prev_timestep)
        beta_prod_t = 1 - alpha_prod_t
        beta_prod_t_prev = 1 - alpha_prod_t_prev
        variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev)
        return variance

    def get_alpha_prod_t_prev(self, prev_timestep: torch.Tensor) -> torch.Tensor:
        return self.model.scheduler.alphas_cumprod[prev_timestep] if prev_timestep >= 0 \
            else self.model.scheduler.final_alpha_cumprod

    def unet_forward(self,
                     sample: torch.FloatTensor,
                     timestep: Union[torch.Tensor, float, int],
                     encoder_hidden_states: torch.Tensor,
                     timestep_cond: Optional[torch.Tensor] = None,
                     class_labels: Optional[torch.Tensor] = None,
                     attention_mask: Optional[torch.Tensor] = None,
                     encoder_attention_mask: Optional[torch.Tensor] = None,
                     return_dict: bool = True,
                     cross_attention_kwargs: Optional[Dict[str, Any]] = None,
                     mid_block_additional_residual: Optional[torch.Tensor] = None,
                     replace_h_space: Optional[torch.Tensor] = None,
                     replace_skip_conns: Optional[Dict[int, torch.Tensor]] = None,
                     zero_out_resconns: Optional[Union[int, List]] = None) -> Tuple:

        # Translation
        encoder_hidden_states_1 = class_labels
        class_labels = None
        encoder_attention_mask_1 = encoder_attention_mask
        encoder_attention_mask = None

        # return self.model.unet(sample, timestep,
        #                        encoder_hidden_states=generated_prompt_embeds,
        #                        encoder_hidden_states_1=encoder_hidden_states_1,
        #                        encoder_attention_mask_1=encoder_attention_mask_1,
        #                        ), None, None

        # By default samples have to be AT least a multiple of the overall upsampling factor.
        # The overall upsampling factor is equal to 2 ** (# num of upsampling layers).
        # However, the upsampling interpolation output size can be forced to fit any upsampling size
        # on the fly if necessary.
        default_overall_up_factor = 2 ** self.model.unet.num_upsamplers

        # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
        forward_upsample_size = False
        upsample_size = None

        if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
            # print("Forward upsample size to force interpolation output size.")
            forward_upsample_size = True

        # ensure attention_mask is a bias, and give it a singleton query_tokens dimension
        # expects mask of shape:
        #   [batch, key_tokens]
        # adds singleton query_tokens dimension:
        #   [batch,                    1, key_tokens]
        # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
        #   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
        #   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
        if attention_mask is not None:
            # assume that mask is expressed as:
            #   (1 = keep,      0 = discard)
            # convert mask into a bias that can be added to attention scores:
            #       (keep = +0,     discard = -10000.0)
            attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
            attention_mask = attention_mask.unsqueeze(1)

        # convert encoder_attention_mask to a bias the same way we do for attention_mask
        if encoder_attention_mask is not None:
            encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0
            encoder_attention_mask = encoder_attention_mask.unsqueeze(1)

        if encoder_attention_mask_1 is not None:
            encoder_attention_mask_1 = (1 - encoder_attention_mask_1.to(sample.dtype)) * -10000.0
            encoder_attention_mask_1 = encoder_attention_mask_1.unsqueeze(1)

        # 1. time
        timesteps = timestep
        if not torch.is_tensor(timesteps):
            is_mps = sample.device.type == "mps"
            if isinstance(timestep, float):
                dtype = torch.float32 if is_mps else torch.float64
            else:
                dtype = torch.int32 if is_mps else torch.int64
            timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
        elif len(timesteps.shape) == 0:
            timesteps = timesteps[None].to(sample.device)

        # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
        timesteps = timesteps.expand(sample.shape[0])

        t_emb = self.model.unet.time_proj(timesteps)

        # `Timesteps` does not contain any weights and will always return f32 tensors
        # but time_embedding might actually be running in fp16. so we need to cast here.
        # there might be better ways to encapsulate this.
        t_emb = t_emb.to(dtype=sample.dtype)

        emb = self.model.unet.time_embedding(t_emb, timestep_cond)
        aug_emb = None

        if self.model.unet.class_embedding is not None:
            if class_labels is None:
                raise ValueError("class_labels should be provided when num_class_embeds > 0")

            if self.model.unet.config.class_embed_type == "timestep":
                class_labels = self.model.unet.time_proj(class_labels)

                # `Timesteps` does not contain any weights and will always return f32 tensors
                # there might be better ways to encapsulate this.
                class_labels = class_labels.to(dtype=sample.dtype)

            class_emb = self.model.unet.class_embedding(class_labels).to(dtype=sample.dtype)

            if self.model.unet.config.class_embeddings_concat:
                emb = torch.cat([emb, class_emb], dim=-1)
            else:
                emb = emb + class_emb

        emb = emb + aug_emb if aug_emb is not None else emb

        if self.model.unet.time_embed_act is not None:
            emb = self.model.unet.time_embed_act(emb)

        # 2. pre-process
        sample = self.model.unet.conv_in(sample)

        # 3. down
        down_block_res_samples = (sample,)
        for downsample_block in self.model.unet.down_blocks:
            if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention:
                sample, res_samples = downsample_block(
                    hidden_states=sample,
                    temb=emb,
                    encoder_hidden_states=encoder_hidden_states,
                    attention_mask=attention_mask,
                    cross_attention_kwargs=cross_attention_kwargs,
                    encoder_attention_mask=encoder_attention_mask,
                    encoder_hidden_states_1=encoder_hidden_states_1,
                    encoder_attention_mask_1=encoder_attention_mask_1,
                )
            else:
                sample, res_samples = downsample_block(hidden_states=sample, temb=emb)

            down_block_res_samples += res_samples

        # 4. mid
        if self.model.unet.mid_block is not None:
            sample = self.model.unet.mid_block(
                sample,
                emb,
                encoder_hidden_states=encoder_hidden_states,
                attention_mask=attention_mask,
                cross_attention_kwargs=cross_attention_kwargs,
                encoder_attention_mask=encoder_attention_mask,
                encoder_hidden_states_1=encoder_hidden_states_1,
                encoder_attention_mask_1=encoder_attention_mask_1,
            )

        if replace_h_space is None:
            h_space = sample.clone()
        else:
            h_space = replace_h_space
            sample = replace_h_space.clone()

        if mid_block_additional_residual is not None:
            sample = sample + mid_block_additional_residual

        extracted_res_conns = {}
        # 5. up
        for i, upsample_block in enumerate(self.model.unet.up_blocks):
            is_final_block = i == len(self.model.unet.up_blocks) - 1

            res_samples = down_block_res_samples[-len(upsample_block.resnets):]
            down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]
            if replace_skip_conns is not None and replace_skip_conns.get(i):
                res_samples = replace_skip_conns.get(i)

            if zero_out_resconns is not None:
                if (type(zero_out_resconns) is int and i >= (zero_out_resconns - 1)) or \
                        type(zero_out_resconns) is list and i in zero_out_resconns:
                    res_samples = [torch.zeros_like(x) for x in res_samples]
                # down_block_res_samples = [torch.zeros_like(x) for x in down_block_res_samples]

            extracted_res_conns[i] = res_samples

            # if we have not reached the final block and need to forward the
            # upsample size, we do it here
            if not is_final_block and forward_upsample_size:
                upsample_size = down_block_res_samples[-1].shape[2:]

            if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
                sample = upsample_block(
                    hidden_states=sample,
                    temb=emb,
                    res_hidden_states_tuple=res_samples,
                    encoder_hidden_states=encoder_hidden_states,
                    cross_attention_kwargs=cross_attention_kwargs,
                    upsample_size=upsample_size,
                    attention_mask=attention_mask,
                    encoder_attention_mask=encoder_attention_mask,
                    encoder_hidden_states_1=encoder_hidden_states_1,
                    encoder_attention_mask_1=encoder_attention_mask_1,
                )
            else:
                sample = upsample_block(
                    hidden_states=sample, temb=emb, res_hidden_states_tuple=res_samples, upsample_size=upsample_size
                )

        # 6. post-process
        if self.model.unet.conv_norm_out:
            sample = self.model.unet.conv_norm_out(sample)
            sample = self.model.unet.conv_act(sample)
        sample = self.model.unet.conv_out(sample)

        if not return_dict:
            return (sample,)

        return UNet2DConditionOutput(sample=sample), h_space, extracted_res_conns


class StableAudWrapper(PipelineWrapper):
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        try:
            self.model = StableAudioPipeline.from_pretrained(self.model_id, token=self.token, local_files_only=True
                                                             ).to(self.device)
        except FileNotFoundError:
            self.model = StableAudioPipeline.from_pretrained(self.model_id, token=self.token, local_files_only=False
                                                             ).to(self.device)
        self.model.transformer.eval()
        self.model.vae.eval()

        if self.double_precision:
            self.model = self.model.to(torch.float64)

    def load_scheduler(self) -> None:
        self.model.scheduler = CosineDPMSolverMultistepScheduler.from_pretrained(
            self.model_id, subfolder="scheduler", token=self.token)

    def encode_text(self, prompts: List[str], negative: bool = False, **kwargs) -> Tuple[torch.Tensor, None, torch.Tensor]:
        text_inputs = self.model.tokenizer(
            prompts,
            padding="max_length",
            max_length=self.model.tokenizer.model_max_length,
            truncation=True,
            return_tensors="pt",
        )

        text_input_ids = text_inputs.input_ids.to(self.device)
        attention_mask = text_inputs.attention_mask.to(self.device)

        self.model.text_encoder.eval()
        with torch.no_grad():
            prompt_embeds = self.model.text_encoder(text_input_ids, attention_mask=attention_mask)[0]

        if negative and attention_mask is not None:  # set the masked tokens to the null embed
            prompt_embeds = torch.where(attention_mask.to(torch.bool).unsqueeze(2), prompt_embeds, 0.0)

        prompt_embeds = self.model.projection_model(text_hidden_states=prompt_embeds).text_hidden_states

        if attention_mask is None:
            raise gr.Error("Shouldn't reach here. Please raise an issue if you do.")
            """prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
            if attention_mask is not None and negative_attention_mask is None:
                negative_attention_mask = torch.ones_like(attention_mask)
            elif attention_mask is None and negative_attention_mask is not None:
                attention_mask = torch.ones_like(negative_attention_mask)"""

        if prompts == [""]:  # empty
            return torch.zeros_like(prompt_embeds, device=prompt_embeds.device), None, None

        prompt_embeds = prompt_embeds * attention_mask.unsqueeze(-1).to(prompt_embeds.dtype)
        prompt_embeds = prompt_embeds * attention_mask.unsqueeze(-1).to(prompt_embeds.dtype)
        return prompt_embeds, None, attention_mask

    def get_fn_STFT(self) -> torch.nn.Module:
        from audioldm.audio import TacotronSTFT
        return TacotronSTFT(
            filter_length=1024,
            hop_length=160,
            win_length=1024,
            n_mel_channels=64,
            sampling_rate=44100,
            mel_fmin=0,
            mel_fmax=22050,
        )

    def vae_encode(self, x: torch.Tensor) -> torch.Tensor:
        x = x.unsqueeze(0)

        audio_vae_length = int(self.model.transformer.config.sample_size * self.model.vae.hop_length)
        audio_shape = (1, self.model.vae.config.audio_channels, audio_vae_length)

        # check num_channels
        if x.shape[1] == 1 and self.model.vae.config.audio_channels == 2:
            x = x.repeat(1, 2, 1)

        audio_length = x.shape[-1]
        audio = x.new_zeros(audio_shape)
        audio[:, :, : min(audio_length, audio_vae_length)] = x[:, :, :audio_vae_length]

        encoded_audio = self.model.vae.encode(audio.to(self.device)).latent_dist
        encoded_audio = encoded_audio.sample()
        return encoded_audio

    def vae_decode(self, x: torch.Tensor) -> torch.Tensor:
        torch.cuda.empty_cache()
        # return self.model.vae.decode(1 / self.model.vae.config.scaling_factor * x).sample
        aud = self.model.vae.decode(x).sample
        return aud[:, :, self.waveform_start:self.waveform_end]

    def setup_extra_inputs(self, x: torch.Tensor, init_timestep: torch.Tensor,
                           extra_info: Optional[Any] = None,
                           audio_start_in_s: float = 0, audio_end_in_s: Optional[float] = None,
                           save_compute: bool = False) -> None:
        max_audio_length_in_s = self.model.transformer.config.sample_size * self.model.vae.hop_length / \
            self.model.vae.config.sampling_rate
        if audio_end_in_s is None:
            audio_end_in_s = max_audio_length_in_s

        if audio_end_in_s - audio_start_in_s > max_audio_length_in_s:
            raise ValueError(
                f"The total audio length requested ({audio_end_in_s-audio_start_in_s}s) is longer "
                f"than the model maximum possible length ({max_audio_length_in_s}). "
                f"Make sure that 'audio_end_in_s-audio_start_in_s<={max_audio_length_in_s}'."
            )

        self.waveform_start = int(audio_start_in_s * self.model.vae.config.sampling_rate)
        self.waveform_end = int(audio_end_in_s * self.model.vae.config.sampling_rate)

        self.seconds_start_hidden_states, self.seconds_end_hidden_states = self.model.encode_duration(
            audio_start_in_s, audio_end_in_s, self.device, False, 1)

        if save_compute:
            self.seconds_start_hidden_states = torch.cat([self.seconds_start_hidden_states, self.seconds_start_hidden_states], dim=0)
            self.seconds_end_hidden_states = torch.cat([self.seconds_end_hidden_states, self.seconds_end_hidden_states], dim=0)

        self.audio_duration_embeds = torch.cat([self.seconds_start_hidden_states,
                                                self.seconds_end_hidden_states], dim=2)

        # 7. Prepare rotary positional embedding
        self.rotary_embedding = get_1d_rotary_pos_embed(
            self.model.rotary_embed_dim,
            x.shape[2] + self.audio_duration_embeds.shape[1],
            use_real=True,
            repeat_interleave_real=False,
        )

        self.model.scheduler._init_step_index(init_timestep)

        # fix lower_order_nums for the reverse step - Option 1: only start from first order
        # self.model.scheduler.lower_order_nums = 0
        # self.model.scheduler.model_outputs = [None] * self.model.scheduler.config.solver_order
        # fix lower_order_nums for the reverse step - Option 2: start from the correct order with history
        t_to_idx = {float(v): k for k, v in enumerate(self.model.scheduler.timesteps)}
        idx = len(self.model.scheduler.timesteps) - t_to_idx[float(init_timestep)] - 1
        self.model.scheduler.model_outputs = [None, extra_info[idx] if extra_info is not None else None]
        self.model.scheduler.lower_order_nums = min(self.model.scheduler.step_index,
                                                    self.model.scheduler.config.solver_order)

        # if rand check:
        #     x *= self.model.scheduler.init_noise_sigma
        # return x

    def sample_xts_from_x0(self, x0: torch.Tensor, num_inference_steps: int = 50) -> torch.Tensor:
        """
        Samples from P(x_1:T|x_0)
        """

        sigmas = self.model.scheduler.sigmas
        shapes = self.get_noise_shape(x0, num_inference_steps + 1)
        xts = torch.zeros(shapes).to(x0.device)
        xts[0] = x0

        timesteps = self.model.scheduler.timesteps.to(self.device)
        t_to_idx = {float(v): k for k, v in enumerate(timesteps)}
        for t in reversed(timesteps):
            # idx = t_to_idx[int(t)]
            idx = num_inference_steps - t_to_idx[float(t)]
            n = torch.randn_like(x0)
            xts[idx] = x0 + n * sigmas[t_to_idx[float(t)]]
        return xts

    def get_zs_from_xts(self, xt: torch.Tensor, xtm1: torch.Tensor, data_pred: torch.Tensor,
                        t: torch.Tensor, numerical_fix: bool = True, first_order: bool = False, **kwargs
                        ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
        # pred of x0
        sigmas = self.model.scheduler.sigmas
        timesteps = self.model.scheduler.timesteps
        solver_order = self.model.scheduler.config.solver_order

        if self.model.scheduler.step_index is None:
            self.model.scheduler._init_step_index(t)
        curr_step_index = self.model.scheduler.step_index

        # Improve numerical stability for small number of steps
        lower_order_final = (curr_step_index == len(timesteps) - 1) and (
            self.model.scheduler.config.euler_at_final
            or (self.model.scheduler.config.lower_order_final and len(timesteps) < 15)
            or self.model.scheduler.config.final_sigmas_type == "zero")
        lower_order_second = ((curr_step_index == len(timesteps) - 2) and
                              self.model.scheduler.config.lower_order_final and len(timesteps) < 15)

        data_pred = self.model.scheduler.convert_model_output(data_pred, sample=xt)
        for i in range(solver_order - 1):
            self.model.scheduler.model_outputs[i] = self.model.scheduler.model_outputs[i + 1]
        self.model.scheduler.model_outputs[-1] = data_pred

        # instead of brownian noise, here we calculate the noise ourselves
        if (curr_step_index == len(timesteps) - 1) and self.model.scheduler.config.final_sigmas_type == "zero":
            z = torch.zeros_like(xt)
        elif first_order or solver_order == 1 or self.model.scheduler.lower_order_nums < 1 or lower_order_final:
            sigma_t, sigma_s = sigmas[curr_step_index + 1], sigmas[curr_step_index]
            h = torch.log(sigma_s) - torch.log(sigma_t)
            z = (xtm1 - (sigma_t / sigma_s * torch.exp(-h)) * xt - (1 - torch.exp(-2.0 * h)) * data_pred) \
                / (sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)))
        elif solver_order == 2 or self.model.scheduler.lower_order_nums < 2 or lower_order_second:
            sigma_t = sigmas[curr_step_index + 1]
            sigma_s0 = sigmas[curr_step_index]
            sigma_s1 = sigmas[curr_step_index - 1]
            m0, m1 = self.model.scheduler.model_outputs[-1], self.model.scheduler.model_outputs[-2]
            h, h_0 = torch.log(sigma_s0) - torch.log(sigma_t), torch.log(sigma_s1) - torch.log(sigma_s0)
            r0 = h_0 / h
            D0, D1 = m0, (1.0 / r0) * (m0 - m1)

            # sde-dpmsolver++
            z = (xtm1 - (sigma_t / sigma_s0 * torch.exp(-h)) * xt
                 - (1 - torch.exp(-2.0 * h)) * D0
                 - 0.5 * (1 - torch.exp(-2.0 * h)) * D1) \
                / (sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)))

        # correction to avoid error accumulation
        if numerical_fix:
            if first_order or solver_order == 1 or self.model.scheduler.lower_order_nums < 1 or lower_order_final:
                xtm1 = self.model.scheduler.dpm_solver_first_order_update(data_pred, sample=xt, noise=z)
            elif solver_order == 2 or self.model.scheduler.lower_order_nums < 2 or lower_order_second:
                xtm1 = self.model.scheduler.multistep_dpm_solver_second_order_update(
                    self.model.scheduler.model_outputs, sample=xt, noise=z)
            # If not perfect recon - maybe TODO fix self.model.scheduler.model_outputs as well?

        if self.model.scheduler.lower_order_nums < solver_order:
            self.model.scheduler.lower_order_nums += 1
        # upon completion increase step index by one
        self.model.scheduler._step_index += 1

        return z, xtm1, self.model.scheduler.model_outputs[-2]

    def get_sr(self) -> int:
        return self.model.vae.config.sampling_rate

    def get_noise_shape(self, x0: torch.Tensor, num_steps: int) -> Tuple[int, int, int]:
        variance_noise_shape = (num_steps,
                                self.model.transformer.config.in_channels,
                                int(self.model.transformer.config.sample_size))
        return variance_noise_shape

    def reverse_step_with_custom_noise(self, model_output: torch.Tensor, timestep: torch.Tensor, sample: torch.Tensor,
                                       variance_noise: Optional[torch.Tensor] = None,
                                       first_order: bool = False, **kwargs
                                       ) -> torch.Tensor:
        if self.model.scheduler.step_index is None:
            self.model.scheduler._init_step_index(timestep)

        # Improve numerical stability for small number of steps
        lower_order_final = (self.model.scheduler.step_index == len(self.model.scheduler.timesteps) - 1) and (
            self.model.scheduler.config.euler_at_final
            or (self.model.scheduler.config.lower_order_final and len(self.model.scheduler.timesteps) < 15)
            or self.model.scheduler.config.final_sigmas_type == "zero"
        )
        lower_order_second = (
            (self.model.scheduler.step_index == len(self.model.scheduler.timesteps) - 2) and
            self.model.scheduler.config.lower_order_final and len(self.model.scheduler.timesteps) < 15
        )

        model_output = self.model.scheduler.convert_model_output(model_output, sample=sample)
        for i in range(self.model.scheduler.config.solver_order - 1):
            self.model.scheduler.model_outputs[i] = self.model.scheduler.model_outputs[i + 1]
        self.model.scheduler.model_outputs[-1] = model_output

        if variance_noise is None:
            if self.model.scheduler.noise_sampler is None:
                self.model.scheduler.noise_sampler = BrownianTreeNoiseSampler(
                    model_output, sigma_min=self.model.scheduler.config.sigma_min,
                    sigma_max=self.model.scheduler.config.sigma_max, seed=None)
            variance_noise = self.model.scheduler.noise_sampler(
                self.model.scheduler.sigmas[self.model.scheduler.step_index],
                self.model.scheduler.sigmas[self.model.scheduler.step_index + 1]).to(model_output.device)

        if first_order or self.model.scheduler.config.solver_order == 1 or \
                self.model.scheduler.lower_order_nums < 1 or lower_order_final:
            prev_sample = self.model.scheduler.dpm_solver_first_order_update(
                model_output, sample=sample, noise=variance_noise)
        elif self.model.scheduler.config.solver_order == 2 or \
                self.model.scheduler.lower_order_nums < 2 or lower_order_second:
            prev_sample = self.model.scheduler.multistep_dpm_solver_second_order_update(
                self.model.scheduler.model_outputs, sample=sample, noise=variance_noise)

        if self.model.scheduler.lower_order_nums < self.model.scheduler.config.solver_order:
            self.model.scheduler.lower_order_nums += 1

        # upon completion increase step index by one
        self.model.scheduler._step_index += 1

        return prev_sample

    def unet_forward(self,
                     sample: torch.FloatTensor,
                     timestep: Union[torch.Tensor, float, int],
                     encoder_hidden_states: torch.Tensor,
                     encoder_attention_mask: Optional[torch.Tensor] = None,
                     return_dict: bool = True,
                     **kwargs) -> Tuple:

        # Create text_audio_duration_embeds and audio_duration_embeds
        embeds = torch.cat([encoder_hidden_states, self.seconds_start_hidden_states, self.seconds_end_hidden_states],
                           dim=1)
        if encoder_attention_mask is None:
            # handle the batched case
            if embeds.shape[0] > 1:
                embeds[0] = torch.zeros_like(embeds[0], device=embeds.device)
            else:
                embeds = torch.zeros_like(embeds, device=embeds.device)

        noise_pred = self.model.transformer(sample,
                                            timestep.unsqueeze(0),
                                            encoder_hidden_states=embeds,
                                            global_hidden_states=self.audio_duration_embeds,
                                            rotary_embedding=self.rotary_embedding)

        if not return_dict:
            return (noise_pred.sample,)

        return noise_pred, None, None


def load_model(model_id: str, device: torch.device,
               double_precision: bool = False, token: Optional[str] = None) -> PipelineWrapper:
    if 'audioldm2' in model_id:
        ldm_stable = AudioLDM2Wrapper(model_id=model_id, device=device, double_precision=double_precision, token=token)
    elif 'stable-audio' in model_id:
        ldm_stable = StableAudWrapper(model_id=model_id, device=device, double_precision=double_precision, token=token)
    ldm_stable.load_scheduler()
    torch.cuda.empty_cache()
    return ldm_stable