Duplicate from Lajonbot/Marketplace-audio

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+muzyka_AI.mp4 filter=lfs diff=lfs merge=lfs -text
+voice_cloning_fraud.mp4 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Lp Music Caps
+emoji: 🎵🎵🎵
+colorFrom: purple
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.33.1
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: Lajonbot/Marketplace-audio
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,300 @@
+from transformers import VitsModel, AutoTokenizer
+import soundfile as sf
+import torch
+from datetime import datetime
+import random
+import time
+from ctransformers import AutoModelForCausalLM
+from datetime import datetime
+import whisper
+from transformers import VitsModel, AutoTokenizer
+import torch
+from transformers import MusicgenForConditionalGeneration, AutoProcessor, set_seed
+import torch
+import numpy as np
+import os
+import argparse
+import gradio as gr
+from timeit import default_timer as timer
+import torch
+import numpy as np
+import pandas as pd
+from huggingface_hub import hf_hub_download
+from model.bart import BartCaptionModel
+from utils.audio_utils import load_audio, STR_CH_FIRST
+from diffusers import DiffusionPipeline
+
+from PIL import Image
+
+def image_grid(imgs, rows, cols):
+    assert len(imgs) == rows*cols
+
+    w, h = imgs[0].size
+    grid = Image.new('RGB', size=(cols*w, rows*h))
+    grid_w, grid_h = grid.size
+    
+    for i, img in enumerate(imgs):
+        grid.paste(img, box=(i%cols*w, i//cols*h))
+    return grid
+
+
+
+def save_to_txt(text_to_save):
+    with open('prompt.txt', 'w', encoding='utf-8') as f:
+        f.write(text_to_save)
+        
+def read_txt():
+    with open('prompt.txt') as f:
+        lines = f.readlines()
+    return lines 
+
+##### Chat z LLAMA ####
+##### Chat z LLAMA ####
+##### Chat z LLAMA ####
+params = {
+        "max_new_tokens":512,
+        "stop":["<end>" ,"<|endoftext|>","[", "<user>"],
+        "temperature":0.7,
+        "top_p":0.8,
+        "stream":True,
+        "batch_size": 8}
+
+
+whisper_model = whisper.load_model("medium").to("cuda")
+print("Whisper Loaded!")
+llm = AutoModelForCausalLM.from_pretrained("Aspik101/trurl-2-7b-pl-instruct_GGML", model_type="llama")
+print("LLM Loaded!")
+tts_model = VitsModel.from_pretrained("facebook/mms-tts-pol")
+tts_model.to("cuda")
+print("TTS Loaded!")
+tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-pol")
+
+pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0",
+                                         torch_dtype=torch.float16,
+                                         use_safetensors=True,
+                                         variant="fp16").to("cuda")
+print("DiffusionPipeline Loaded!")
+
+model_audio_gen = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small").to("cuda")
+processor_audio_gen = AutoProcessor.from_pretrained("facebook/musicgen-small")
+
+with gr.Blocks() as chat_demo:
+    chatbot = gr.Chatbot()
+    audio_input = gr.Audio(source="microphone", type="filepath", show_label=False)
+    submit_audio = gr.Button("Submit Audio")
+    clear = gr.Button("Clear")
+    audio_output = gr.Audio('temp_file.wav', label="Generated Audio (wav)", type='filepath', autoplay=False)
+    
+    def translate(audio):
+        print("__Wysyłam nagranie do whisper!")
+        transcription = whisper_model.transcribe(audio, language="pl")
+        return transcription["text"]
+    
+    def read_text(text):
+        print("Tutaj jest tekst to przeczytania!", text[-1][-1])
+        inputs = tokenizer(text[-1][-1], return_tensors="pt").to("cuda")
+        with torch.no_grad():
+            output = tts_model(**inputs).waveform.squeeze().cpu().numpy()
+        sf.write('temp_file.wav', output, tts_model.config.sampling_rate)
+        return 'temp_file.wav'
+    
+    def user(audio_data, history):
+        if audio_data:
+            user_message = translate(audio_data)
+            print("USER!:")
+            print("", history + [[user_message, None]])
+            return history + [[user_message, None]]
+
+    def parse_history(hist):
+        history_ = ""
+        for q, a in hist:
+            history_ += f"<user>: {q } \n"
+            if a:
+                history_ += f"<assistant>: {a} \n"
+        return history_
+
+    def bot(history):
+        print(f"When: {datetime.today().strftime('%Y-%m-%d %H:%M:%S')}")
+        prompt = f"Jesteś AI assystentem. Odpowiadaj krótko i po polsku. {parse_history(history)}. <assistant>:"
+        stream = llm(prompt, **params)
+        history[-1][1] = ""
+        answer_save = ""
+        for character in stream:
+            history[-1][1] += character
+            answer_save += character
+            time.sleep(0.005)
+            yield history
+
+    submit_audio.click(user, [audio_input, chatbot], [chatbot], queue=False).then(bot, chatbot, chatbot).then(read_text, chatbot, audio_output)
+    clear.click(lambda: None, None, chatbot, queue=False)
+
+
+##### Audio Gen ####
+##### Audio Gen ####
+##### Audio Gen ####
+
+
+
+sampling_rate = model_audio_gen.audio_encoder.config.sampling_rate
+frame_rate = model_audio_gen.audio_encoder.config.frame_rate
+text_encoder = model_audio_gen.get_text_encoder()
+
+def generate_audio(decade, genre, instrument, guidance_scale=8, audio_length_in_s=20, seed=0):
+    prompt = " ".join([decade, genre, 'track with ', instrument])
+    save_to_txt(prompt)
+    inputs = processor_audio_gen(
+        text=[prompt, "drums"],
+        padding=True,
+        return_tensors="pt",
+        ).to(device)
+
+    with torch.no_grad():
+        encoder_outputs = text_encoder(**inputs)
+
+    max_new_tokens = int(frame_rate * audio_length_in_s)
+
+    set_seed(seed)
+    audio_values = model_audio_gen.generate(inputs.input_ids[0][None, :], attention_mask=inputs.attention_mask, encoder_outputs=encoder_outputs, do_sample=True, guidance_scale=guidance_scale, max_new_tokens=max_new_tokens)
+    sf.write('generated_audio.wav', audio_values.cpu()[0][0], 32_000)
+    audio_values = (audio_values.cpu().numpy() * 32767).astype(np.int16)
+    return (sampling_rate, audio_values)
+
+
+
+audio_gen = gr.Interface(
+    fn=generate_audio,
+    inputs=[
+        # gr.Text(label="Negative prompt", value="drums"),
+        gr.Radio(["50s", " 60s", "70s", "80s", "90s"], label="decade", info=""),
+        gr.Radio(["classic", "rock", "pop", "metal", "jazz", "synth"], label="genre", info=""),
+        gr.Radio(["acoustic guitar", "electric guitar", "drums", "saxophone", "keyboard", "accordion", "fiddle"], label="instrument", info=""),
+        gr.Slider(1.5, 10, value=8, step=0.5, label="Guidance scale"),
+        gr.Slider(5, 30, value=20, step=5, label="Audio length in s"),
+        # gr.Slider(0, 10, value=0, step=1, label="Seed"),
+    ],
+    outputs=[
+        gr.Audio(label="Generated Music", type="numpy"),
+    ]#,
+    # examples=EXAMPLES,
+)
+
+#### Audio desc and Stable ###
+#### Audio desc and Stable ###
+#### Audio desc and Stable ###
+
+if os.path.isfile("transfer.pth") == False:
+    torch.hub.download_url_to_file('https://huggingface.co/seungheondoh/lp-music-caps/resolve/main/transfer.pth', 'transfer.pth')
+    torch.hub.download_url_to_file('https://huggingface.co/seungheondoh/lp-music-caps/resolve/main/folk.wav', 'folk.wav')
+    torch.hub.download_url_to_file('https://huggingface.co/seungheondoh/lp-music-caps/resolve/main/electronic.mp3', 'electronic.mp3')
+    torch.hub.download_url_to_file('https://huggingface.co/seungheondoh/lp-music-caps/resolve/main/orchestra.wav', 'orchestra.wav')
+
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+example_list = ['folk.wav', 'electronic.mp3', 'orchestra.wav']
+model = BartCaptionModel(max_length = 128)
+pretrained_object = torch.load('./transfer.pth', map_location='cpu')
+state_dict = pretrained_object['state_dict']
+model.load_state_dict(state_dict)
+if torch.cuda.is_available():
+    torch.cuda.set_device(device)
+    model = model.cuda(device)
+model.eval()
+
+
+
+
+
+def get_audio(audio_path, duration=10, target_sr=16000):
+    n_samples = int(duration * target_sr)
+    audio, sr = load_audio(
+        path= audio_path,
+        ch_format= STR_CH_FIRST,
+        sample_rate= target_sr,
+        downmix_to_mono= True,
+    )
+    if len(audio.shape) == 2:
+        audio = audio.mean(0, False)  # to mono
+    input_size = int(n_samples)
+    if audio.shape[-1] < input_size:  # pad sequence
+        pad = np.zeros(input_size)
+        pad[: audio.shape[-1]] = audio
+        audio = pad
+    ceil = int(audio.shape[-1] // n_samples)
+    audio = torch.from_numpy(np.stack(np.split(audio[:ceil * n_samples], ceil)).astype('float32'))
+    return audio
+
+def captioning(audio_path):
+    audio_tensor = get_audio(audio_path = audio_path)
+    if torch.cuda.is_available():
+        audio_tensor = audio_tensor.to(device)
+    with torch.no_grad():
+        output = model.generate(
+            samples=audio_tensor,
+            num_beams=5,
+        )
+    inference = ""
+    number_of_chunks = range(audio_tensor.shape[0])
+    for chunk, text in zip(number_of_chunks, output):
+        time = f"[{chunk * 10}:00-{(chunk + 1) * 10}:00]"
+        inference += f"{time}\n{text} \n \n"
+    return inference
+
+title = ""
+description = ""
+
+article = ""
+def captioning():
+    audio_path = 'generated_audio.wav'
+    audio_tensor = get_audio(audio_path=audio_path)
+    
+    if torch.cuda.is_available():
+        audio_tensor = audio_tensor.to(device)
+    
+    with torch.no_grad():
+        output = model.generate(
+            samples=audio_tensor,
+            num_beams=5)
+    
+    inference = ""
+    number_of_chunks = range(audio_tensor.shape[0])
+    for chunk, text in zip(number_of_chunks, output):
+        time = f"[{chunk * 10}:00-{(chunk + 1) * 10}:00]"
+        inference += f"{time}\n{text}  \n \n"
+    
+    prompt = read_txt()
+    print(prompt[0])
+    # Generuj obraz na podstawie tekstu
+    #generated_images = pipe(prompt=prompt[0]*5 + inference + prompt[0]*5).images
+    #image = generated_images[0]
+    
+    num_images = 3
+    prompt = [prompt[0]*5 + inference + prompt[0]*5] * num_images
+    images = pipe(prompt,  height=768, width=768).images
+    grid = image_grid(images, rows=1, cols=3)
+    
+    return inference, grid
+
+audio_desc = gr.Interface(fn=captioning,
+                          inputs=None,
+                          outputs=[
+                              gr.Textbox(label="Caption generated by LP-MusicCaps Transfer Model"),
+                              gr.Image(label="Generated Image")  # Dodane wyjście dla obrazu
+                          ],
+                          title=title,
+                          description=description,
+                          article=article, 
+                          cache_examples=False
+                          )
+
+music = gr.Video("muzyka_AI.mp4")
+voice_cloning = gr.Video("voice_cloning_fraud.mp4")
+
+##### Run Alll #######
+##### Run Alll #######
+##### Run Alll #######
+
+
+demo_all = gr.TabbedInterface([music, audio_gen, audio_desc, voice_cloning, chat_demo], ["1.Music", "2.Audio Generation", "3.Image Generation", "4.Voice Cloning", "5.Chat with LLama"])
+
+demo_all.queue()
+demo_all.launch()

model/bart.py

@@ -0,0 +1,151 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+from .modules import AudioEncoder
+from transformers import BartForConditionalGeneration, BartTokenizer, BartConfig
+
+class BartCaptionModel(nn.Module):
+    def __init__(self, n_mels=128, num_of_conv=6, sr=16000, duration=10, max_length=128, label_smoothing=0.1, bart_type="facebook/bart-base", audio_dim=768):
+        super(BartCaptionModel, self).__init__()
+        # non-finetunning case
+        bart_config = BartConfig.from_pretrained(bart_type)
+        self.tokenizer = BartTokenizer.from_pretrained(bart_type)
+        self.bart = BartForConditionalGeneration(bart_config)
+        
+        self.n_sample = sr * duration
+        self.hop_length = int(0.01 * sr) # hard coding hop_size
+        self.n_frames = int(self.n_sample // self.hop_length)
+        self.num_of_stride_conv = num_of_conv - 1
+        self.n_ctx = int(self.n_frames // 2**self.num_of_stride_conv) + 1
+        self.audio_encoder = AudioEncoder(
+            n_mels = n_mels, # hard coding n_mel
+            n_ctx = self.n_ctx, 
+            audio_dim = audio_dim, 
+            text_dim = self.bart.config.hidden_size,
+            num_of_stride_conv = self.num_of_stride_conv
+        )
+
+        self.max_length = max_length
+        self.loss_fct = nn.CrossEntropyLoss(label_smoothing= label_smoothing, ignore_index=-100)
+
+    @property
+    def device(self):
+        return list(self.parameters())[0].device
+
+    def shift_tokens_right(self, input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
+        """
+        Shift input ids one token to the right.ls
+        """
+        shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+        shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
+        shifted_input_ids[:, 0] = decoder_start_token_id
+
+        if pad_token_id is None:
+            raise ValueError("self.model.config.pad_token_id has to be defined.")
+        # replace possible -100 values in labels by `pad_token_id`
+        shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+        return shifted_input_ids
+
+    def forward_encoder(self, audio):
+        audio_embs = self.audio_encoder(audio)
+        encoder_outputs = self.bart.model.encoder(
+            input_ids=None,
+            inputs_embeds=audio_embs,
+            return_dict=True
+        )["last_hidden_state"]
+        return encoder_outputs, audio_embs
+
+    def forward_decoder(self, text, encoder_outputs):
+        text = self.tokenizer(text,
+                              padding='longest',
+                              truncation=True,
+                              max_length=self.max_length,
+                              return_tensors="pt")
+        input_ids = text["input_ids"].to(self.device)
+        attention_mask = text["attention_mask"].to(self.device)
+
+        decoder_targets = input_ids.masked_fill(
+            input_ids == self.tokenizer.pad_token_id, -100
+        )
+
+        decoder_input_ids = self.shift_tokens_right(
+            decoder_targets, self.bart.config.pad_token_id, self.bart.config.decoder_start_token_id
+        )
+
+        decoder_outputs = self.bart(
+            input_ids=None,
+            attention_mask=None,
+            decoder_input_ids=decoder_input_ids,
+            decoder_attention_mask=attention_mask,
+            inputs_embeds=None,
+            labels=None,
+            encoder_outputs=(encoder_outputs,),
+            return_dict=True
+        )
+        lm_logits = decoder_outputs["logits"]
+        loss = self.loss_fct(lm_logits.view(-1, self.tokenizer.vocab_size), decoder_targets.view(-1))
+        return loss
+
+    def forward(self, audio, text):
+        encoder_outputs, _ = self.forward_encoder(audio)
+        loss = self.forward_decoder(text, encoder_outputs)
+        return loss
+
+    def generate(self,
+                 samples,
+                 use_nucleus_sampling=False,
+                 num_beams=5,
+                 max_length=128,
+                 min_length=2,
+                 top_p=0.9,
+                 repetition_penalty=1.0,
+                 ):
+
+        # self.bart.force_bos_token_to_be_generated = True
+        audio_embs = self.audio_encoder(samples)
+        encoder_outputs = self.bart.model.encoder(
+            input_ids=None,
+            attention_mask=None,
+            head_mask=None,
+            inputs_embeds=audio_embs,
+            output_attentions=None,
+            output_hidden_states=None,
+            return_dict=True)
+
+        input_ids = torch.zeros((encoder_outputs['last_hidden_state'].size(0), 1)).long().to(self.device)
+        input_ids[:, 0] = self.bart.config.decoder_start_token_id
+        decoder_attention_mask = torch.ones((encoder_outputs['last_hidden_state'].size(0), 1)).long().to(self.device)
+        if use_nucleus_sampling:
+            outputs = self.bart.generate(
+                input_ids=None,
+                attention_mask=None,
+                decoder_input_ids=input_ids,
+                decoder_attention_mask=decoder_attention_mask,
+                encoder_outputs=encoder_outputs,
+                max_length=max_length,
+                min_length=min_length,
+                do_sample=True,
+                top_p=top_p,
+                num_return_sequences=1,
+                repetition_penalty=1.1)
+        else:
+            outputs = self.bart.generate(input_ids=None,
+                                            attention_mask=None,
+                                            decoder_input_ids=input_ids,
+                                            decoder_attention_mask=decoder_attention_mask,
+                                            encoder_outputs=encoder_outputs,
+                                            head_mask=None,
+                                            decoder_head_mask=None,
+                                            inputs_embeds=None,
+                                            decoder_inputs_embeds=None,
+                                            use_cache=None,
+                                            output_attentions=None,
+                                            output_hidden_states=None,
+                                            max_length=max_length,
+                                            min_length=min_length,
+                                            num_beams=num_beams,
+                                            repetition_penalty=repetition_penalty)
+
+        captions = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)
+        return captions

model/modules.py

@@ -0,0 +1,95 @@
+### code reference: https://github.com/openai/whisper/blob/main/whisper/audio.py
+
+import os
+import torch
+import torchaudio
+import numpy as np
+import torch.nn.functional as F
+from torch import Tensor, nn
+from typing import Dict, Iterable, Optional
+
+# hard-coded audio hyperparameters
+SAMPLE_RATE = 16000
+N_FFT = 1024
+N_MELS = 128
+HOP_LENGTH = int(0.01 * SAMPLE_RATE)
+DURATION = 10
+N_SAMPLES = int(DURATION * SAMPLE_RATE) 
+N_FRAMES = N_SAMPLES // HOP_LENGTH + 1 
+
+def sinusoids(length, channels, max_timescale=10000):
+    """Returns sinusoids for positional embedding"""
+    log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
+    inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
+    scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
+    return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
+
+class MelEncoder(nn.Module):
+    """
+    time-frequency represntation
+    """
+    def __init__(self, 
+                sample_rate= 16000,
+                f_min=0,
+                f_max=8000,
+                n_fft=1024,
+                win_length=1024,
+                hop_length = int(0.01 * 16000),
+                n_mels = 128,
+                power = None,
+                pad= 0,
+                normalized= False,
+                center= True,
+                pad_mode= "reflect"
+                ):
+        super(MelEncoder, self).__init__()
+        self.window = torch.hann_window(win_length)
+        self.spec_fn = torchaudio.transforms.Spectrogram(
+            n_fft = n_fft,
+            win_length = win_length,
+            hop_length = hop_length,
+            power = power
+        )
+        self.mel_scale = torchaudio.transforms.MelScale(
+            n_mels, 
+            sample_rate,
+            f_min,
+            f_max,
+            n_fft // 2 + 1)
+        
+        self.amplitude_to_db = torchaudio.transforms.AmplitudeToDB()
+
+    def forward(self, wav):
+        spec = self.spec_fn(wav)
+        power_spec = spec.real.abs().pow(2)
+        mel_spec = self.mel_scale(power_spec)
+        mel_spec = self.amplitude_to_db(mel_spec) # Log10(max(reference value and amin))
+        return mel_spec
+
+class AudioEncoder(nn.Module):
+    def __init__(
+        self, n_mels: int, n_ctx: int, audio_dim: int, text_dim: int, num_of_stride_conv: int,
+    ):
+        super().__init__()
+        self.mel_encoder = MelEncoder(n_mels=n_mels)
+        self.conv1 = nn.Conv1d(n_mels, audio_dim, kernel_size=3, padding=1)
+        self.conv_stack = nn.ModuleList([])
+        for _ in range(num_of_stride_conv):
+            self.conv_stack.append(
+                nn.Conv1d(audio_dim, audio_dim, kernel_size=3, stride=2, padding=1)
+            )
+        # self.proj = nn.Linear(audio_dim, text_dim, bias=False)
+        self.register_buffer("positional_embedding", sinusoids(n_ctx, text_dim))
+
+    def forward(self, x: Tensor):
+        """
+        x : torch.Tensor, shape = (batch_size, waveform)
+            single channel wavform
+        """
+        x = self.mel_encoder(x) # (batch_size, n_mels, n_ctx)
+        x = F.gelu(self.conv1(x))
+        for conv in self.conv_stack:
+            x = F.gelu(conv(x))
+        x = x.permute(0, 2, 1)
+        x = (x + self.positional_embedding).to(x.dtype)
+        return x

muzyka_AI.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:480742943da1b14e194684919a8e531e383503318c28420a29f723468c3407dc
+size 6376447

prompt.txt

@@ -0,0 +1 @@
+ma to być

requirements.txt

@@ -0,0 +1,13 @@
+torch
+torchaudio
+transformers
+ctransformers --no-binary=ctransformers
+librosa >= 0.8
+pip>=23.2
+gradio_client==0.2.7
+invisible_watermark
+safetensors
+diffusers
+soundfile
+openai-whisper
+accelerate

utils/audio_utils.py

@@ -0,0 +1,247 @@
+STR_CLIP_ID = 'clip_id'
+STR_AUDIO_SIGNAL = 'audio_signal'
+STR_TARGET_VECTOR = 'target_vector'
+
+
+STR_CH_FIRST = 'channels_first'
+STR_CH_LAST = 'channels_last'
+
+import io
+import os
+import tqdm
+import logging
+import subprocess
+from typing import Tuple
+from pathlib import Path
+
+# import librosa
+import numpy as np
+import soundfile as sf
+
+import itertools
+from numpy.fft import irfft
+
+def _resample_load_ffmpeg(path: str, sample_rate: int, downmix_to_mono: bool) -> Tuple[np.ndarray, int]:
+    """
+    Decoding, downmixing, and downsampling by librosa.
+    Returns a channel-first audio signal.
+
+    Args:
+        path:
+        sample_rate:
+        downmix_to_mono:
+
+    Returns:
+        (audio signal, sample rate)
+    """
+
+    def _decode_resample_by_ffmpeg(filename, sr):
+        """decode, downmix, and resample audio file"""
+        channel_cmd = '-ac 1 ' if downmix_to_mono else ''  # downmixing option
+        resampling_cmd = f'-ar {str(sr)}' if sr else ''  # downsampling option
+        cmd = f"ffmpeg -i \"{filename}\" {channel_cmd} {resampling_cmd} -f wav -"
+        p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        return out
+
+    src, sr = sf.read(io.BytesIO(_decode_resample_by_ffmpeg(path, sr=sample_rate)))
+    return src.T, sr
+
+
+def _resample_load_librosa(path: str, sample_rate: int, downmix_to_mono: bool, **kwargs) -> Tuple[np.ndarray, int]:
+    """
+    Decoding, downmixing, and downsampling by librosa.
+    Returns a channel-first audio signal.
+    """
+    src, sr = librosa.load(path, sr=sample_rate, mono=downmix_to_mono, **kwargs)
+    return src, sr
+
+
+def load_audio(
+    path: str or Path,
+    ch_format: str,
+    sample_rate: int = None,
+    downmix_to_mono: bool = False,
+    resample_by: str = 'ffmpeg',
+    **kwargs,
+) -> Tuple[np.ndarray, int]:
+    """A wrapper of librosa.load that:
+        - forces the returned audio to be 2-dim,
+        - defaults to sr=None, and
+        - defaults to downmix_to_mono=False.
+
+    The audio decoding is done by `audioread` or `soundfile` package and ultimately, often by ffmpeg.
+    The resampling is done by `librosa`'s child package `resampy`.
+
+    Args:
+        path: audio file path
+        ch_format: one of 'channels_first' or 'channels_last'
+        sample_rate: target sampling rate. if None, use the rate of the audio file
+        downmix_to_mono:
+        resample_by (str): 'librosa' or 'ffmpeg'. it decides backend for audio decoding and resampling.
+        **kwargs: keyword args for librosa.load - offset, duration, dtype, res_type.
+
+    Returns:
+        (audio, sr) tuple
+    """
+    if ch_format not in (STR_CH_FIRST, STR_CH_LAST):
+        raise ValueError(f'ch_format is wrong here -> {ch_format}')
+
+    if os.stat(path).st_size > 8000:
+        if resample_by == 'librosa':
+            src, sr = _resample_load_librosa(path, sample_rate, downmix_to_mono, **kwargs)
+        elif resample_by == 'ffmpeg':
+            src, sr = _resample_load_ffmpeg(path, sample_rate, downmix_to_mono)
+        else:
+            raise NotImplementedError(f'resample_by: "{resample_by}" is not supposred yet')
+    else:
+        raise ValueError('Given audio is too short!')
+    return src, sr
+
+    # if src.ndim == 1:
+    #     src = np.expand_dims(src, axis=0)
+    # # now always 2d and channels_first
+
+    # if ch_format == STR_CH_FIRST:
+    #     return src, sr
+    # else:
+    #     return src.T, sr
+
+def ms(x):
+    """Mean value of signal `x` squared.
+    :param x: Dynamic quantity.
+    :returns: Mean squared of `x`.
+    """
+    return (np.abs(x)**2.0).mean()
+
+def normalize(y, x=None):
+    """normalize power in y to a (standard normal) white noise signal.
+    Optionally normalize to power in signal `x`.
+    #The mean power of a Gaussian with :math:`\\mu=0` and :math:`\\sigma=1` is 1.
+    """
+    if x is not None:
+        x = ms(x)
+    else:
+        x = 1.0
+    return y * np.sqrt(x / ms(y))
+
+def noise(N, color='white', state=None):
+    """Noise generator.
+    :param N: Amount of samples.
+    :param color: Color of noise.
+    :param state: State of PRNG.
+    :type state: :class:`np.random.RandomState`
+    """
+    try:
+        return _noise_generators[color](N, state)
+    except KeyError:
+        raise ValueError("Incorrect color.")
+
+def white(N, state=None):
+    """
+    White noise.
+    :param N: Amount of samples.
+    :param state: State of PRNG.
+    :type state: :class:`np.random.RandomState`
+    White noise has a constant power density. It's narrowband spectrum is therefore flat.
+    The power in white noise will increase by a factor of two for each octave band,
+    and therefore increases with 3 dB per octave.
+    """
+    state = np.random.RandomState() if state is None else state
+    return state.randn(N)
+
+def pink(N, state=None):
+    """
+    Pink noise.
+    :param N: Amount of samples.
+    :param state: State of PRNG.
+    :type state: :class:`np.random.RandomState`
+    Pink noise has equal power in bands that are proportionally wide.
+    Power density decreases with 3 dB per octave.
+    """
+    state = np.random.RandomState() if state is None else state
+    uneven = N % 2
+    X = state.randn(N // 2 + 1 + uneven) + 1j * state.randn(N // 2 + 1 + uneven)
+    S = np.sqrt(np.arange(len(X)) + 1.)  # +1 to avoid divide by zero
+    y = (irfft(X / S)).real
+    if uneven:
+        y = y[:-1]
+    return normalize(y)
+
+def blue(N, state=None):
+    """
+    Blue noise.
+    :param N: Amount of samples.
+    :param state: State of PRNG.
+    :type state: :class:`np.random.RandomState`
+    Power increases with 6 dB per octave.
+    Power density increases with 3 dB per octave.
+    """
+    state = np.random.RandomState() if state is None else state
+    uneven = N % 2
+    X = state.randn(N // 2 + 1 + uneven) + 1j * state.randn(N // 2 + 1 + uneven)
+    S = np.sqrt(np.arange(len(X)))  # Filter
+    y = (irfft(X * S)).real
+    if uneven:
+        y = y[:-1]
+    return normalize(y)
+
+def brown(N, state=None):
+    """
+    Violet noise.
+    :param N: Amount of samples.
+    :param state: State of PRNG.
+    :type state: :class:`np.random.RandomState`
+    Power decreases with -3 dB per octave.
+    Power density decreases with 6 dB per octave.
+    """
+    state = np.random.RandomState() if state is None else state
+    uneven = N % 2
+    X = state.randn(N // 2 + 1 + uneven) + 1j * state.randn(N // 2 + 1 + uneven)
+    S = (np.arange(len(X)) + 1)  # Filter
+    y = (irfft(X / S)).real
+    if uneven:
+        y = y[:-1]
+    return normalize(y)
+
+def violet(N, state=None):
+    """
+    Violet noise. Power increases with 6 dB per octave.
+    :param N: Amount of samples.
+    :param state: State of PRNG.
+    :type state: :class:`np.random.RandomState`
+    Power increases with +9 dB per octave.
+    Power density increases with +6 dB per octave.
+    """
+    state = np.random.RandomState() if state is None else state
+    uneven = N % 2
+    X = state.randn(N // 2 + 1 + uneven) + 1j * state.randn(N // 2 + 1 + uneven)
+    S = (np.arange(len(X)))  # Filter
+    y = (irfft(X * S)).real
+    if uneven:
+        y = y[:-1]
+    return normalize(y)
+
+_noise_generators = {
+    'white': white,
+    'pink': pink,
+    'blue': blue,
+    'brown': brown,
+    'violet': violet,
+}
+
+def noise_generator(N=44100, color='white', state=None):
+    """Noise generator.
+    :param N: Amount of unique samples to generate.
+    :param color: Color of noise.
+    Generate `N` amount of unique samples and cycle over these samples.
+    """
+    #yield from itertools.cycle(noise(N, color)) # Python 3.3
+    for sample in itertools.cycle(noise(N, color, state)):
+        yield sample
+
+def heaviside(N):
+    """Heaviside.
+    Returns the value 0 for `x < 0`, 1 for `x > 0`, and 1/2 for `x = 0`.
+    """
+    return 0.5 * (np.sign(N) + 1)

voice_cloning_fraud.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:440d118fdb3e6e210c5435cec6bf50d1c61190a2e567b62ba39137cc9274ce3b
+size 4672978