Duplicate from seungheondoh/lp-music-caps

Co-authored-by: seungheon.doh <seungheondoh@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+orchestra.wav filter=lfs diff=lfs merge=lfs -text
+folk.wav filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,47 @@
+---
+license: mit
+datasets:
+- seungheondoh/LP-MusicCaps-MSD
+- seungheondoh/LP-MusicCaps-MC
+language:
+- en
+metrics:
+- bleu
+- bertscore
+tags:
+- music
+- music-captioning
+---
+
+- **Repository:** [LP-MusicCaps repository](https://github.com/seungheondoh/lp-music-caps)
+- **Paper:** [ArXiv](https://arxiv.org/abs/2307.16372)
+
+# :sound: LP-MusicCaps: LLM-Based Pseudo Music Captioning
+
+[![Demo Video](https://i.imgur.com/cgi8NsD.jpg)](https://youtu.be/ezwYVaiC-AM)
+
+This is a implementation of [LP-MusicCaps: LLM-Based Pseudo Music Captioning](#). This project aims to generate captions for music. 1) Tag-to-Caption: Using existing tags, We leverage the power of OpenAI's GPT-3.5 Turbo API to generate high-quality and contextually relevant captions based on music tag. 2) Audio-to-Caption: Using music-audio and pseudo caption pairs, we train a cross-model encoder-decoder model for end-to-end music captioning
+
+> [**LP-MusicCaps: LLM-Based Pseudo Music Captioning**](#)   
+> SeungHeon Doh, Keunwoo Choi, Jongpil Lee, Juhan Nam   
+> To appear ISMIR 2023   
+
+
+## TL;DR
+
+
+<p align = "center">
+<img src = "https://i.imgur.com/2LC0nT1.png">
+</p>
+
+- **[1.Tag-to-Caption: LLM Captioning](https://github.com/seungheondoh/lp-music-caps/tree/main/lpmc/llm_captioning)**: Generate caption from given tag input.
+- **[2.Pretrain Music Captioning Model](https://github.com/seungheondoh/lp-music-caps/tree/main/lpmc/music_captioning)**: Generate pseudo caption from given audio.
+- **[3.Transfer Music Captioning Model](https://github.com/seungheondoh/lp-music-caps/tree/main/lpmc/music_captioning/transfer.py)**: Generate human level caption from given audio.
+
+## Open Source Material
+
+- [pre-trained models](https://huggingface.co/seungheondoh/lp-music-caps) 
+- [music-pseudo caption dataset](https://huggingface.co/datasets/seungheondoh/LP-MusicCaps-MSD)
+- [demo](https://huggingface.co/spaces/seungheondoh/LP-Music-Caps-demo) 
+
+are available online for future research. example of dataset in [notebook](https://github.com/seungheondoh/lp-music-caps/blob/main/notebook/Dataset.ipynb)

folk.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed9c5e4e03b706c464993ab770388fd3a52b240e4c054ec279322dd48c0cb7a8
+size 1918830

orchestra.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97135f273616d39b11db37a1ec61e0bd729b82c085f7dcdc472b26a37affca8a
+size 1323632

pretrain.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15df9119df8957b29ef18a891c3a1e295804733e07d991177e5a77e4fb121b8d
+size 1783710751

pretrain.yaml

@@ -0,0 +1,27 @@
+framework: bart
+data_dir: ../../dataset
+train_data: msd_balence
+text_type: all
+arch: transformer
+workers: 12
+epochs: 4096
+warmup_epochs: 125
+start_epoch: 0
+batch_size: 256
+world_size: 1
+lr: 0.0001
+min_lr: 1.0e-09
+rank: 0
+dist_url: tcp://localhost:12312
+dist_backend: nccl
+seed: null
+gpu: 0
+print_freq: 100
+multiprocessing_distributed: false
+cos: true
+bart_pretrain: false
+label_smoothing: 0.1
+use_early_stopping: false
+eval_sample: 0
+max_length: 110
+distributed: false

supervised.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3adca350245b333729748e9c955855c6d373c5831d8d8feca4e199d48e3e01ee
+size 1783651987

supervised.yaml

@@ -0,0 +1,26 @@
+framework: bart
+data_dir: ../../dataset
+train_data: music_caps
+text_type: gt
+arch: transformer
+workers: 8
+epochs: 100
+warmup_epochs: 1
+start_epoch: 0
+batch_size: 64
+world_size: 1
+lr: 0.0001
+min_lr: 1.0e-09
+rank: 0
+dist_url: tcp://localhost:12312
+dist_backend: nccl
+seed: null
+gpu: 0
+print_freq: 100
+multiprocessing_distributed: false
+cos: true
+bart_pretrain: false
+label_smoothing: 0.1
+use_early_stopping: false
+eval_sample: 64
+max_length: 128

transfer.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d04e457e045a09c7c5037222eaed3ffe35f8689b3753a2ce6094c5d5792f9bc
+size 1783650705

transfer.yaml

@@ -0,0 +1,26 @@
+framework: bart
+data_dir: ../../dataset
+train_data: msd_balence
+text_type: all
+arch: transformer
+workers: 8
+epochs: 100
+warmup_epochs: 20
+start_epoch: 0
+batch_size: 64
+world_size: 1
+lr: 0.0001
+min_lr: 1.0e-09
+rank: 0
+dist_url: tcp://localhost:12312
+dist_backend: nccl
+seed: null
+gpu: 1
+print_freq: 10
+multiprocessing_distributed: false
+cos: true
+bart_pretrain: false
+label_smoothing: 0.1
+use_early_stopping: false
+eval_sample: 64
+max_length: 128