Upload folder using huggingface_hub

Browse files

Files changed (31) hide show

.gitattributes +8 -27
README.md +132 -0
config.yaml +93 -0
example.png +0 -0
hparams.yaml +15 -0
overrides.yaml +22 -0
pytorch_model.bin +3 -0
reproducible_research/dihard3_custom_split/development.txt +62 -0
reproducible_research/dihard3_custom_split/train.txt +192 -0
reproducible_research/expected_outputs/osd/AMI.development.rttm +0 -0
reproducible_research/expected_outputs/osd/AMI.test.rttm +0 -0
reproducible_research/expected_outputs/osd/DIHARD.development.rttm +0 -0
reproducible_research/expected_outputs/osd/DIHARD.test.rttm +0 -0
reproducible_research/expected_outputs/osd/VoxConverse.development.rttm +1614 -0
reproducible_research/expected_outputs/osd/VoxConverse.test.rttm +0 -0
reproducible_research/expected_outputs/rsg/AMI.development.rttm +0 -0
reproducible_research/expected_outputs/rsg/AMI.test.rttm +0 -0
reproducible_research/expected_outputs/rsg/DIHARD.development.rttm +0 -0
reproducible_research/expected_outputs/rsg/DIHARD.test.rttm +0 -0
reproducible_research/expected_outputs/rsg/VoxConverse.development.rttm +0 -0
reproducible_research/expected_outputs/vad/AMI.development.rttm +0 -0
reproducible_research/expected_outputs/vad/AMI.test.rttm +0 -0
reproducible_research/expected_outputs/vad/DIHARD.development.rttm +0 -0
reproducible_research/expected_outputs/vad/DIHARD.test.rttm +0 -0
reproducible_research/expected_outputs/vad/VoxConverse.development.rttm +0 -0
reproducible_research/expected_outputs/vad/VoxConverse.test.rttm +0 -0
reproducible_research/expected_outputs/vbx/AMI.rttm +0 -0
reproducible_research/expected_outputs/vbx/DIHARD.rttm +0 -0
reproducible_research/expected_outputs/vbx/VoxConverse.rttm +0 -0
tfevents.bin +3 -0
train.log +18 -0

.gitattributes CHANGED Viewed

@@ -1,35 +1,16 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text

README.md ADDED Viewed

	@@ -0,0 +1,132 @@

+---
+tags:
+- pyannote
+- audio
+- voice
+- speech
+- speaker
+- speaker-segmentation
+- voice-activity-detection
+- overlapped-speech-detection
+- resegmentation
+datasets:
+- ami
+- dihard
+- voxconverse
+license: mit
+inference: false
+---
+# pyannote.audio // speaker segmentation
+![Example](example.png)
+Model from *[End-to-end speaker segmentation for overlap-aware resegmentation](http://arxiv.org/abs/2104.04045)*,
+by Hervé Bredin and Antoine Laurent.
+Relies on pyannote.audio 2.0 currently in development: see [installation instructions](https://github.com/pyannote/pyannote-audio/tree/develop#installation).
+## Support
+For commercial enquiries and scientific consulting, please contact [me](mailto:herve@niderb.fr).
+For [technical questions](https://github.com/pyannote/pyannote-audio/discussions) and [bug reports](https://github.com/pyannote/pyannote-audio/issues), please check [pyannote.audio](https://github.com/pyannote/pyannote-audio) Github repository.
+## Usage
+### Voice activity detection
+```python
+from pyannote.audio.pipelines import VoiceActivityDetection
+pipeline = VoiceActivityDetection(segmentation="pyannote/segmentation")
+HYPER_PARAMETERS = {
+  # onset/offset activation thresholds
+  "onset": 0.5, "offset": 0.5,
+  # remove speech regions shorter than that many seconds.
+  "min_duration_on": 0.0,
+  # fill non-speech regions shorter than that many seconds.
+  "min_duration_off": 0.0
+}
+pipeline.instantiate(HYPER_PARAMETERS)
+vad = pipeline("audio.wav")
+# `vad` is a pyannote.core.Annotation instance containing speech regions
+```
+### Overlapped speech detection
+```python
+from pyannote.audio.pipelines import OverlappedSpeechDetection
+pipeline = OverlappedSpeechDetection(segmentation="pyannote/segmentation")
+pipeline.instantiate(HYPER_PARAMETERS)
+osd = pipeline("audio.wav")
+# `osd` is a pyannote.core.Annotation instance containing overlapped speech regions
+```
+### Resegmentation
+```python
+from pyannote.audio.pipelines import Resegmentation
+pipeline = Resegmentation(segmentation="pyannote/segmentation",
+                          diarization="baseline")
+pipeline.instantiate(HYPER_PARAMETERS)
+resegmented_baseline = pipeline({"audio": "audio.wav", "baseline": baseline})
+# where `baseline` should be provided as a pyannote.core.Annotation instance
+```
+### Raw scores
+```python
+from pyannote.audio import Inference
+inference = Inference("pyannote/segmentation")
+segmentation = inference("audio.wav")
+# `segmentation` is a pyannote.core.SlidingWindowFeature
+# instance containing raw segmentation scores like the
+# one pictured above (output)
+```
+## Reproducible research
+In order to reproduce the results of the paper ["End-to-end speaker segmentation for overlap-aware resegmentation
+"](https://arxiv.org/abs/2104.04045), use the following hyper-parameters:
+Voice activity detection  | `onset` | `offset` | `min_duration_on` | `min_duration_off`
+----------------|---------|----------|-------------------|-------------------
+AMI Mix-Headset | 0.684   | 0.577    | 0.181             | 0.037
+DIHARD3         | 0.767   | 0.377    | 0.136             | 0.067
+VoxConverse     | 0.767   | 0.713    | 0.182             | 0.501
+Overlapped speech detection | `onset` | `offset` | `min_duration_on` | `min_duration_off`
+----------------|---------|----------|-------------------|-------------------
+AMI Mix-Headset | 0.448   | 0.362    | 0.116             | 0.187
+DIHARD3         | 0.430   | 0.320    | 0.091             | 0.144
+VoxConverse     | 0.587   | 0.426    | 0.337             | 0.112
+Resegmentation of VBx | `onset` | `offset` | `min_duration_on` | `min_duration_off`
+----------------|---------|----------|-------------------|-------------------
+AMI Mix-Headset | 0.542   | 0.527    | 0.044             | 0.705
+DIHARD3         | 0.592   | 0.489    | 0.163             | 0.182
+VoxConverse     | 0.537   | 0.724    | 0.410             | 0.563
+Expected outputs (and VBx baseline) are also provided in the `/reproducible_research` sub-directories.
+## Citation
+```bibtex
+@inproceedings{Bredin2021,
+  Title = {{End-to-end speaker segmentation for overlap-aware resegmentation}},
+  Author = {{Bredin}, Herv{\'e} and {Laurent}, Antoine},
+  Booktitle = {Proc. Interspeech 2021},
+  Address = {Brno, Czech Republic},
+  Month = {August},
+  Year = {2021},
+```
+```bibtex
+@inproceedings{Bredin2020,
+  Title = {{pyannote.audio: neural building blocks for speaker diarization}},
+  Author = {{Bredin}, Herv{\\\\'e} and {Yin}, Ruiqing and {Coria}, Juan Manuel and {Gelly}, Gregory and {Korshunov}, Pavel and {Lavechin}, Marvin and {Fustes}, Diego and {Titeux}, Hadrien and {Bouaziz}, Wassim and {Gill}, Marie-Philippe},
+  Booktitle = {ICASSP 2020, IEEE International Conference on Acoustics, Speech, and Signal Processing},
+  Address = {Barcelona, Spain},
+  Month = {May},
+  Year = {2020},
+}
+```

config.yaml ADDED Viewed

	@@ -0,0 +1,93 @@

+protocol: X.SpeakerDiarization.Custom
+patience: 20
+task:
+  _target_: pyannote.audio.tasks.Segmentation
+  duration: 5.0
+  warm_up: 0.0
+  balance: null
+  overlap:
+    probability: 0.5
+    snr_min: 0.0
+    snr_max: 10.0
+  weight: null
+  batch_size: 32
+  num_workers: 10
+  pin_memory: false
+  loss: bce
+  vad_loss: bce
+model:
+  _target_: pyannote.audio.models.segmentation.PyanNet
+  sincnet:
+    stride: 10
+  lstm:
+    num_layers: 4
+    monolithic: true
+    dropout: 0.5
+  linear:
+    num_layers: 2
+optimizer:
+  _target_: torch.optim.Adam
+  lr: 0.001
+  betas:
+  - 0.9
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 0
+  amsgrad: false
+trainer:
+  _target_: pytorch_lightning.Trainer
+  accelerator: ddp
+  accumulate_grad_batches: 1
+  amp_backend: native
+  amp_level: O2
+  auto_lr_find: false
+  auto_scale_batch_size: false
+  auto_select_gpus: true
+  benchmark: true
+  check_val_every_n_epoch: 1
+  checkpoint_callback: true
+  deterministic: false
+  fast_dev_run: false
+  flush_logs_every_n_steps: 100
+  gpus: -1
+  gradient_clip_val: 0.5
+  limit_test_batches: 1.0
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  log_every_n_steps: 50
+  log_gpu_memory: null
+  max_epochs: 1000
+  max_steps: null
+  min_epochs: 1
+  min_steps: null
+  num_nodes: 1
+  num_processes: 1
+  num_sanity_val_steps: 2
+  overfit_batches: 0.0
+  precision: 32
+  prepare_data_per_node: true
+  process_position: 0
+  profiler: null
+  progress_bar_refresh_rate: 1
+  reload_dataloaders_every_epoch: false
+  replace_sampler_ddp: true
+  sync_batchnorm: false
+  terminate_on_nan: false
+  tpu_cores: null
+  track_grad_norm: -1
+  truncated_bptt_steps: null
+  val_check_interval: 1.0
+  weights_save_path: null
+  weights_summary: top
+augmentation:
+  transform: Compose
+  params:
+    shuffle: false
+    transforms:
+    - transform: AddBackgroundNoise
+      params:
+        background_paths: /gpfswork/rech/eie/commun/data/background/musan
+        min_snr_in_db: 5.0
+        max_snr_in_db: 15.0
+        mode: per_example
+        p: 0.9

example.png ADDED Viewed

hparams.yaml ADDED Viewed

	@@ -0,0 +1,15 @@

+linear:
+  hidden_size: 128
+  num_layers: 2
+lstm:
+  batch_first: true
+  bidirectional: true
+  dropout: 0.5
+  hidden_size: 128
+  monolithic: true
+  num_layers: 4
+num_channels: 1
+sample_rate: 16000
+sincnet:
+  sample_rate: 16000
+  stride: 10

overrides.yaml ADDED Viewed

	@@ -0,0 +1,22 @@

+- protocol=X.SpeakerDiarization.Custom
+- task=Segmentation
+- task.batch_size=32
+- task.num_workers=10
+- task.duration=5.
+- task.warm_up=0.
+- task.loss=bce
+- task.vad_loss=bce
+- patience=20
+- model=PyanNet
+- +model.sincnet.stride=10
+- +model.lstm.num_layers=4
+- +model.lstm.monolithic=True
+- +model.lstm.dropout=0.5
+- +model.linear.num_layers=2
+- optimizer=Adam
+- optimizer.lr=0.001
+- trainer.benchmark=True
+- trainer.gradient_clip_val=0.5
+- trainer.gpus=-1
+- trainer.accelerator=ddp
+- +augmentation=background

pytorch_model.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d2e72ce20167e5eb05ce163b7af9762e92ef5fec7313435b676b74b8498afe
+size 17739960

reproducible_research/dihard3_custom_split/development.txt ADDED Viewed

	@@ -0,0 +1,62 @@

+DH_DEV_0001
+DH_DEV_0023
+DH_DEV_0071
+DH_DEV_0078
+DH_DEV_0002
+DH_DEV_0015
+DH_DEV_0039
+DH_DEV_0003
+DH_DEV_0004
+DH_DEV_0006
+DH_DEV_0010
+DH_DEV_0012
+DH_DEV_0016
+DH_DEV_0024
+DH_DEV_0028
+DH_DEV_0033
+DH_DEV_0041
+DH_DEV_0042
+DH_DEV_0044
+DH_DEV_0047
+DH_DEV_0048
+DH_DEV_0056
+DH_DEV_0005
+DH_DEV_0008
+DH_DEV_0009
+DH_DEV_0017
+DH_DEV_0018
+DH_DEV_0007
+DH_DEV_0030
+DH_DEV_0037
+DH_DEV_0011
+DH_DEV_0027
+DH_DEV_0062
+DH_DEV_0013
+DH_DEV_0022
+DH_DEV_0025
+DH_DEV_0029
+DH_DEV_0031
+DH_DEV_0038
+DH_DEV_0046
+DH_DEV_0058
+DH_DEV_0014
+DH_DEV_0021
+DH_DEV_0026
+DH_DEV_0036
+DH_DEV_0043
+DH_DEV_0049
+DH_DEV_0055
+DH_DEV_0060
+DH_DEV_0068
+DH_DEV_0076
+DH_DEV_0077
+DH_DEV_0090
+DH_DEV_0032
+DH_DEV_0035
+DH_DEV_0065
+DH_DEV_0034
+DH_DEV_0123
+DH_DEV_0133
+DH_DEV_0051
+DH_DEV_0052
+DH_DEV_0054

reproducible_research/dihard3_custom_split/train.txt ADDED Viewed

	@@ -0,0 +1,192 @@

+DH_DEV_0080
+DH_DEV_0102
+DH_DEV_0122
+DH_DEV_0144
+DH_DEV_0145
+DH_DEV_0173
+DH_DEV_0176
+DH_DEV_0195
+DH_DEV_0214
+DH_DEV_0221
+DH_DEV_0225
+DH_DEV_0246
+DH_DEV_0066
+DH_DEV_0079
+DH_DEV_0103
+DH_DEV_0113
+DH_DEV_0124
+DH_DEV_0141
+DH_DEV_0206
+DH_DEV_0216
+DH_DEV_0230
+DH_DEV_0059
+DH_DEV_0063
+DH_DEV_0064
+DH_DEV_0070
+DH_DEV_0072
+DH_DEV_0074
+DH_DEV_0075
+DH_DEV_0081
+DH_DEV_0082
+DH_DEV_0084
+DH_DEV_0091
+DH_DEV_0094
+DH_DEV_0098
+DH_DEV_0100
+DH_DEV_0106
+DH_DEV_0109
+DH_DEV_0114
+DH_DEV_0116
+DH_DEV_0126
+DH_DEV_0127
+DH_DEV_0128
+DH_DEV_0130
+DH_DEV_0135
+DH_DEV_0137
+DH_DEV_0138
+DH_DEV_0142
+DH_DEV_0143
+DH_DEV_0156
+DH_DEV_0166
+DH_DEV_0174
+DH_DEV_0178
+DH_DEV_0186
+DH_DEV_0191
+DH_DEV_0196
+DH_DEV_0209
+DH_DEV_0223
+DH_DEV_0234
+DH_DEV_0235
+DH_DEV_0237
+DH_DEV_0238
+DH_DEV_0239
+DH_DEV_0240
+DH_DEV_0241
+DH_DEV_0248
+DH_DEV_0251
+DH_DEV_0252
+DH_DEV_0019
+DH_DEV_0020
+DH_DEV_0045
+DH_DEV_0050
+DH_DEV_0053
+DH_DEV_0073
+DH_DEV_0087
+DH_DEV_0089
+DH_DEV_0096
+DH_DEV_0132
+DH_DEV_0139
+DH_DEV_0167
+DH_DEV_0177
+DH_DEV_0184
+DH_DEV_0201
+DH_DEV_0210
+DH_DEV_0228
+DH_DEV_0236
+DH_DEV_0040
+DH_DEV_0057
+DH_DEV_0105
+DH_DEV_0108
+DH_DEV_0163
+DH_DEV_0188
+DH_DEV_0193
+DH_DEV_0204
+DH_DEV_0208
+DH_DEV_0212
+DH_DEV_0217
+DH_DEV_0093
+DH_DEV_0146
+DH_DEV_0154
+DH_DEV_0155
+DH_DEV_0158
+DH_DEV_0183
+DH_DEV_0187
+DH_DEV_0229
+DH_DEV_0245
+DH_DEV_0061
+DH_DEV_0067
+DH_DEV_0069
+DH_DEV_0083
+DH_DEV_0086
+DH_DEV_0088
+DH_DEV_0118
+DH_DEV_0121
+DH_DEV_0147
+DH_DEV_0153
+DH_DEV_0160
+DH_DEV_0164
+DH_DEV_0165
+DH_DEV_0168
+DH_DEV_0172
+DH_DEV_0181
+DH_DEV_0192
+DH_DEV_0194
+DH_DEV_0202
+DH_DEV_0203
+DH_DEV_0218
+DH_DEV_0250
+DH_DEV_0253
+DH_DEV_0254
+DH_DEV_0095
+DH_DEV_0097
+DH_DEV_0099
+DH_DEV_0104
+DH_DEV_0107
+DH_DEV_0110
+DH_DEV_0111
+DH_DEV_0115
+DH_DEV_0117
+DH_DEV_0129
+DH_DEV_0148
+DH_DEV_0149
+DH_DEV_0150
+DH_DEV_0151
+DH_DEV_0152
+DH_DEV_0157
+DH_DEV_0161
+DH_DEV_0171
+DH_DEV_0182
+DH_DEV_0185
+DH_DEV_0190
+DH_DEV_0197
+DH_DEV_0198
+DH_DEV_0200
+DH_DEV_0205
+DH_DEV_0213
+DH_DEV_0215
+DH_DEV_0219
+DH_DEV_0220
+DH_DEV_0222
+DH_DEV_0224
+DH_DEV_0231
+DH_DEV_0242
+DH_DEV_0243
+DH_DEV_0247
+DH_DEV_0249
+DH_DEV_0092
+DH_DEV_0112
+DH_DEV_0119
+DH_DEV_0131
+DH_DEV_0136
+DH_DEV_0175
+DH_DEV_0199
+DH_DEV_0207
+DH_DEV_0232
+DH_DEV_0134
+DH_DEV_0159
+DH_DEV_0169
+DH_DEV_0170
+DH_DEV_0180
+DH_DEV_0189
+DH_DEV_0211
+DH_DEV_0226
+DH_DEV_0244
+DH_DEV_0085
+DH_DEV_0101
+DH_DEV_0120
+DH_DEV_0125
+DH_DEV_0140
+DH_DEV_0162
+DH_DEV_0179
+DH_DEV_0227
+DH_DEV_0233

reproducible_research/expected_outputs/osd/AMI.development.rttm ADDED Viewed