|
import os
|
|
import sys
|
|
import soundfile as sf
|
|
from tqdm import tqdm
|
|
|
|
def is_significant_audio(file_path, silence_threshold=-40, silence_percent=90):
|
|
"""
|
|
Check if an audio file contains significant non-silent parts.
|
|
"""
|
|
try:
|
|
data, samplerate = sf.read(file_path)
|
|
if len(data) == 0:
|
|
return False
|
|
|
|
|
|
energy = (data ** 2).mean()
|
|
silence_ratio = (energy < silence_threshold).sum() / len(data) * 100
|
|
return silence_ratio < silence_percent
|
|
except Exception as e:
|
|
print(f"Error processing {file_path}: {e}")
|
|
return False
|
|
|
|
def filter_manifest(manifest_path, output_path, dataset_dir):
|
|
"""
|
|
Read the manifest file, check for silence, and write filtered files.
|
|
"""
|
|
with open(manifest_path, 'r') as f:
|
|
lines = f.readlines()
|
|
|
|
filtered_lines = [lines[0]]
|
|
for line in tqdm(lines[1:], desc=f"Processing {manifest_path}"):
|
|
file_path = os.path.join(dataset_dir, line.split("\t")[0])
|
|
if is_significant_audio(file_path):
|
|
filtered_lines.append(line)
|
|
else:
|
|
print(f"Skipping file due to silence: {file_path}")
|
|
|
|
with open(output_path, 'w') as f_out:
|
|
f_out.writelines(filtered_lines)
|
|
|
|
if __name__ == "__main__":
|
|
train_manifest = sys.argv[1]
|
|
valid_manifest = sys.argv[2]
|
|
output_dir = sys.argv[3]
|
|
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
dataset_dir = "dataset"
|
|
filter_manifest(train_manifest, os.path.join(output_dir, "train.tsv"), dataset_dir)
|
|
filter_manifest(valid_manifest, os.path.join(output_dir, "valid.tsv"), dataset_dir)
|
|
|
