Upload major_area_meeting_transcripts.py with huggingface_hub

major_area_meeting_transcripts.py

@@ -0,0 +1,201 @@
+import datasets
+import os
+import csv
+import json
+from train_url import _TRAIN_DATA_URLS
+from validation_url import _VALIDATION_DATA_URLS
+
+
+_HOMEPAGE = "https://www.aihub.or.kr/aihubdata/data/view.do?currMenu=115&topMenu=100&aihubDataSe=data&dataSetSn=464"
+_DESCRIPTION = "AI HUB 주요 영역별 회의 음성 데이터셋"
+
+
+class MajorAreaMeetingConfig(datasets.BuilderConfig):
+    def __init__(self, name, **kwargs):
+        super(MajorAreaMeetingConfig, self).__init__(
+            version=datasets.Version("1.0.0", ""),
+            name=name,
+            **kwargs,
+        )
+        if name == "all":
+            self.data_urls = {
+                "train": _TRAIN_DATA_URLS["공중파방송"]
+                + _TRAIN_DATA_URLS["기타녹음"]
+                + _TRAIN_DATA_URLS["라디오"]
+                + _TRAIN_DATA_URLS["인터넷방송"],
+                "validation": _VALIDATION_DATA_URLS["공중파방송"]
+                + _VALIDATION_DATA_URLS["기타녹음"]
+                + _VALIDATION_DATA_URLS["라디오"]
+                + _VALIDATION_DATA_URLS["인터넷방송"],
+            }
+            self.transcript_urls = {
+                "train": [
+                    path.replace("data.tar.gz", "data.jsonl")
+                    for path in self.data_urls["train"]
+                ],
+                "validation": [
+                    path.replace("data.tar.gz", "data.jsonl")
+                    for path in self.data_urls["validation"]
+                ],
+            }
+        else:
+            self.transcript_urls = {
+                "train": _TRAIN_DATA_URLS[name],
+                "validation": _VALIDATION_DATA_URLS[name],
+            }
+
+
+class MajorAreaMeeting(datasets.GeneratorBasedBuilder):
+
+    BUILDER_CONFIGS = [
+        MajorAreaMeetingConfig(name="all"),
+        MajorAreaMeetingConfig(name="공중파방송"),
+        MajorAreaMeetingConfig(name="기타녹음"),
+        MajorAreaMeetingConfig(name="라디오"),
+        MajorAreaMeetingConfig(name="인터넷방송"),
+    ]
+
+    def _info(self):
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "id": datasets.Value("string"),
+                    "path": datasets.Value("string"),
+                    "audio": datasets.features.Audio(sampling_rate=16_000),
+                    "speaker_id": datasets.Value("string", id=None),
+                    "speaker_name": datasets.Value("string", id=None),
+                    "speaker_age": datasets.Value("string", id=None),
+                    "speaker_occupation": datasets.Value("string", id=None),
+                    "speaker_role": datasets.Value("string", id=None),
+                    "form": datasets.Value("string"),
+                    "original_form": datasets.Value("string"),
+                    "environment": datasets.Value("string"),
+                    "isIdiom": datasets.Value("bool"),
+                    "hangeulToEnglish": datasets.Sequence(
+                        {
+                            "id": datasets.Value("string", id=None),
+                            "hangeul": datasets.Value("string", id=None),
+                            "english": datasets.Value("string", id=None),
+                            "begin": datasets.Value("int32"),
+                            "end": datasets.Value("int32"),
+                        }
+                    ),
+                    "hangeulToNumber": datasets.Sequence(
+                        {
+                            "id": datasets.Value("string", id=None),
+                            "hangeul": datasets.Value("string", id=None),
+                            "number": datasets.Value("string", id=None),
+                            "begin": datasets.Value("int32"),
+                            "end": datasets.Value("int32"),
+                        }
+                    ),
+                    "term": datasets.Sequence(
+                        {
+                            "id": datasets.Value("string", id=None),
+                            "hangeul": datasets.Value("string", id=None),
+                            "begin": datasets.Value("int32"),
+                            "end": datasets.Value("int32"),
+                        }
+                    ),
+                }
+            ),
+        )
+
+    def _split_generators(self, dl_manager):
+        transcripts = dl_manager.download(
+            {
+                "train": self.config.transcript_urls["train"],
+                "validation": self.config.transcript_urls["validation"],
+            }
+        )
+        audio_filenames_paths = dl_manager.download(
+            {
+                "train": self.config.data_urls["train"],
+                "validation": self.config.data_urls["validation"],
+            }
+        )
+
+        audio_archives = audio_filenames_paths
+
+        local_extracted_archives = (
+            dl_manager.extract(audio_archives) if not dl_manager.is_streaming else {}
+        )
+
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TRAIN,
+                gen_kwargs={
+                    "transcript_paths": [
+                        transcript for transcript in transcripts["train"]
+                    ],
+                    "audio_archives": [
+                        dl_manager.iter_archive(archive)
+                        for archive in audio_archives["train"]
+                    ],
+                    "local_extracted_archive": local_extracted_archives.get("train"),
+                },
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.VALIDATION,
+                gen_kwargs={
+                    "transcript_paths": [
+                        transcript for transcript in transcripts["validation"]
+                    ],
+                    "audio_archives": [
+                        dl_manager.iter_archive(archive)
+                        for archive in audio_archives["validation"]
+                    ],
+                    "local_extracted_archive": local_extracted_archives.get(
+                        "validation"
+                    ),
+                },
+            ),
+        ]
+
+    def _generate_examples(
+        self,
+        transcript_paths,
+        audio_archives,
+        local_extracted_archive,
+    ):
+
+        transcripts = {}
+
+        with open(transcript_paths[0], "r", encoding="utf-8") as file:
+            for line in file:
+                data = json.loads(line)
+                transcripts[data["id"]] = data
+
+        for archive_idx, audio_archive in enumerate(audio_archives):
+            for audio_filename, file in audio_archive:
+                local_audio_file_path = (
+                    os.path.join(local_extracted_archive[archive_idx], audio_filename)
+                    if local_extracted_archive
+                    else None
+                )
+                data = transcripts[os.path.basename(audio_filename)[:-4]]
+
+                yield audio_filename, {
+                    "id": data["id"],
+                    "speaker_id": data["speaker_id"],
+                    "speaker_name": data["speaker_name"],
+                    "speaker_age": data["speaker_age"],
+                    "speaker_occupation": data["speaker_occupation"],
+                    "speaker_role": data["speaker_role"],
+                    "form": data["form"],
+                    "original_form": data["original_form"],
+                    "environment": data["environment"],
+                    "isIdiom": data["isIdiom"],
+                    "hangeulToEnglish": data["hangulToEnglish"],
+                    "hangeulToNumber": data["hangulToNumber"],
+                    "term": data["term"],
+                    "audio": {
+                        "path": (
+                            local_audio_file_path
+                            if local_audio_file_path
+                            else audio_filename
+                        ),
+                        "bytes": file.read(),
+                    },
+                }