File size: 503 Bytes
14e4843 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
#!/usr/bin/env python3
import glob
import os
from datasets import load_dataset
folder_path = 'isp-data-json/' # Replace with your folder path
# Search for all .json files in the folder
json_files = glob.glob(os.path.join(folder_path, '*.jsonl'))
path = 'pminervini/inverse-scaling'
for json_path in json_files:
base_name = os.path.basename(json_path)
name = base_name.split("_")[0]
ds = load_dataset("json", data_files={'data': json_path})
ds.push_to_hub(path, config_name=name)
|