File size: 408 Bytes
1d4a13a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
from clean import clean_text
from datasets import load_dataset
dataset_v0 = load_dataset('oscar', "unshuffled_deduplicated_nl", split='train', streaming=True)
def f(obj):
obj["text"] = clean_text(obj["text"])
return obj
dataset_v1 = dataset_v0.map(f)
it = iter(dataset_v0)
print(next(it))
print(next(it))
print(next(it))
it = iter(dataset_v1)
print(next(it))
print(next(it))
print(next(it))
|