from clean import clean_text

from datasets import load_dataset

dataset_v0 = load_dataset('oscar', "unshuffled_deduplicated_nl", split='train', streaming=True)


def f(obj):
    obj["text"] = clean_text(obj["text"])
    return obj


dataset_v1 = dataset_v0.map(f)
it = iter(dataset_v0)

print(next(it))
print(next(it))
print(next(it))

it = iter(dataset_v1)

print(next(it))
print(next(it))
print(next(it))