Spaces:

amaye15
/

object-segmentation-processing

Running

amaye15 commited on Aug 26

Commit

e8afc94

•

1 Parent(s): 022365f

define size of push

Files changed (1) hide show

app.py CHANGED Viewed

@@ -67,7 +67,7 @@ def process_and_push_data():
     # Process data using the generator and push it to the hub
     ds_processed = Dataset.from_generator(get_data)
-    ds_processed.push_to_hub(TARGET_REPO)
     logger.info("Data processed and pushed to the hub.")

     # Process data using the generator and push it to the hub
     ds_processed = Dataset.from_generator(get_data)
+    ds_processed.push_to_hub(TARGET_REPO, max_shard_size="1GB")
     logger.info("Data processed and pushed to the hub.")