diff --git a/Load data & train tokenizer.ipynb b/Load data & train tokenizer.ipynb index 63fb53aab7e05a845a07bfe40ff94918edf6451f..02e6b6cbdcfdb7907e874e958ba1ee131da6bf58 100644 --- a/Load data & train tokenizer.ipynb +++ b/Load data & train tokenizer.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 71, "id": "723b5d4d", "metadata": {}, "outputs": [], @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 72, "id": "f4a5edee", "metadata": {}, "outputs": [], @@ -42,6 +42,55 @@ "from transformers import AutoConfig\n" ] }, + { + "cell_type": "code", + "execution_count": 74, + "id": "9241a429", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "348a4dd4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + } + ], + "source": [ + "tokenizer = AutoTokenizer.from_pretrained(\"./\")" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "595f318e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "36450" + ] + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.vocab_size" + ] + }, { "cell_type": "code", "execution_count": 3, @@ -66,23 +115,1049 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "id": "39b9fc3d", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "2848" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len([x for x in data_files if isinstance(x, str)])" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 61, "id": "ba855add", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "['/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00943-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00018-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01012-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00625-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00070-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00108-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00315-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00056-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00140-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00128-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00221-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00394-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00469-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00547-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00444-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00000-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00129-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00229-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00335-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00792-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00090-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00584-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00986-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00618-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00824-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00114-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00034-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00465-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00185-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01013-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00310-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00071-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00030-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00132-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00074-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00480-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00460-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00847-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00783-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00141-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00967-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00145-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00586-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00188-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00745-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00047-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00850-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00124-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00952-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00333-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00005-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00760-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00882-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00581-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00164-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00120-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00509-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00167-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00180-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00017-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00167-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01004-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00756-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00728-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00033-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00551-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00132-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00231-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00924-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00725-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00362-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00123-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01000-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00161-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00344-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00213-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00721-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01011-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00446-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00235-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00061-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00671-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00294-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00177-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00081-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00407-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00113-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00030-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00293-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00147-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00698-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00598-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00006-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00354-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00860-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00841-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00481-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00129-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00109-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00478-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00667-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00390-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00525-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00449-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00126-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00016-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00197-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00762-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00247-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00982-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00130-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00184-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00063-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00445-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00116-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00675-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00838-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00726-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00191-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00832-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00110-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00012-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00098-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00257-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00130-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00659-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00355-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00487-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00324-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00459-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00439-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01007-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00706-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00512-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00073-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00051-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00911-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00013-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00987-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00188-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00220-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00885-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00905-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00813-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00326-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01015-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00457-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00562-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00503-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00845-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00755-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00969-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00949-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00668-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00042-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00146-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00302-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00050-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00002-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00068-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00608-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00616-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00573-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00127-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00171-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00149-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00516-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00176-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00451-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00597-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00311-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00747-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00430-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00743-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00561-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00161-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00534-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00029-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00448-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00022-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00096-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00736-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00672-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00533-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00295-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00438-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00776-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00176-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00375-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00298-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00501-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00145-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00152-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00524-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00693-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00252-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00772-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00102-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00468-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00165-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00199-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00409-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00095-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00816-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00136-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00908-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00942-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00611-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00304-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00825-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00880-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00157-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00643-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00602-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00050-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00009-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00995-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00175-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00025-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00147-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00039-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00104-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00140-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00278-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00663-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00058-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00846-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00314-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00486-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00073-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00622-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00153-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00630-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00042-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00740-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00172-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00121-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01023-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00156-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00759-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00148-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00007-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00811-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00270-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00360-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00541-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00176-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00121-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00948-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00628-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00106-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00208-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00172-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00072-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00920-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00006-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00582-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00983-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00594-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00461-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00134-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00118-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00081-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00093-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00277-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00377-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00034-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00424-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00421-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00162-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00488-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01016-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00703-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00748-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00866-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00096-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00170-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00248-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00669-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00436-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00085-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00036-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00058-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00695-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01020-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00817-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00844-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00477-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00224-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00464-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00564-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00442-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00065-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00592-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01017-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00181-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00273-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00957-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00153-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00035-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00543-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00940-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00526-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00275-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00161-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00812-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00858-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00992-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00769-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00015-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00753-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00413-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00435-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00351-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00031-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00180-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00021-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00734-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00854-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00859-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00018-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00219-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00836-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00895-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01009-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00632-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00530-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00508-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00709-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00077-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00578-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00945-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00182-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00139-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00047-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00035-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00035-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00387-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00001-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00804-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00474-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00383-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00181-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01021-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00679-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00338-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00179-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00042-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00961-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00010-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00091-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00423-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00290-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00947-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00133-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00380-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00946-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00604-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00045-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00082-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00493-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00552-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00152-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00732-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00427-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00216-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00746-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00057-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00781-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00918-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00134-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00046-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00453-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00099-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00704-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00361-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00067-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00510-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00088-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00210-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00325-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00605-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00080-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00651-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00367-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00822-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00041-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00358-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00142-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00491-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00892-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00190-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00356-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00068-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00359-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00087-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00452-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00554-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00259-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00086-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00904-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00012-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00105-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00032-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00996-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00192-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00443-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00909-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00938-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00162-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00214-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00447-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00839-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00856-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00476-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00371-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00504-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00253-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00921-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00408-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00000-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00973-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01001-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00048-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00179-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00146-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00261-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00902-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00092-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00126-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00026-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00154-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00640-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00994-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00156-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00228-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00038-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00654-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00852-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00128-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00268-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00933-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00492-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00056-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00232-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00808-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00398-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00401-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00386-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00179-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00650-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00197-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00093-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00114-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00626-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00317-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00336-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00012-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00606-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00340-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00861-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00089-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00515-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00378-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00684-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00647-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00196-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00025-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00266-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00204-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00022-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00097-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00160-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00059-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00113-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00182-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00060-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00368-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00001-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00353-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00062-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00198-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00175-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00026-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00143-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00016-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00008-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00189-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00334-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00071-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00519-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00773-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00159-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00624-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00105-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00109-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00112-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00754-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00184-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01005-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00289-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00136-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00194-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00775-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00768-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00402-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00868-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00827-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00689-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00894-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00802-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00980-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00661-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00523-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00631-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00073-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00490-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00473-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00173-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00319-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00791-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00321-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00194-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00715-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00132-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00100-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00053-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00433-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00455-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00142-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00011-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00585-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00798-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00842-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00050-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00657-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00102-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00750-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00072-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00716-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00702-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00285-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00002-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00020-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00761-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01014-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00553-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00181-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00500-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00287-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00422-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00076-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00511-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00246-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00092-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00159-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00320-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00869-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00194-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00031-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00855-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00158-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00098-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00102-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00686-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00167-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00887-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00737-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00155-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00069-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00016-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00299-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00168-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00462-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00416-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00627-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00567-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00559-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00799-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00364-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00172-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00521-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00187-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00062-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00119-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00030-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00027-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00739-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00041-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00888-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00934-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00019-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00154-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00330-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00786-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00939-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00066-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00043-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00881-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00391-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00112-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00332-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00593-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01022-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00127-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00141-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00629-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00953-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00242-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00054-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00112-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00207-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00990-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00463-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00145-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00009-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00713-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00414-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00119-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00874-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00682-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00150-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00122-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00193-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00535-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00610-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00198-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00023-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00212-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00470-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00678-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00192-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00117-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00040-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00027-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00349-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00576-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00549-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00023-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00612-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00171-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00003-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00502-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00084-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00884-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00309-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00690-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00696-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00784-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00031-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00280-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00697-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00536-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00195-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00712-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00107-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00454-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00150-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00203-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00806-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00999-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00496-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00404-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00857-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00771-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00185-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00045-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00475-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00575-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00879-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00357-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00665-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00057-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00951-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00979-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00906-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00062-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00024-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00495-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00692-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00095-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00014-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00201-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00805-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00039-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00388-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00032-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00589-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00186-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00677-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00411-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00641-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00061-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00079-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00028-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00119-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00527-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00346-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00720-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00829-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00558-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00064-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00676-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00774-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00574-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00899-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00596-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00074-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00069-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00125-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00341-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00456-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00393-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00020-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00258-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00514-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00108-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00271-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00089-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00091-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00146-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00044-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00590-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00008-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00074-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00914-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00296-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00800-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00163-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00190-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00484-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00144-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00100-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00991-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00965-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00507-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00916-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00563-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00269-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00123-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00084-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00889-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00744-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00862-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00777-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00719-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00014-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00087-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00144-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00051-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00196-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00226-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00071-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00028-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00024-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00080-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00428-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00579-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00619-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00607-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00968-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00052-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00020-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00128-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00620-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00372-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00187-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00090-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00099-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00084-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00100-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00171-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00120-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00544-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00997-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00078-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00801-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00041-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00699-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00571-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00054-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00034-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00871-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00710-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00653-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00803-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00107-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00382-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00539-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00155-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00782-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00531-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00912-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00680-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00714-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00052-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00255-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00944-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00494-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00603-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00316-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00049-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00158-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00191-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00010-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00066-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00322-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00250-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00656-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00963-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00262-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00168-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00810-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00072-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00138-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00272-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00313-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00318-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00863-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00077-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00126-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00742-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00738-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00780-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00189-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00658-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00701-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00286-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00568-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00853-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00931-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00717-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00138-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00046-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00059-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00118-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00993-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00158-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00166-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00044-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00864-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00017-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00093-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00638-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00327-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00385-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00988-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00117-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00120-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00237-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00941-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00307-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00157-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00705-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00613-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00014-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00244-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00978-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00113-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00730-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00151-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00583-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00870-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00415-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00183-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00065-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00075-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00193-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00022-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00935-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00376-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00192-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00793-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00149-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00160-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00365-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00412-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00998-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00039-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00621-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00182-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00615-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00040-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00168-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00148-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00117-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00345-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00241-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00013-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00896-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00149-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00061-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00180-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00082-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00114-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00021-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00153-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00420-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00002-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00056-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00007-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00110-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00160-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00018-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00164-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00174-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00555-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00028-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00985-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00397-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00588-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00101-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00066-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00797-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00143-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00103-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00954-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00649-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00722-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00545-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00700-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00254-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00482-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00079-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00550-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00645-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00572-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00837-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00329-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00108-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00635-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00116-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00054-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00283-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00080-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00037-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00483-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00059-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00591-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00694-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00134-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00225-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00206-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00970-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00569-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00169-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00472-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00929-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00130-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00300-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00138-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00890-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00005-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00891-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00913-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00038-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00019-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00518-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00830-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-01002-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00363-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00789-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00053-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00683-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00111-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00431-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00223-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00809-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00767-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00642-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00218-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00052-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00685-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00876-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00347-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00027-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00024-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00828-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00075-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00133-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00927-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00831-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00749-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00279-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00005-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00634-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00546-of-01024.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00055-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00849-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00165-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00209-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00029-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00198-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00757-of-01024.json.gz',\n", + " '/data/nrc_uniq_cleaned_20210223/part-00124-47c217ad-de48-4b5e-b177-ef4bcd69ad2b-c000.json.gz',\n", + " '/data/nu_uniq_cleaned_20210225/part-00105-f31777cc-bf2a-4d2c-95c8-22f5eec50039-c000.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00323-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00498-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00795-of-01024.json.gz',\n", + " '/data/c4_cleaned2/cleaned2_c4-nl.tfrecord-00366-of-01024.json.gz',\n", + " ...]" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_files" + ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 45, "id": "59076aa7", "metadata": {}, "outputs": [ @@ -90,68 +1165,145 @@ "name": "stdout", "output_type": "stream", "text": [ - "Number of files 20 after adding /data/c4_cleaned\n" + "Number of files 2448 after adding /data/c4_cleaned2 glob *.gz\n", + "Number of files 2648 after adding /data/nrc_uniq_cleaned_20210223 glob *.gz\n", + "Number of files 2848 after adding /data/nu_uniq_cleaned_20210225 glob *.gz\n" ] + }, + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "#59G c4_cleaned compressed\n", - "#937M nrc_uniq_cleaned_20210223 compressed\n", - "#410M nu_uniq_cleaned_20210225 compressed\n", - "#9.9G oscar_nl_cleaned compressed\n", - "\n", - "\n", - "\n", - "data_files = []\n", - "SEED=42\n", - "def add_jsonlines_dir(path):\n", - " global data_files\n", - " #data_files += glob.glob(f\"{path}/*47*.gz\")\n", - " #data_files += glob.glob(f\"{path}/*32*.gz\")\n", - " #data_files += glob.glob(f\"{path}/*59*.gz\")\n", - " data_files += glob.glob(f\"{path}/*11*.gz\")\n", - " print(f\"Number of files {len(data_files)} after adding {path}\")\n", - " \n", - "add_jsonlines_dir(\"/data/c4_cleaned\")\n", - "#add_jsonlines_dir(\"/data/nrc_uniq_cleaned_20210223\")\n", - "#add_jsonlines_dir(\"/data/nu_uniq_cleaned_20210225\")\n", - "#add_jsonlines_dir(\"/data/oscar_nl_cleaned\") This one gives an error like field url not in \n", - "\n" + " datafiles = []\n", + " import glob\n", + " import random\n", + " SEED = 12345\n", + " def add_jsonlines_dir(path, filespec):\n", + " global data_files\n", + " data_files += glob.glob(f\"{path}/{filespec}\")\n", + " data_files = list(set(data_files))\n", + " print(f\"Number of files {len(data_files)} after adding {path} glob {filespec}\")\n", + " add_jsonlines_dir(f\"/data/c4_cleaned2\", \"*.gz\")\n", + " add_jsonlines_dir(f\"/data/nrc_uniq_cleaned_20210223\", \"*.gz\")\n", + " add_jsonlines_dir(f\"/data/nu_uniq_cleaned_20210225\", \"*.gz\")\n", + " datafiles" ] }, { "cell_type": "code", - "execution_count": 40, - "id": "fc9519d2", + "execution_count": 38, + "id": "7c5980cd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Number of files 209 after adding /data/oscar_nl_cleaned\n", - "95%: 199\n", - "Got 199 training files and 10 validation files\n" + "Number of files 1424 after adding /data/c4_cleaned2 glob *.gz\n", + "Number of files 1424 after adding /data/nrc_uniq_cleaned_20210223 glob *.gz\n", + "Number of files 1424 after adding /data/nu_uniq_cleaned_20210225 glob *.gz\n" + ] + } + ], + "source": [ + "train_val_files()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "969b8fa4", + "metadata": {}, + "outputs": [], + "source": [ + " random.Random(SEED).shuffle(data_files)\n", + " total = len(data_files)\n", + " print(total)\n", + " perc = 0.05\n", + " val_size = int(perc * total)\n", + " train_size = total - val_size\n", + " train = data_files[:train_size]\n", + " val = data_files[train_size:]\n", + " print(f\"Got {len(train)} training files and {perc*100} % {len(val)} validation files\")\n", + " assert list(set(train) & set(val)) == [], \"Train overlaps with test\"\n", + " return train, val" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f92c2b13", + "metadata": {}, + "outputs": [], + "source": [ + " datafiles = []\n", + " import glob\n", + " import random\n", + " SEED = 12345\n", + " def add_jsonlines_dir(path, filespec):\n", + " global data_files\n", + " data_files += glob.glob(f\"{path}/{filespec}\")\n", + " data_files = list(set(data_files))\n", + " print(f\"Number of files {len(data_files)} after adding {path} glob {filespec}\")\n", + " add_jsonlines_dir(f\"/data/c4_cleaned2\", \"*.gz\")\n", + " add_jsonlines_dir(f\"/data/nrc_uniq_cleaned_20210223\", \"*.gz\")\n", + " add_jsonlines_dir(f\"/data/nu_uniq_cleaned_20210225\", \"*.gz\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ffb5036", + "metadata": {}, + "outputs": [], + "source": [ + "datasets[\"train\"] = load_dataset(datafiles, split=\"train[5%:]\")\n", + "datasets[\"validation\"] = load_dataset(datafiles, split=\"train[5%:]\")" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "31e5a164", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of files 1424 after adding /data/c4_cleaned2 glob *.gz\n", + "Number of files 1424 after adding /data/nrc_uniq_cleaned_20210223 glob *.gz\n", + "Number of files 1424 after adding /data/nu_uniq_cleaned_20210225 glob *.gz\n", + "1424\n", + "Got 1353 training files and 5.0 % 71 validation files\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Using custom data configuration default-00e4c1e272015fdb\n" + "WARNING:datasets.builder:Using custom data configuration default-28929211ee23e224\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Downloading and preparing dataset json/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /home/dat/.cache/huggingface/datasets/json/default-00e4c1e272015fdb/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723...\n" + "Downloading and preparing dataset json/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /home/dat/.cache/huggingface/datasets/json/default-28929211ee23e224/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723...\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7fc9159a741a4853abb8fa1abcb8bd4c", + "model_id": "e7b73482da6744639826bd7a677f17ff", "version_major": 2, "version_minor": 0 }, @@ -163,24 +1315,141 @@ "output_type": "display_data" }, { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "db9fc4eb87094fa9aef909f8e8d41124", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "0 tables [00:00, ? tables/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, + "ename": "JSONDecodeError", + "evalue": "Extra data: line 2 column 1 (char 651)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mArrowInvalid\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/datasets/packaged_modules/json/json.py\u001b[0m in \u001b[0;36m_generate_tables\u001b[0;34m(self, files)\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"rb\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m pa_table = paj.read_json(\n\u001b[0m\u001b[1;32m 82\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mread_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpa_read_options\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparse_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpa_parse_options\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/pyarrow/_json.pyx\u001b[0m in \u001b[0;36mpyarrow._json.read_json\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/pyarrow/error.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.pyarrow_internal_check_status\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/pyarrow/error.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.check_status\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mArrowInvalid\u001b[0m: JSON parse error: Missing a closing quotation mark in string. in row 93", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_371965/265278772.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_val_files\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdatasets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'json'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_files\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'validation'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/datasets/load.py\u001b[0m in \u001b[0;36mload_dataset\u001b[0;34m(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, keep_in_memory, save_infos, script_version, use_auth_token, task, streaming, **config_kwargs)\u001b[0m\n\u001b[1;32m 839\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 840\u001b[0m \u001b[0;31m# Download and prepare data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 841\u001b[0;31m builder_instance.download_and_prepare(\n\u001b[0m\u001b[1;32m 842\u001b[0m \u001b[0mdownload_config\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdownload_config\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 843\u001b[0m \u001b[0mdownload_mode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdownload_mode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/datasets/builder.py\u001b[0m in \u001b[0;36mdownload_and_prepare\u001b[0;34m(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, **download_and_prepare_kwargs)\u001b[0m\n\u001b[1;32m 581\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"HF google storage unreachable. Downloading and preparing it from source\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdownloaded_from_gcs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 583\u001b[0;31m self._download_and_prepare(\n\u001b[0m\u001b[1;32m 584\u001b[0m \u001b[0mdl_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdl_manager\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverify_infos\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverify_infos\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mdownload_and_prepare_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 585\u001b[0m )\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/datasets/builder.py\u001b[0m in \u001b[0;36m_download_and_prepare\u001b[0;34m(self, dl_manager, verify_infos, **prepare_split_kwargs)\u001b[0m\n\u001b[1;32m 659\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 660\u001b[0m \u001b[0;31m# Prepare split will record examples associated to the split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 661\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_prepare_split\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_generator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mprepare_split_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 662\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 663\u001b[0m raise OSError(\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/datasets/builder.py\u001b[0m in \u001b[0;36m_prepare_split\u001b[0;34m(self, split_generator)\u001b[0m\n\u001b[1;32m 1125\u001b[0m \u001b[0mgenerator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_generate_tables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0msplit_generator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1126\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mArrowWriter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfpath\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mwriter\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1127\u001b[0;31m for key, table in utils.tqdm(\n\u001b[0m\u001b[1;32m 1128\u001b[0m \u001b[0mgenerator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\" tables\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mleave\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_verbosity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNOTSET\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1129\u001b[0m ):\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/tqdm/notebook.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 253\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 254\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm_notebook\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 255\u001b[0m \u001b[0;31m# return super(tqdm...) will not catch exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1177\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1178\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1179\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1180\u001b[0m \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.local/lib/python3.8/site-packages/datasets/packaged_modules/json/json.py\u001b[0m in \u001b[0;36m_generate_tables\u001b[0;34m(self, files)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mpa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mArrowInvalid\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"utf-8\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 87\u001b[0m raise ValueError(\n\u001b[1;32m 88\u001b[0m \u001b[0;34mf\"Not able to read records in the JSON file at {file}. \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.8/json/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(fp, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[0mkwarg\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0motherwise\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mJSONDecoder\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mused\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 292\u001b[0m \"\"\"\n\u001b[0;32m--> 293\u001b[0;31m return loads(fp.read(),\n\u001b[0m\u001b[1;32m 294\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobject_hook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobject_hook\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 295\u001b[0m \u001b[0mparse_float\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_float\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparse_int\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_int\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.8/json/__init__.py\u001b[0m in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0mparse_int\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 356\u001b[0m parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[0;32m--> 357\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 358\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.8/json/decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 339\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Extra data\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 341\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mJSONDecodeError\u001b[0m: Extra data: line 2 column 1 (char 651)" + ] + } + ], + "source": [ + "train, val = train_val_files()\n", + "datasets = load_dataset('json', data_files={'train': train, 'validation': val})" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "69445179", + "metadata": {}, + "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Dataset json downloaded and prepared to /home/dat/.cache/huggingface/datasets/json/default-00e4c1e272015fdb/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723. Subsequent calls will reuse this data.\n" + "Number of files 1024 after adding /data/c4_cleaned2 glob *json.gz\n", + "Number of files 1224 after adding /data/nrc_uniq_cleaned_20210223 glob *.gz\n", + "Number of files 1424 after adding /data/nu_uniq_cleaned_20210225 glob *.gz\n" + ] + } + ], + "source": [ + "import glob\n", + "import random\n", + "SEED = 12345\n", + "data_files = []\n", + "def add_jsonlines_dir(path, filespec):\n", + " global data_files\n", + " data_files += glob.glob(f\"{path}/{filespec}\")\n", + " data_files = list(set(data_files))\n", + " print(f\"Number of files {len(data_files)} after adding {path} glob {filespec}\")\n", + "add_jsonlines_dir(f\"/data/c4_cleaned2\", \"*json.gz\")\n", + "add_jsonlines_dir(f\"/data/nrc_uniq_cleaned_20210223\", \"*.gz\")\n", + "add_jsonlines_dir(f\"/data/nu_uniq_cleaned_20210225\", \"*.gz\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "5b1c04f8", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:datasets.builder:Using custom data configuration default-5aa1f90e962b1369\n", + "WARNING:datasets.builder:Reusing dataset json (/home/dat/.cache/huggingface/datasets/json/default-5aa1f90e962b1369/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723)\n" + ] + } + ], + "source": [ + "datasets = load_dataset('json', data_files={'train': train, 'validation': val})" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "47db602d", + "metadata": {}, + "outputs": [], + "source": [ + "dataset_iterator = iter(datasets['train'])" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "03a23a9c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'text': 'Welke school kiezen?\\nSchoolinformatie in je brievenbus ... graag of liever niet?\\nKinderen die geboren zijn tussen 15 november en 31 december 2016 stappen pas in op 1 september 2019.\\nVoor deze groep is inschrijving mogelijk tijdens de eerste voorrangsperiode: van 1 maart tot 16 maart 2018.\\n2: kinderen van personeel van de school.\\nVoor deze groep is inschrijving mogelijk tijdens de tweede voorrangsperiode: op 22 maart en 23 maart 2018.\\nVanaf 18 april tot 4mei 2018 loopt de algemene inschrijvingsperiode en dan kunnen alle kinderen ingeschreven worden.\\nTIP: Wacht niet tot de laatste dag om je kind in te schrijven. Hoe sneller je inschrijft, hoe groter de kans dat er plaats is in de school die je kiest.\\nKies bij voorkeur een school in de buurt van waar je woont; dat heeft enkel maar voordelen. Bezoek zeker vooraf de school van je keuze. Je kan er uitleg vragen en de schoolomgeving ontdekken.\\nIn welke school zijn er vrije plaatsen?\\nVanaf 30 maart 2018 vanaf 17u vind je informatie over de vrije plaatsen op www.lop.be Zo weet je hoeveel plaatsen in jouw school beschikbaar zijn.\\n• Wil je bepaalde scholen bezoeken samen met andere ouders uit je buurt? Neem contact op met School in zicht.\\nMeer informatie over inschrijven in een school vind je ook op de website van Onderwijs Vlaanderen.\\nBlijft het onduidelijk?\\nAlle scholen van Bonheiden, Mechelen, Sint-Katelijne-Waver en Zemst schrijven de kinderen in op hetzelfde moment.\\nIs je kind een jongere broer of zus die in 2016 geboren is? Maak dan een afspraak met de school.\\nIs je kind een oudere broer of zus die voor 2016 geboren is? Ga naar de school tijdens de voorrangsperiode. De inschrijvingen starten op 1 maart om 8u30 stipt en eindigen op 16 maart om 16u. Let op, het aantal plaatsen is in elke school beperkt en wie eerst komt wordt eerst ingeschreven.\\nLet op: vanaf 17 maart heeft de jongere broer/zus niet langer voorrang. Je kan hem/haar dan alleen tijdens de algemene inschrijvingsperiode inschrijven.\\nDe inschrijvingen starten op 18 april 2018 om 8u30 stipt en eindigen op 4 mei2018 om 16u. Let op, het aantal plaatsen is in elke school beperkt en wie eerst komt wordt eerst ingeschreven.\\nNeem de identiteitskaart van je kind of een ander identiteitsbewijs mee (bv. Kids-ID, paspoort, identiteitsbewijs voor kinderen onder 12 jaar).\\nSTAP 1; Je hebt een gesprek met de directeur. Je krijgt informatie over het schoolreglement en het pedagogisch project van de school. Alleen als je akkoord gaat met het schoolreglement en het pedagogisch project kan je je kind inschrijven.\\nHeeft de moeder van het kind een diploma van hoger secundair onderwijs?\\nHeeft het gezin een schooltoelage voor het schooljaar 2017-2018 en/of 2018-2019?\\nAan de hand van de antwoorden wordt je kind ingedeeld als een indicatorleerling of een niet-indicatorleerling.\\nSTAP 3; Op basis van het aantal beschikbare plaatsen wordt je kind wel of niet ingeschreven. Er zijn 3 mogelijkheden.\\nJe kind wordt onmiddellijk ingeschreven in het inschrijvingsregister van de school.\\nEr is nog wel plaats in de school maar niet meer in de groep waarvoor je kind in aanmerking komt.\\n- De directeur kan je niet onmiddellijk zeggen of er uiteindelijk plaats zal zijn of niet.\\n- Je kind komt op de wachtlijst van de school. De wachtlijst geeft geen garantie op een plaats. Als een ander ingeschreven kind de school verlaat, kan de eerste op de wachtlijst zich inschrijven.\\n- Wat moet je doen? Zoek zo snel mogelijk een andere school voor je kind.\\nMechelen heeft een uitgebreid onderwijsaanbod. Op www.mechelen.be/scholen vind je een overzicht.\\nHet stadsbestuur vindt het erg belangrijk dat ouders voldoende geïnformeerd worden over het scholenaanbod in Mechelen en de inschrijvingsprocedure in Mechelse scholen. Daarom krijgen de scholen en een aantal organisaties de mogelijkheid om aan inwoners van Mechelen met schoolgaande kinderen via de post informatie te bezorgen over het basis- of secundair onderwijs en de inschrijvingsprocedure in Mechelse scholen..\\nDeze informatie kan in functie zijn van de leeftijdsgroepen 2,5 jarigen (kleuters), 6 jarigen (lagere school) en de 12 jarigen (secundaire school).\\nGrote Markt 21, 2800 Mechelen.', 'timestamp': datetime.datetime(2018, 10, 23, 5, 45), 'url': 'https://www.mechelen.be/schrijf-je-kind-tijdig-in-op-school', 'id': 42}\n" + ] + } + ], + "source": [ + "print(next(dataset_iterator))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "fc9519d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of files 0 after adding /data/oscar_nl_cleaned2\n" + ] + }, + { + "ename": "NameError", + "evalue": "name 'data_dir' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_371965/3501862563.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;31m#add_jsonlines_dir(\"/data/nu_cleaned_idtextfmt\",\"*.gz\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0madd_jsonlines_dir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{data_dir}/c4_cleaned\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"*73*.gz\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'data_dir' is not defined" ] } ], @@ -200,9 +1469,13 @@ " print(f\"Number of files {len(data_files)} after adding {path}\")\n", " \n", "#add_jsonlines_dir(\"/home/dat/subset_c4_cleannl\",\"*.gz\") \n", - "add_jsonlines_dir(\"/data/oscar_nl_cleaned\",\"*.gz\")\n", + "add_jsonlines_dir(\"/data/oscar_nl_cleaned2\",\"*.gz\")\n", "#add_jsonlines_dir(\"/data/nrc_cleaned_idtextfmt\",\"*.gz\")\n", "#add_jsonlines_dir(\"/data/nu_cleaned_idtextfmt\",\"*.gz\")\n", + "\n", + "add_jsonlines_dir(f\"{data_dir}/c4_cleaned\", \"*73*.gz\")\n", + "\n", + " \n", "random.Random(SEED).shuffle(data_files)\n", "total = len(data_files)\n", "val_size = int(0.05 * total)\n", @@ -218,6 +1491,19 @@ "assert list(set(train) & set(val)) == [], 'train overlaps with test'\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "71cac0b7", + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import Dataset\n", + "datasets['train'] = Dataset.from_file(\"/home/dat/.cache/huggingface/datasets/json/default-3eb349358dcf6436/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723/json-train.arrow\") \n", + "datasets['validation'] = Dataset.from_file(\"/home/dat/.cache/huggingface/datasets/json/default-3eb349358dcf6436/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723/json-validation.arrow\") \n", + "\n" + ] + }, { "cell_type": "code", "execution_count": 41, diff --git a/checkpoint_60000 b/checkpoint_60000 new file mode 100644 index 0000000000000000000000000000000000000000..9c9239859aa999d490233fbef854f1101b83b3d6 --- /dev/null +++ b/checkpoint_60000 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73e6d7222b2cee297be0891db385dcce6e0cbff6ec3697c08118513955f8aaf7 +size 769729450 diff --git a/events.out.tfevents.1626173264.t1v-n-f5c06ea1-w-0.340852.3.v2 b/events.out.tfevents.1626173264.t1v-n-f5c06ea1-w-0.340852.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..7700ce070b481fbede7f8666b1553f3046e542fe --- /dev/null +++ b/events.out.tfevents.1626173264.t1v-n-f5c06ea1-w-0.340852.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73fdfc3eb9d8111b1e3460227717a3942adfe9263bca08b7fd2bfab9af98d9a1 +size 38186 diff --git a/events.out.tfevents.1626174131.t1v-n-f5c06ea1-w-0.343920.3.v2 b/events.out.tfevents.1626174131.t1v-n-f5c06ea1-w-0.343920.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..0b618567ca6704930d17b06ce8f7aa6409ac0bc5 --- /dev/null +++ b/events.out.tfevents.1626174131.t1v-n-f5c06ea1-w-0.343920.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc6f0b5b354bd4d8d13834613ece71ac9d948186313bc3fde5e2e132a1c9cab +size 40 diff --git a/events.out.tfevents.1626174670.t1v-n-f5c06ea1-w-0.346512.3.v2 b/events.out.tfevents.1626174670.t1v-n-f5c06ea1-w-0.346512.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..ef254567b0250227c39ad73f86d91234454f2874 --- /dev/null +++ b/events.out.tfevents.1626174670.t1v-n-f5c06ea1-w-0.346512.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74cf77c0a672ad1201614ba6642a4f3a27b9cf021d0e88eb362c7f38ee86304 +size 40 diff --git a/events.out.tfevents.1626175237.t1v-n-f5c06ea1-w-0.349243.3.v2 b/events.out.tfevents.1626175237.t1v-n-f5c06ea1-w-0.349243.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..6d0c6276aa45fe8ffa43c6ce5073cfc64175a974 --- /dev/null +++ b/events.out.tfevents.1626175237.t1v-n-f5c06ea1-w-0.349243.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be5c2acf821fd2ce776ff5e434706cb933a0fa323f0bb1a82dadd832f1f589d4 +size 40 diff --git a/events.out.tfevents.1626176074.t1v-n-f5c06ea1-w-0.351681.3.v2 b/events.out.tfevents.1626176074.t1v-n-f5c06ea1-w-0.351681.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..2bd0a7192f7426ad07ccc105b192a8d7fac15915 --- /dev/null +++ b/events.out.tfevents.1626176074.t1v-n-f5c06ea1-w-0.351681.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b085d5029d052defe00b26c54b6357e9d05cbc5ad38cdd2f12537ed0b90008d2 +size 441341 diff --git a/events.out.tfevents.1626180467.t1v-n-f5c06ea1-w-0.354027.3.v2 b/events.out.tfevents.1626180467.t1v-n-f5c06ea1-w-0.354027.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..8ee1515d4efe084c0f5e47d3c8e4452b3fdd5932 --- /dev/null +++ b/events.out.tfevents.1626180467.t1v-n-f5c06ea1-w-0.354027.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:973eec9b2b17e54f3ee35dc0c4b85a4a3ecf5488cb59f5619d7c635641bfe7b6 +size 40 diff --git a/events.out.tfevents.1626180750.t1v-n-f5c06ea1-w-0.355855.3.v2 b/events.out.tfevents.1626180750.t1v-n-f5c06ea1-w-0.355855.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..0353884509118b71f02ad8b4aed53a67f5fe4cbf --- /dev/null +++ b/events.out.tfevents.1626180750.t1v-n-f5c06ea1-w-0.355855.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:013fc500b7fdd46262ee2b2ed5a3624249adef426d0b134944080ccf90d363ed +size 40 diff --git a/events.out.tfevents.1626181600.t1v-n-f5c06ea1-w-0.357816.3.v2 b/events.out.tfevents.1626181600.t1v-n-f5c06ea1-w-0.357816.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..dce8dcbcfd05c2ee72f4df774eed651ecbbb392d --- /dev/null +++ b/events.out.tfevents.1626181600.t1v-n-f5c06ea1-w-0.357816.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d4a519b8f1c293258e292768822980b487ef0e02bbfe9d6a3132b8c2fdd791 +size 40 diff --git a/events.out.tfevents.1626181889.t1v-n-f5c06ea1-w-0.360037.3.v2 b/events.out.tfevents.1626181889.t1v-n-f5c06ea1-w-0.360037.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..96b9ec12defa025742d9f762660fa0ff1ab7213f --- /dev/null +++ b/events.out.tfevents.1626181889.t1v-n-f5c06ea1-w-0.360037.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c1ed9142ba98f2f7197e2a44361331a8c112af5dba98d7fc9f0bcab6228ae8c +size 40 diff --git a/events.out.tfevents.1626182175.t1v-n-f5c06ea1-w-0.362298.3.v2 b/events.out.tfevents.1626182175.t1v-n-f5c06ea1-w-0.362298.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..7a880c89f2b2db62cbc2feb163fb87688f247eb9 --- /dev/null +++ b/events.out.tfevents.1626182175.t1v-n-f5c06ea1-w-0.362298.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29cc2c143c306c4619802094513459dbb71c4730d3cdfb879e7224923ddfe7ea +size 40 diff --git a/events.out.tfevents.1626182874.t1v-n-f5c06ea1-w-0.365284.3.v2 b/events.out.tfevents.1626182874.t1v-n-f5c06ea1-w-0.365284.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..94175aa826220f778b1fe8a3ea13777fa050d7d2 --- /dev/null +++ b/events.out.tfevents.1626182874.t1v-n-f5c06ea1-w-0.365284.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24aa4302db5d02121389fc7f8944025588034aedd21f772c2b71224e3a0b0d13 +size 220634 diff --git a/events.out.tfevents.1626184460.t1v-n-f5c06ea1-w-0.369028.3.v2 b/events.out.tfevents.1626184460.t1v-n-f5c06ea1-w-0.369028.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..2c1a61e7909ecd5ba10b7ea8a605a64059077bc1 --- /dev/null +++ b/events.out.tfevents.1626184460.t1v-n-f5c06ea1-w-0.369028.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5631bf443386a4e37d77053e55ba4517153d5f6d7f77b616258d9c78e6901f +size 367772 diff --git a/events.out.tfevents.1626242600.t1v-n-f5c06ea1-w-0.491835.3.v2 b/events.out.tfevents.1626242600.t1v-n-f5c06ea1-w-0.491835.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..1e0413e8276b74856a3af43f4224558e100dd690 --- /dev/null +++ b/events.out.tfevents.1626242600.t1v-n-f5c06ea1-w-0.491835.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f94f6c2d80b0e0d6247997634649101caefa3ad8ab4f408b529ad38f86c8770 +size 40 diff --git a/events.out.tfevents.1626285315.t1v-n-f5c06ea1-w-0.533662.3.v2 b/events.out.tfevents.1626285315.t1v-n-f5c06ea1-w-0.533662.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..87363498e33d461dd2a08a6a591b458c707c50e0 --- /dev/null +++ b/events.out.tfevents.1626285315.t1v-n-f5c06ea1-w-0.533662.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29b681f16c441caf85381c9def58d19f4479a2460146d2cfb68991f8327f01fe +size 40 diff --git a/events.out.tfevents.1626286793.t1v-n-f5c06ea1-w-0.547087.3.v2 b/events.out.tfevents.1626286793.t1v-n-f5c06ea1-w-0.547087.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..39ee85f427f33d05db2a1c004dc36db126c0f5aa --- /dev/null +++ b/events.out.tfevents.1626286793.t1v-n-f5c06ea1-w-0.547087.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53d63b11450875138751afac48c611f4da76fadc0affb0ec98896b35dbad9728 +size 40 diff --git a/events.out.tfevents.1626287584.t1v-n-f5c06ea1-w-0.550207.3.v2 b/events.out.tfevents.1626287584.t1v-n-f5c06ea1-w-0.550207.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..c36aa13c9743ed2693159f6b683ba1d1851156fd --- /dev/null +++ b/events.out.tfevents.1626287584.t1v-n-f5c06ea1-w-0.550207.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62cc6dc4bf215d99f8685629bf632f82d65fc7f1127d876ded332b31b5432064 +size 40 diff --git a/events.out.tfevents.1626288936.t1v-n-f5c06ea1-w-0.553832.3.v2 b/events.out.tfevents.1626288936.t1v-n-f5c06ea1-w-0.553832.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..ea41bfaca16c2a1704d3ba859012dc303bd4d110 --- /dev/null +++ b/events.out.tfevents.1626288936.t1v-n-f5c06ea1-w-0.553832.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fccf6070edac76c190b8bb8de4e37b889dd1b18835777203f9d16ac658aaf71 +size 40 diff --git a/events.out.tfevents.1626290714.t1v-n-f5c06ea1-w-0.557554.3.v2 b/events.out.tfevents.1626290714.t1v-n-f5c06ea1-w-0.557554.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..da2254da2feed4f01cb7eee3751c4c61d499257f --- /dev/null +++ b/events.out.tfevents.1626290714.t1v-n-f5c06ea1-w-0.557554.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d46028802a38f383ce27081e90ff848e3da863ac08c341f101eed1b20a39556c +size 40 diff --git a/events.out.tfevents.1626292080.t1v-n-f5c06ea1-w-0.560928.3.v2 b/events.out.tfevents.1626292080.t1v-n-f5c06ea1-w-0.560928.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..04fde11f6f8158b3f35a6e9f79f86c692e906c37 --- /dev/null +++ b/events.out.tfevents.1626292080.t1v-n-f5c06ea1-w-0.560928.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e89d0090ae1228c609a140c2a20fbdfb208480a0dd16aced968756947a93f0 +size 147065 diff --git a/events.out.tfevents.1626292866.t1v-n-f5c06ea1-w-0.563390.3.v2 b/events.out.tfevents.1626292866.t1v-n-f5c06ea1-w-0.563390.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..68fca31131fb7d3625de9d2f730ad65a5da7b3a0 --- /dev/null +++ b/events.out.tfevents.1626292866.t1v-n-f5c06ea1-w-0.563390.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5607707732c41fb3bac9b56702cf2a006ba526d98638e0352ba54e809c6eff +size 40 diff --git a/events.out.tfevents.1626293250.t1v-n-f5c06ea1-w-0.565261.3.v2 b/events.out.tfevents.1626293250.t1v-n-f5c06ea1-w-0.565261.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..1e6468e4d65afc8ae03343977348842ea5dcefb9 --- /dev/null +++ b/events.out.tfevents.1626293250.t1v-n-f5c06ea1-w-0.565261.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83bed69057844c7af14e165d87c9678d28135297ab5bd374d1e0d80ebd31966f +size 221057 diff --git a/events.out.tfevents.1626294676.t1v-n-f5c06ea1-w-0.568447.3.v2 b/events.out.tfevents.1626294676.t1v-n-f5c06ea1-w-0.568447.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..a7f4c53129d2aefedef8cd0b40e94959902d2c76 --- /dev/null +++ b/events.out.tfevents.1626294676.t1v-n-f5c06ea1-w-0.568447.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:050b6dc69ea5a9946fc01c76d67ea00913117399f1a37e0f24db39f39c52e76f +size 73565 diff --git a/events.out.tfevents.1626295212.t1v-n-f5c06ea1-w-0.570637.3.v2 b/events.out.tfevents.1626295212.t1v-n-f5c06ea1-w-0.570637.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..616ecf3caec3d1dd869afbe0d76416530405bc9e --- /dev/null +++ b/events.out.tfevents.1626295212.t1v-n-f5c06ea1-w-0.570637.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2818b40b384ff7f5a57fe1c4994ebbd02140f7221904f527cfc0a9a115334a79 +size 184532 diff --git a/events.out.tfevents.1626296457.t1v-n-f5c06ea1-w-0.573688.3.v2 b/events.out.tfevents.1626296457.t1v-n-f5c06ea1-w-0.573688.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..8d1730afda171ca1ce12e20779c407e5204494d4 --- /dev/null +++ b/events.out.tfevents.1626296457.t1v-n-f5c06ea1-w-0.573688.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3d8a6aa5b0177a3c337963bad77cc5cea9ed722032941dbac474d03b5a3261 +size 40 diff --git a/events.out.tfevents.1626296630.t1v-n-f5c06ea1-w-0.575437.3.v2 b/events.out.tfevents.1626296630.t1v-n-f5c06ea1-w-0.575437.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..98ba3d480ece5014084f97a7ce7c4991b7193203 --- /dev/null +++ b/events.out.tfevents.1626296630.t1v-n-f5c06ea1-w-0.575437.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:932b70a150d991f6939f853c7b54516d5309f2d6c19761fa96a50999bf2199e7 +size 147993 diff --git a/flax_model.msgpack b/flax_model.msgpack index 9e3a0e84b8c57848e79e6a12ca3faa107265e78a..751f890f6238f0851bf4bbae770f4be4d705ec53 100644 --- a/flax_model.msgpack +++ b/flax_model.msgpack @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:19dddbba6ad2a0aa9c5c22f1b9750b90fcd0b7c8f3007cbd6af9a17d447fa417 -size 256576390 +oid sha256:422812fccdda54c02543ac5e994b33b54e510e0474439fbe9360d5190787d38e +size 510090043 diff --git a/run.sh b/run.sh index 26f81f6e68b753370bb00e0561378e7c8da64f6a..60d89bd0284be69aa160df914b62f97282e07e18 100644 --- a/run.sh +++ b/run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -export TOKENIZERS_PARALLELISM=0 +#export TOKENIZERS_PARALLELISM=0 python ./run_mlm_flax.py \ --push_to_hub \ @@ -14,18 +14,20 @@ python ./run_mlm_flax.py \ --overwrite_output_dir \ --adam_beta1="0.9" \ --adam_beta2="0.98" \ - --logging_steps="500" \ - --eval_steps="92768" \ - --num_train_epochs="5" \ - --preprocessing_num_workers="64" \ - --save_steps="20000" \ - --learning_rate="5e-5" \ + --logging_steps="250" \ + --eval_steps="500" \ + --num_train_epochs="3" \ + --preprocessing_num_workers="96" \ + --save_steps="1250" \ + --learning_rate="1e-4" \ --per_device_train_batch_size="2" \ --per_device_eval_batch_size="2" \ --save_total_limit="5"\ - --gradient_accumulation_steps="2" \ + --max_eval_samples="500"\ + --overwrite_cache False \ + --gradient_accumulation_steps="4" \ + #--resume_from_checkpoint="./"\ #--adafactor \ #--dtype="bfloat16" \ - #--resume_from_checkpoint="./"\ diff --git a/run_mlm_flax.py b/run_mlm_flax.py index d713d14ffa71329e96a4a9c0d723ba3e6572a51f..ff8cb6505219a4cf0637e98295a066e21a1019e0 100644 --- a/run_mlm_flax.py +++ b/run_mlm_flax.py @@ -20,20 +20,18 @@ text file or a dataset. Here is the full list of checkpoints on the hub that can be fine-tuned by this script: https://huggingface.co/models?filter=masked-lm """ -import shutil import logging import os import sys import time from dataclasses import dataclass, field -from ast import Str # You can also adapt this script on your own masked language modeling task. Pointers for this are left as comments. from pathlib import Path from typing import Dict, List, Optional, Tuple import numpy as np -from datasets import load_dataset +from datasets import load_dataset, DatasetDict from tqdm import tqdm import flax @@ -56,13 +54,12 @@ from transformers import ( is_tensorboard_available, set_seed, ) -from transformers.testing_utils import CaptureLogger -from flax.serialization import to_bytes, from_bytes -from importlib.util import find_spec +import json from flax.training import checkpoints from flax.jax_utils import unreplicate from flax.training.checkpoints import save_checkpoint, restore_checkpoint -import json +from importlib.util import find_spec + MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys()) MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) @@ -104,8 +101,10 @@ class ModelArguments: "help": "Floating-point format in which the model weights should be initialized and trained. Choose one of `[float32, float16, bfloat16]`." }, ) - - + save_optimizer: Optional[bool] = field( + default=True, + metadata={"help": "Whether to store full train state including optimizer."}, + ) @dataclass @@ -120,11 +119,6 @@ class DataTrainingArguments: dataset_config_name: Optional[str] = field( default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} ) - train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."}) - validation_file: Optional[str] = field( - default=None, - metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."}, - ) train_ref_file: Optional[str] = field( default=None, metadata={"help": "An optional input train ref data file for whole word masking in Chinese."}, @@ -136,6 +130,9 @@ class DataTrainingArguments: overwrite_cache: bool = field( default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} ) + + + validation_split_percentage: Optional[int] = field( default=5, metadata={ @@ -167,6 +164,17 @@ class DataTrainingArguments: default=False, metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."}, ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + }, + ) + + + + @flax.struct.dataclass @@ -266,33 +274,73 @@ def write_eval_metric(summary_writer, eval_metrics, step): for metric_name, value in eval_metrics.items(): summary_writer.scalar(f"eval_{metric_name}", value, step) -def mb_item(x): - return x.item() if hasattr(x, "item") else x - -#checkpoint functions - - - - - -def rotate_checkpoints(ckpt_dir: str, save_total_limit: int): - "Removes older checkpoints so that `save_total_limit` checkpoints are kept" - # TODO: what to remove is decided using step number only, we might want to improve that - ckpts = [str(x) for x in Path(ckpt_dir).glob("ckpt-*")] - # sort checkpoints by step - ckpts_sorted = sorted(ckpts, key=lambda x: int(x.split('-')[-1])) - ckpts_to_delete = ckpts_sorted[:-save_total_limit] - for ckpt in ckpts_to_delete: - logger.info(f"Deleting older checkpoint [{ckpt}] due to save_total_limit ({save_total_limit})") - shutil.rmtree(ckpt) +def _zeros_tree_like(inp_tree): + return jax.tree_map(jnp.zeros_like, inp_tree) + +def fake_update(state): + fake_updates = _zeros_tree_like(state.params) + _, new_inner_opt_state = state.tx.inner_opt.update(fake_updates, state.opt_state.inner_opt_state, state.params) + opt_state = state.opt_state + new_opt_state = optax.MultiStepsState(mini_step=opt_state.mini_step, + gradient_step=opt_state.gradient_step, + inner_opt_state=new_inner_opt_state, + acc_grads=opt_state.acc_grads) + return state.replace(opt_state=new_opt_state) + +def reinstantiate_states(opt_state): + new_state = [] + for state in opt_state: + cls = getattr(optax, type(state).__name__) + new_state.append(cls(**{k:getattr(state, k) for k in state._fields})) + return new_state + +def restore_model_checkpoint(save_dir, state): + logger.info(f"RESTORING CHECKPOINT FROM {save_dir}...") + with open(os.path.join(save_dir, "flax_model.msgpack"), "rb") as f: + params = from_bytes(state.params, f.read()) + + with open(os.path.join(save_dir, "opt_state.msgpack"), "rb") as f: + opt_state = from_bytes(state.opt_state, f.read()) + + with open(os.path.join(save_dir, "training_state.json"), "r") as f: + training_state = json.load(f) + step = training_state["step"] + + logger.info("checkpoint restored") + # reinstantiate inner opt state to avoid type conflict + if hasattr(opt_state, "inner_opt_state"): + print("restoring state of multisteps optimizer") + inner_opt_state = reinstantiate_states(opt_state.inner_opt_state) + ms_state_dict = {k:getattr(state.opt_state, k) for k in state.opt_state._fields} + ms_state_dict["inner_opt_state"] = inner_opt_state + opt_state = optax.MultiStepsState(**ms_state_dict) + + return state.replace(step=step, params=params, opt_state=opt_state) + +def save_model_checkpoint(model, save_dir, state, with_opt:bool=True, push_to_hub:bool=False): + """ + If `push_to_hub` is True, will save to `save_dir`. Otherwise will save to `save_dir/ckpt-{step}`. + """ + state = jax_utils.unreplicate(state) + logger.info(f"SAVING CHECKPOINT IN {save_dir}...") + if not push_to_hub: + save_dir = f"{save_dir}/ckpt-{mb_item(state.step)-1}" + model.save_pretrained( + save_dir, + params=state.params, + push_to_hub=push_to_hub, + commit_message=f"Saving weights and logs at step {mb_item(state.step)-1}", + ) + if with_opt: + with open(os.path.join(save_dir, "opt_state.msgpack"), "wb") as f: + f.write(to_bytes(state.opt_state)) + with open(os.path.join(save_dir, "training_state.json"), "w") as f: + json.dump({"step": state.step.item()}, f) + logger.info("checkpoint saved") - -class TrainState(train_state.TrainState): - grad_accum: jnp.ndarray - if __name__ == "__main__": # See all possible arguments in src/transformers/training_args.py # or by passing the --help flag to this script. @@ -360,52 +408,70 @@ if __name__ == "__main__": cache_dir=model_args.cache_dir, ) else: - #data_files = {} - #if data_args.train_file is not None: - # data_files["train"] = data_args.train_file - #if data_args.validation_file is not None: - # data_files["validation"] = data_args.validation_file - #extension = data_args.train_file.split(".")[-1] - #if extension == "txt": - # extension = "text" - #datasets = load_dataset(extension, data_files=data_files, cache_dir=model_args.cache_dir) - - #data_dir = "/home/yeb" - # data_dir = "/home/yeb/Developer/data" + import glob + import random data_files = [] - def train_val_files(): - import glob - import random - SEED = 42 - def add_jsonlines_dir(path): - global data_files - data_files += glob.glob(f"{path}/*.gz") - - add_jsonlines_dir("/home/dat/subset_c4_cleannl") - add_jsonlines_dir("/data/oscar_nl_cleaned") - add_jsonlines_dir("/data/nrc_cleaned_idtextfmt") - add_jsonlines_dir("/data/nu_cleaned_idtextfmt") - random.Random(SEED).shuffle(data_files) - total = len(data_files) - val_size = int(0.05 * total) - train_size = total - val_size - print(f"95%: {train_size}") - train = data_files[:train_size] - val = data_files[train_size:] - print(f"Got {len(train)} training files and {len(val)} validation files") - assert list(set(train) & set(val)) == [], "Train overlaps with test" - return train, val - train, val = train_val_files() - datasets = load_dataset('json', data_files={'train': train, 'validation': val}) - datasets["train"] = datasets["train"].select(range(int(0.8*len(datasets["train"])))) - datasets["validation"] = datasets["validation"].select(range(int(0.8*len(datasets["validation"])))) - #datasets["train"] = datasets["train"].select(range(10000)) - #datasets["validation"] = datasets["validation"].select(range(10000)) + def add_jsonlines_dir(path, filespec): + global data_files + data_files += glob.glob(f"{path}/{filespec}") + data_files = list(set(data_files)) + print(f"Number of files {len(data_files)} after adding {path} glob {filespec}") + add_jsonlines_dir(f"/data/c4_cleaned2", "*.gz") + add_jsonlines_dir(f"/data/nrc_uniq_cleaned_20210223", "*.gz") + add_jsonlines_dir(f"/data/nu_uniq_cleaned_20210225", "*.gz") + random.Random(42).shuffle(data_files) + total = len(data_files) + print(total) + perc = 0.05 + val_size = int(perc * total) + train_size = total - val_size + train = data_files[:train_size] + val = data_files[train_size:] + print(f"Got {len(train)} training files and {perc * 100} % {len(val)} validation files") + assert list(set(train) & set(val)) == [], "Train overlaps with test" + load_grouped = True + if not load_grouped: + datasets = load_dataset('json', data_files={'train': train, 'validation': val}) + + #from datasets import Dataset + #dataset = Dataset.from_file("/home/dat/.cache/huggingface/datasets/json/default-9add402b38836560/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723/json-train.arrow") + #dataset = Dataset.from_file("/home/dat/.cache/huggingface/datasets/json/default-9add402b38836560/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723/json-validation.arrow") + def mb_item(x): + return x.item() if hasattr(x, "item") else x + def save_model_checkpoint(model, save_dir, state, with_opt:bool=True, push_to_hub:bool=False): + """ + If `push_to_hub` is True, will save to `save_dir`. Otherwise will save to `save_dir/ckpt-{step}`. + """ + state = jax_utils.unreplicate(state) + logger.info(f"SAVING CHECKPOINT IN {save_dir}...") + if not push_to_hub: + save_dir = f"{save_dir}/ckpt-{mb_item(state.step)-1}" + model.save_pretrained( + save_dir, + params=state.params, + push_to_hub=push_to_hub, + commit_message=f"Saving weights and logs at step {mb_item(state.step)-1}", + ) + if with_opt: + with open(os.path.join(save_dir, "opt_state.msgpack"), "wb") as f: + f.write(to_bytes(state.opt_state)) + with open(os.path.join(save_dir, "training_state.json"), "w") as f: + json.dump({"step": state.step.item()}, f) + logger.info("checkpoint saved") + + # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at + # https://huggingface.co/docs/datasets/loading_datasets.html. + + # Load pretrained model and tokenizer + + # Distributed training: + # The .from_pretrained methods guarantee that only one local process can concurrently + # download model & vocab. if model_args.config_name: config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir) elif model_args.model_name_or_path: @@ -430,90 +496,97 @@ if __name__ == "__main__": # Preprocessing the datasets. # First we tokenize all the texts. - if training_args.do_train: - column_names = datasets["train"].column_names - else: - column_names = datasets["validation"].column_names - text_column_name = "text" if "text" in column_names else column_names[0] - - max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length) + if load_grouped: + logger.info("Loading tokenized and grouped dataset") + tokenized_datasets = DatasetDict.load_from_disk("/data/tokenized_data") + logger.info("Setting max validation examples to ") + print(f"Number of validation examples {data_args.max_eval_samples}") + tokenized_datasets["train"]= tokenized_datasets["train"].select(range(20000)) + if data_args.max_eval_samples is not None: + tokenized_datasets["validation"] = tokenized_datasets["validation"].select(range(data_args.max_eval_samples)) + else: + if training_args.do_train: + column_names = datasets["train"].column_names + else: + column_names = datasets["validation"].column_names + text_column_name = "text" if "text" in column_names else column_names[0] + + max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length) + + if data_args.line_by_line: + # When using line_by_line, we just tokenize each nonempty line. + padding = "max_length" if data_args.pad_to_max_length else False + + def tokenize_function(examples): + # Remove empty lines + examples = [line for line in examples if len(line) > 0 and not line.isspace()] + return tokenizer( + examples, + return_special_tokens_mask=True, + padding=padding, + truncation=True, + max_length=max_seq_length, + ) - if data_args.line_by_line: - # When using line_by_line, we just tokenize each nonempty line. - padding = "max_length" if data_args.pad_to_max_length else False - - def tokenize_function(examples): - # Remove empty lines - examples = [line for line in examples if len(line) > 0 and not line.isspace()] - return tokenizer( - examples, - return_special_tokens_mask=True, - padding=padding, - truncation=True, - max_length=max_seq_length, + tokenized_datasets = datasets.map( + tokenize_function, + input_columns=[text_column_name], + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, ) - tokenized_datasets = datasets.map( - tokenize_function, - input_columns=[text_column_name], - batched=True, - num_proc=data_args.preprocessing_num_workers, - remove_columns=column_names, - load_from_cache_file=not data_args.overwrite_cache, - ) - - else: - # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts. - # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more - # efficient when it receives the `special_tokens_mask`. - def tokenize_function(examples): - return tokenizer(examples[text_column_name], return_special_tokens_mask=True) - - tokenized_datasets = datasets.map( - tokenize_function, - batched=True, - num_proc=data_args.preprocessing_num_workers, - remove_columns=column_names, - load_from_cache_file=not data_args.overwrite_cache, - ) - - # Main data processing function that will concatenate all texts from our dataset and generate chunks of - # max_seq_length. - def group_texts(examples): - # Concatenate all texts. - concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()} - total_length = len(concatenated_examples[list(examples.keys())[0]]) - # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can - # customize this part to your needs. - if total_length >= max_seq_length: - total_length = (total_length // max_seq_length) * max_seq_length - # Split by chunks of max_len. - result = { - k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)] - for k, t in concatenated_examples.items() - } - return result - - # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a - # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value - # might be slower to preprocess. - # - # To speed up this part, we use multiprocessing. See the documentation of the map method for more information: - # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map - lm_datasets = tokenized_datasets.map( - group_texts, - batched=True, - batch_size=100, - num_proc=data_args.preprocessing_num_workers, - load_from_cache_file=not data_args.overwrite_cache, - ) - train_dataset = lm_datasets["train"] - eval_dataset = lm_datasets["validation"] - + else: + # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts. + # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more + # efficient when it receives the `special_tokens_mask`. + def tokenize_function(examples): + return tokenizer(examples[text_column_name], return_special_tokens_mask=True) + + tokenized_datasets = datasets.map( + tokenize_function, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + ) + # Main data processing function that will concatenate all texts from our dataset and generate chunks of + # max_seq_length. + def group_texts(examples): + # Concatenate all texts. + concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()} + total_length = len(concatenated_examples[list(examples.keys())[0]]) + # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can + # customize this part to your needs. + if total_length >= max_seq_length: + total_length = (total_length // max_seq_length) * max_seq_length + # Split by chunks of max_len. + result = { + k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)] + for k, t in concatenated_examples.items() + } + return result + + # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a + # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value + # might be slower to preprocess. + # + # To speed up this part, we use multiprocessing. See the documentation of the map method for more information: + # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map + tokenized_datasets = tokenized_datasets.map( + group_texts, + batched=True, + num_proc=data_args.preprocessing_num_workers, + load_from_cache_file=not data_args.overwrite_cache, + ) + #tokenized_datasets.save_to_disk("/data/tokenized_data") + #print ("tokenized_datasets saved to disk") + # Enable tensorboard only on the master node has_tensorboard = is_tensorboard_available() if has_tensorboard and jax.process_index() == 0: @@ -531,7 +604,6 @@ if __name__ == "__main__": "Unable to display metrics through TensorBoard because the package is not installed: " "Please run pip install tensorboard to enable." ) - # enable wandb tracking has_wandb = find_spec("wandb") is not None if jax.process_index() == 0 and has_wandb and ("wandb" in training_args.report_to): try: @@ -547,7 +619,6 @@ if __name__ == "__main__": except ImportError as e: print(e) has_wandb = False - # Data collator # This one will take care of randomly masking the tokens. data_collator = FlaxDataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=data_args.mlm_probability) @@ -567,10 +638,10 @@ if __name__ == "__main__": # Store some constant num_epochs = int(training_args.num_train_epochs) - train_batch_size = int(training_args.per_device_train_batch_size) * jax.device_count() + train_batch_size = int(training_args.per_device_train_batch_size) * jax.device_count() * training_args.gradient_accumulation_steps eval_batch_size = int(training_args.per_device_eval_batch_size) * jax.device_count() - num_train_steps = len(train_dataset) // train_batch_size * num_epochs + num_train_steps = len(tokenized_datasets["train"]) // train_batch_size * num_epochs # Create learning rate schedule warmup_fn = optax.linear_schedule( @@ -605,6 +676,7 @@ if __name__ == "__main__": learning_rate=linear_decay_lr_schedule_fn, ) else: + from optax import clip_by_global_norm optimizer = optax.adamw( learning_rate=linear_decay_lr_schedule_fn, b1=training_args.adam_beta1, @@ -613,22 +685,26 @@ if __name__ == "__main__": weight_decay=training_args.weight_decay, mask=decay_mask_fn, ) + optimizer = optax.chain( + optax.clip_by_global_norm(1.), + optimizer + ) - #if training_args.gradient_accumulation_steps > 1: - # optimizer = optax.MultiSteps(optimizer, training_args.gradient_accumulation_steps) - #grad_accum_steps = training_args.gradient_accumulation_steps + if training_args.gradient_accumulation_steps > 1: + optimizer = optax.MultiSteps(optimizer, training_args.gradient_accumulation_steps) + grad_accum_steps = training_args.gradient_accumulation_steps # Setup train state - - - state = TrainState.create(apply_fn=model.__call__, params=model.params, tx=optimizer,grad_accum=jax.tree_map(jnp.zeros_like, model.params)) - + state = train_state.TrainState.create(apply_fn=model.__call__, params=model.params, tx=optimizer) + if training_args.resume_from_checkpoint: - state = restore_checkpoint(training_args.resume_from_checkpoint, state) - resume_step = mb_item(state.step.item()) + state = restore_model_checkpoint(training_args.resume_from_checkpoint, state) + resume_step = mb_item(state.step) + if training_args.adafactor: + state = fake_update(state) else: resume_step = 0 - + # Define gradient update step fn def train_step(state, batch, dropout_rng): @@ -646,30 +722,17 @@ if __name__ == "__main__": # take average loss = loss.sum() / label_mask.sum() - return loss / training_args.gradient_accumulation_steps + return loss grad_fn = jax.value_and_grad(loss_fn) - loss, grads = grad_fn(state.params) - grad_accum = jax.tree_multimap(lambda x, y: x + y, grads, state.grad_accum) - - def update_fn(): - grads = jax.tree_map(lambda x: x / training_args.gradient_accumulation_steps, grad_accum) - grads = jax.lax.pmean(grad_accum, "batch") - new_state = state.apply_gradients(grads=grads,grad_accum=jax.tree_map(jnp.zeros_like, grads)) - return new_state - - new_state = jax.lax.cond( - state.step % training_args.gradient_accumulation_steps == 0, - lambda _: update_fn(), - lambda _: state.replace(grad_accum=grad_accum, step=state.step + 1), - None, - ) - + loss, grad = grad_fn(state.params) + grad = jax.lax.pmean(grad, "batch") + new_state = state.apply_gradients(grads=grad) + metrics = jax.lax.pmean( - {"loss": loss, "learning_rate": linear_decay_lr_schedule_fn(state.step)}, axis_name="batch" # + {"loss": loss, "learning_rate": linear_decay_lr_schedule_fn(state.step // grad_accum_steps)}, axis_name="batch" ) - #return new_state.replace(new_dropout_rng=new_dropout_rng), metrics return new_state, metrics, new_dropout_rng # Create parallel version of the train step @@ -700,7 +763,10 @@ if __name__ == "__main__": state = jax_utils.replicate(state) train_time = 0 - epochs = tqdm(range(num_epochs), desc=f"Epoch ... (1/{num_epochs})", position=0) + steps_per_epoch = len(tokenized_datasets["train"]) // train_batch_size + resume_epoch = resume_step // (steps_per_epoch * grad_accum_steps) + epochs = tqdm(range(num_epochs), desc=f"Epoch ... ({resume_epoch+1}/{num_epochs})", position=0) + logger.info(f"Skipping to epoch {resume_epoch} step {resume_step // grad_accum_steps}") for epoch in epochs: # ======================== Training ================================ train_start = time.time() @@ -708,54 +774,53 @@ if __name__ == "__main__": # Create sampling rng rng, input_rng = jax.random.split(rng) - steps_per_epoch = len(train_dataset) // train_batch_size # Generate an epoch by shuffling sampling indices from the train dataset - num_train_samples = len(train_dataset) + num_train_samples = len(tokenized_datasets["train"]) train_samples_idx = jax.random.permutation(input_rng, jnp.arange(num_train_samples)) - train_batch_idx = generate_batch_splits(train_samples_idx, train_batch_size) #// grad_accum_steps + train_batch_idx = generate_batch_splits(train_samples_idx, train_batch_size // grad_accum_steps) # Gather the indexes for creating the batch and do a training step - for step, batch_idx in enumerate(tqdm(train_batch_idx, desc="Training...", position=1,initial=resume_step)): #grad_accum - samples = [train_dataset[int(idx)] for idx in batch_idx] + for step, batch_idx in enumerate(tqdm(train_batch_idx, desc="Training...", position=1,initial=resume_step // grad_accum_steps)): + samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx] model_inputs = data_collator(samples, pad_to_multiple_of=16) - # Model forward model_inputs = shard(model_inputs.data) state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) train_metrics.append(train_metric) - cur_step = epoch * (num_train_samples // train_batch_size) + step + cur_step = epoch * (num_train_samples // train_batch_size * grad_accum_steps) + step if cur_step < resume_step: continue - if (cur_step % training_args.logging_steps) == 0 and cur_step > 0: # * grad_accum_steps + if cur_step % training_args.logging_steps * grad_accum_steps == 0 and cur_step > 0: # Save metrics train_metric = jax_utils.unreplicate(train_metric) train_time += time.time() - train_start if has_tensorboard and jax.process_index() == 0: write_train_metric(summary_writer, train_metrics, train_time, cur_step) + if has_wandb and jax.process_index() == 0 and ("wandb" in training_args.report_to): # TODO: add accumulation of metrics _metrics = {k if k=="learning_rate" else f"train_{k}":mb_item(v.mean()) for k, v in train_metric.items()} wandb.log({"training_step":cur_step, **_metrics}, commit=True) - + epochs.write( f"Step... ({cur_step} | Loss: {train_metric['loss']}, Learning Rate: {train_metric['learning_rate']})" ) train_metrics = [] - if cur_step % (training_args.eval_steps) == 0 and cur_step > 0: #* grad_accum_steps + if cur_step % training_args.eval_steps * grad_accum_steps == 0 and cur_step > 0: # ======================== Evaluating ============================== - num_eval_samples = len(eval_dataset) + num_eval_samples = len(tokenized_datasets["validation"]) eval_samples_idx = jnp.arange(num_eval_samples) eval_batch_idx = generate_batch_splits(eval_samples_idx, eval_batch_size) eval_metrics = [] for i, batch_idx in enumerate(tqdm(eval_batch_idx, desc="Evaluating ...", position=2)): - samples = [eval_dataset[int(idx)] for idx in batch_idx] + samples = [tokenized_datasets["validation"][int(idx)] for idx in batch_idx] model_inputs = data_collator(samples, pad_to_multiple_of=16) # Model forward @@ -775,30 +840,17 @@ if __name__ == "__main__": # Save metrics if has_tensorboard and jax.process_index() == 0: write_eval_metric(summary_writer, eval_metrics, cur_step) - if has_wandb and jax.process_index() == 0 and ("wandb" in training_args.report_to): _metrics = {f"eval_{k}":mb_item(v) for k, v in eval_metrics.items()} wandb.log({"eval_step":cur_step, **_metrics}) - if (cur_step % training_args.save_steps == 0 ) and cur_step > 0: # + if cur_step % training_args.save_steps == 0 * grad_accum_steps and cur_step > 0: # save checkpoint after each epoch and push checkpoint to the hub if jax.process_index() == 0: - params = jax.device_get(jax.tree_map(lambda x: x[0], state.params)) - model.save_pretrained( - training_args.output_dir, - params=params, - push_to_hub=training_args.push_to_hub, - commit_message=f"Saving weights and logs of step {cur_step}", - ) - save_checkpoint(training_args.output_dir, jax_utils.unreplicate(state), cur_step, keep=training_args.save_total_limit, overwrite=True) + save_model_checkpoint(model, training_args.output_dir, state, with_opt=model_args.save_optimizer, + push_to_hub=training_args.push_to_hub) if training_args.save_total_limit is not None: rotate_checkpoints(training_args.output_dir, training_args.save_total_limit) - + if jax.process_index() == 0: - params = jax.device_get(jax.tree_map(lambda x: x[0], state.params)) - model.save_pretrained( - training_args.output_dir, - params=params, - push_to_hub=training_args.push_to_hub, - commit_message=f"Saving weights and logs of step {cur_step}", - ) + save_model_checkpoint(model, training_args.output_dir, state, with_opt=model_args.save_optimizer, push_to_hub=training_args.push_to_hub) \ No newline at end of file diff --git a/run_mlm_flax_no_accum.py b/run_mlm_flax_no_accum.py new file mode 100644 index 0000000000000000000000000000000000000000..7f3ef2d339f666eb4ab4d937ccc796082905ac92 --- /dev/null +++ b/run_mlm_flax_no_accum.py @@ -0,0 +1,776 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2021 The HuggingFace Team All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fine-tuning the library models for masked language modeling (BERT, ALBERT, RoBERTa...) with whole word masking on a +text file or a dataset. + +Here is the full list of checkpoints on the hub that can be fine-tuned by this script: +https://huggingface.co/models?filter=masked-lm +""" +import logging +import os +import sys +import time +from dataclasses import dataclass, field + +# You can also adapt this script on your own masked language modeling task. Pointers for this are left as comments. +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +from datasets import load_dataset, DatasetDict +from tqdm import tqdm + +import flax +import jax +import jax.numpy as jnp +import optax +from flax import jax_utils, traverse_util +from flax.training import train_state +from flax.training.common_utils import get_metrics, onehot, shard +from transformers import ( + CONFIG_MAPPING, + FLAX_MODEL_FOR_MASKED_LM_MAPPING, + AutoConfig, + AutoTokenizer, + FlaxAutoModelForMaskedLM, + HfArgumentParser, + PreTrainedTokenizerBase, + TensorType, + TrainingArguments, + is_tensorboard_available, + set_seed, +) +import json +from flax.training import checkpoints +from flax.jax_utils import unreplicate +from flax.training.checkpoints import save_checkpoint, restore_checkpoint +from importlib.util import find_spec + + +MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys()) +MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. + """ + + model_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": "The model checkpoint for weights initialization." + "Don't set if you want to train a model from scratch." + }, + ) + model_type: Optional[str] = field( + default=None, + metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)}, + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + cache_dir: Optional[str] = field( + default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"} + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + dtype: Optional[str] = field( + default="float32", + metadata={ + "help": "Floating-point format in which the model weights should be initialized and trained. Choose one of `[float32, float16, bfloat16]`." + }, + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + dataset_name: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + dataset_config_name: Optional[str] = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + train_ref_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input train ref data file for whole word masking in Chinese."}, + ) + validation_ref_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input validation ref data file for whole word masking in Chinese."}, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + ) + + + + validation_split_percentage: Optional[int] = field( + default=5, + metadata={ + "help": "The percentage of the train set used as validation set in case there's no validation split" + }, + ) + max_seq_length: Optional[int] = field( + default=None, + metadata={ + "help": "The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated. Default to the max input length of the model." + }, + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + mlm_probability: float = field( + default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"} + ) + pad_to_max_length: bool = field( + default=False, + metadata={ + "help": "Whether to pad all samples to `max_seq_length`. " + "If False, will pad the samples dynamically when batching to the maximum length in the batch." + }, + ) + line_by_line: bool = field( + default=False, + metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."}, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + }, + ) + + + + + + +@flax.struct.dataclass +class FlaxDataCollatorForLanguageModeling: + """ + Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they + are not all of the same length. + + Args: + tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`): + The tokenizer used for encoding the data. + mlm_probability (:obj:`float`, `optional`, defaults to 0.15): + The probability with which to (randomly) mask tokens in the input. + + .. note:: + + For best performance, this data collator should be used with a dataset having items that are dictionaries or + BatchEncoding, with the :obj:`"special_tokens_mask"` key, as returned by a + :class:`~transformers.PreTrainedTokenizer` or a :class:`~transformers.PreTrainedTokenizerFast` with the + argument :obj:`return_special_tokens_mask=True`. + """ + + tokenizer: PreTrainedTokenizerBase + mlm_probability: float = 0.15 + + def __post_init__(self): + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for masked language modeling. " + "You should pass `mlm=False` to train on causal language modeling instead." + ) + + def __call__(self, examples: List[Dict[str, np.ndarray]], pad_to_multiple_of: int) -> Dict[str, np.ndarray]: + # Handle dict or lists with proper padding and conversion to tensor. + batch = self.tokenizer.pad(examples, pad_to_multiple_of=pad_to_multiple_of, return_tensors=TensorType.NUMPY) + + # If special token mask has been preprocessed, pop it from the dict. + special_tokens_mask = batch.pop("special_tokens_mask", None) + + batch["input_ids"], batch["labels"] = self.mask_tokens( + batch["input_ids"], special_tokens_mask=special_tokens_mask + ) + return batch + + def mask_tokens( + self, inputs: np.ndarray, special_tokens_mask: Optional[np.ndarray] + ) -> Tuple[jnp.ndarray, jnp.ndarray]: + """ + Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. + """ + labels = inputs.copy() + # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`) + probability_matrix = np.full(labels.shape, self.mlm_probability) + special_tokens_mask = special_tokens_mask.astype("bool") + + probability_matrix[special_tokens_mask] = 0.0 + masked_indices = np.random.binomial(1, probability_matrix).astype("bool") + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = np.random.binomial(1, np.full(labels.shape, 0.8)).astype("bool") & masked_indices + inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token) + + # 10% of the time, we replace masked input tokens with random word + indices_random = np.random.binomial(1, np.full(labels.shape, 0.5)).astype("bool") + indices_random &= masked_indices & ~indices_replaced + + random_words = np.random.randint(self.tokenizer.vocab_size, size=labels.shape, dtype="i4") + inputs[indices_random] = random_words[indices_random] + + # The rest of the time (10% of the time) we keep the masked input tokens unchanged + return inputs, labels + + +def generate_batch_splits(samples_idx: jnp.ndarray, batch_size: int) -> jnp.ndarray: + num_samples = len(samples_idx) + samples_to_remove = num_samples % batch_size + + if samples_to_remove != 0: + samples_idx = samples_idx[:-samples_to_remove] + sections_split = num_samples // batch_size + batch_idx = np.split(samples_idx, sections_split) + return batch_idx + + +def write_train_metric(summary_writer, train_metrics, train_time, step): + summary_writer.scalar("train_time", train_time, step) + + train_metrics = get_metrics(train_metrics) + for key, vals in train_metrics.items(): + tag = f"train_{key}" + for i, val in enumerate(vals): + summary_writer.scalar(tag, val, step - len(vals) + i + 1) + + +def write_eval_metric(summary_writer, eval_metrics, step): + for metric_name, value in eval_metrics.items(): + summary_writer.scalar(f"eval_{metric_name}", value, step) + +def rotate_checkpoints(ckpt_dir:str, save_total_limit:int): + "Removes older checkpoints so that `save_total_limit` checkpoints are kept" + # TODO: what to remove is decided using step number only, we might want to improve that + ckpts = [str(x) for x in Path(ckpt_dir).glob("ckpt-*")] + # sort checkpoints by step + ckpts_sorted = sorted(ckpts, key=lambda x: int(x.split('-')[-1])) + ckpts_to_delete = ckpts_sorted[:-save_total_limit] + for ckpt in ckpts_to_delete: + logger.info(f"Deleting older checkpoint [{ckpt}] due to save_total_limit ({save_total_limit})") + shutil.rmtree(ckpt) + + +if __name__ == "__main__": + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + if ( + os.path.exists(training_args.output_dir) + and os.listdir(training_args.output_dir) + and training_args.do_train + and not training_args.overwrite_output_dir + ): + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty." + "Use --overwrite_output_dir to overcome." + ) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + level="NOTSET", + datefmt="[%X]", + ) + + # Log on each process the small summary: + logger = logging.getLogger(__name__) + + # Set the verbosity to info of the Transformers logger (on main process only): + logger.info(f"Training/evaluation parameters {training_args}") + + # Set seed before initializing model. + set_seed(training_args.seed) + + # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below) + # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/ + # (the dataset will be downloaded automatically from the datasets Hub). + # + # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called + # 'text' is found. You can easily tweak this behavior (see below). + # + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if data_args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir) + + if "validation" not in datasets.keys(): + datasets["validation"] = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + split=f"train[:{data_args.validation_split_percentage}%]", + cache_dir=model_args.cache_dir, + ) + datasets["train"] = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + split=f"train[{data_args.validation_split_percentage}%:]", + cache_dir=model_args.cache_dir, + ) + else: + import glob + import random + data_files = [] + def add_jsonlines_dir(path, filespec): + global data_files + data_files += glob.glob(f"{path}/{filespec}") + data_files = list(set(data_files)) + print(f"Number of files {len(data_files)} after adding {path} glob {filespec}") + add_jsonlines_dir(f"/data/c4_cleaned2", "*.gz") + add_jsonlines_dir(f"/data/nrc_uniq_cleaned_20210223", "*.gz") + add_jsonlines_dir(f"/data/nu_uniq_cleaned_20210225", "*.gz") + random.Random(42).shuffle(data_files) + total = len(data_files) + print(total) + perc = 0.05 + val_size = int(perc * total) + train_size = total - val_size + train = data_files[:train_size] + val = data_files[train_size:] + print(f"Got {len(train)} training files and {perc * 100} % {len(val)} validation files") + assert list(set(train) & set(val)) == [], "Train overlaps with test" + load_grouped = True + if not load_grouped: + datasets = load_dataset('json', data_files={'train': train, 'validation': val}) + + #from datasets import Dataset + + #dataset = Dataset.from_file("/home/dat/.cache/huggingface/datasets/json/default-9add402b38836560/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723/json-train.arrow") + #dataset = Dataset.from_file("/home/dat/.cache/huggingface/datasets/json/default-9add402b38836560/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723/json-validation.arrow") + + + def mb_item(x): + return x.item() if hasattr(x, "item") else x + + # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at + # https://huggingface.co/docs/datasets/loading_datasets.html. + + # Load pretrained model and tokenizer + + # Distributed training: + # The .from_pretrained methods guarantee that only one local process can concurrently + # download model & vocab. + if model_args.config_name: + config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir) + elif model_args.model_name_or_path: + config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir) + else: + config = CONFIG_MAPPING[model_args.model_type]() + logger.warning("You are instantiating a new config instance from scratch.") + + if model_args.tokenizer_name: + tokenizer = AutoTokenizer.from_pretrained( + model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer + ) + elif model_args.model_name_or_path: + tokenizer = AutoTokenizer.from_pretrained( + model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer + ) + else: + raise ValueError( + "You are instantiating a new tokenizer from scratch. This is not supported by this script." + "You can do it from another script, save it, and load it from here, using --tokenizer_name." + ) + + # Preprocessing the datasets. + # First we tokenize all the texts. + + if load_grouped: + logger.info("Loading tokenized and grouped dataset") + tokenized_datasets = DatasetDict.load_from_disk("/data/tokenized_data") + logger.info("Setting max validation examples to ") + print(f"Number of validation examples {data_args.max_eval_samples}") + tokenized_datasets["train"]= tokenized_datasets["train"].select(range(20000)) + if data_args.max_eval_samples is not None: + tokenized_datasets["validation"] = tokenized_datasets["validation"].select(range(data_args.max_eval_samples)) + else: + if training_args.do_train: + column_names = datasets["train"].column_names + else: + column_names = datasets["validation"].column_names + text_column_name = "text" if "text" in column_names else column_names[0] + + max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length) + + if data_args.line_by_line: + # When using line_by_line, we just tokenize each nonempty line. + padding = "max_length" if data_args.pad_to_max_length else False + + def tokenize_function(examples): + # Remove empty lines + examples = [line for line in examples if len(line) > 0 and not line.isspace()] + return tokenizer( + examples, + return_special_tokens_mask=True, + padding=padding, + truncation=True, + max_length=max_seq_length, + ) + + tokenized_datasets = datasets.map( + tokenize_function, + input_columns=[text_column_name], + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + ) + + else: + # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts. + # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more + # efficient when it receives the `special_tokens_mask`. + def tokenize_function(examples): + return tokenizer(examples[text_column_name], return_special_tokens_mask=True) + + tokenized_datasets = datasets.map( + tokenize_function, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + ) + + # Main data processing function that will concatenate all texts from our dataset and generate chunks of + # max_seq_length. + def group_texts(examples): + # Concatenate all texts. + concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()} + total_length = len(concatenated_examples[list(examples.keys())[0]]) + # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can + # customize this part to your needs. + if total_length >= max_seq_length: + total_length = (total_length // max_seq_length) * max_seq_length + # Split by chunks of max_len. + result = { + k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)] + for k, t in concatenated_examples.items() + } + return result + + # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a + # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value + # might be slower to preprocess. + # + # To speed up this part, we use multiprocessing. See the documentation of the map method for more information: + # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map + tokenized_datasets = tokenized_datasets.map( + group_texts, + batched=True, + num_proc=data_args.preprocessing_num_workers, + load_from_cache_file=not data_args.overwrite_cache, + ) + + #tokenized_datasets.save_to_disk("/data/tokenized_data") + #print ("tokenized_datasets saved to disk") + + + # Enable tensorboard only on the master node + has_tensorboard = is_tensorboard_available() + if has_tensorboard and jax.process_index() == 0: + try: + from flax.metrics.tensorboard import SummaryWriter + + summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir)) + except ImportError as ie: + has_tensorboard = False + logger.warning( + f"Unable to display metrics through TensorBoard because some package are not installed: {ie}" + ) + else: + logger.warning( + "Unable to display metrics through TensorBoard because the package is not installed: " + "Please run pip install tensorboard to enable." + ) + has_wandb = find_spec("wandb") is not None + if jax.process_index() == 0 and has_wandb and ("wandb" in training_args.report_to): + try: + import wandb + wandb.init( + entity="wandb", + project="hf-flax-pino-roberta", + sync_tensorboard=True + ) + wandb.config.update(training_args) + wandb.config.update(model_args) + wandb.config.update(data_args) + except ImportError as e: + print(e) + has_wandb = False + # Data collator + # This one will take care of randomly masking the tokens. + data_collator = FlaxDataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=data_args.mlm_probability) + + # Initialize our training + rng = jax.random.PRNGKey(training_args.seed) + dropout_rngs = jax.random.split(rng, jax.local_device_count()) + + if model_args.model_name_or_path: + model = FlaxAutoModelForMaskedLM.from_pretrained( + model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype) + ) + else: + model = FlaxAutoModelForMaskedLM.from_config( + config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype) + ) + + # Store some constant + num_epochs = int(training_args.num_train_epochs) + train_batch_size = int(training_args.per_device_train_batch_size) * jax.device_count() + eval_batch_size = int(training_args.per_device_eval_batch_size) * jax.device_count() + + num_train_steps = len(tokenized_datasets["train"]) // train_batch_size * num_epochs + + # Create learning rate schedule + warmup_fn = optax.linear_schedule( + init_value=0.0, end_value=training_args.learning_rate, transition_steps=training_args.warmup_steps + ) + decay_fn = optax.linear_schedule( + init_value=training_args.learning_rate, + end_value=0, + transition_steps=num_train_steps - training_args.warmup_steps, + ) + linear_decay_lr_schedule_fn = optax.join_schedules( + schedules=[warmup_fn, decay_fn], boundaries=[training_args.warmup_steps] + ) + + # We use Optax's "masking" functionality to not apply weight decay + # to bias and LayerNorm scale parameters. decay_mask_fn returns a + # mask boolean with the same structure as the parameters. + # The mask is True for parameters that should be decayed. + # Note that this mask is specifically adapted for FlaxBERT-like models. + # For other models, one should correct the layer norm parameter naming + # accordingly. + def decay_mask_fn(params): + flat_params = traverse_util.flatten_dict(params) + flat_mask = {path: (path[-1] != "bias" and path[-2:] != ("LayerNorm", "scale")) for path in flat_params} + return traverse_util.unflatten_dict(flat_mask) + + # create adam optimizer + if training_args.adafactor: + # We use the default parameters here to initialize adafactor, + # For more details about the parameters please check https://github.com/deepmind/optax/blob/ed02befef9bf81cbbf236be3d2b0e032e9ed4a40/optax/_src/alias.py#L74 + optimizer = optax.adafactor( + learning_rate=linear_decay_lr_schedule_fn, + ) + else: + optimizer = optax.adamw( + learning_rate=linear_decay_lr_schedule_fn, + b1=training_args.adam_beta1, + b2=training_args.adam_beta2, + eps=training_args.adam_epsilon, + weight_decay=training_args.weight_decay, + mask=decay_mask_fn, + ) + optimizer = optax.chain( + optax.clip_grad_by_global_norm(1.), + optimizer + ) + + # Setup train state + state = train_state.TrainState.create(apply_fn=model.__call__, params=model.params, tx=optimizer) + + if training_args.resume_from_checkpoint: + state = restore_checkpoint(training_args.resume_from_checkpoint, state) + resume_step = mb_item(state.step.item()) + else: + resume_step = 0 + + + # Define gradient update step fn + def train_step(state, batch, dropout_rng): + dropout_rng, new_dropout_rng = jax.random.split(dropout_rng) + + def loss_fn(params): + labels = batch.pop("labels") + + logits = state.apply_fn(**batch, params=params, dropout_rng=dropout_rng, train=True)[0] + + # compute loss, ignore padded input tokens + label_mask = jnp.where(labels > 0, 1.0, 0.0) + loss = optax.softmax_cross_entropy(logits, onehot(labels, logits.shape[-1])) * label_mask + + # take average + loss = loss.sum() / label_mask.sum() + + return loss + + grad_fn = jax.value_and_grad(loss_fn) + loss, grad = grad_fn(state.params) + grad = jax.lax.pmean(grad, "batch") + new_state = state.apply_gradients(grads=grad) + + metrics = jax.lax.pmean( + {"loss": loss, "learning_rate": linear_decay_lr_schedule_fn(state.step)}, axis_name="batch" + ) + + return new_state, metrics, new_dropout_rng + + # Create parallel version of the train step + p_train_step = jax.pmap(train_step, "batch", donate_argnums=(0,)) + + # Define eval fn + def eval_step(params, batch): + labels = batch.pop("labels") + + logits = model(**batch, params=params, train=False)[0] + + # compute loss, ignore padded input tokens + label_mask = jnp.where(labels > 0, 1.0, 0.0) + loss = optax.softmax_cross_entropy(logits, onehot(labels, logits.shape[-1])) * label_mask + + # compute accuracy + accuracy = jnp.equal(jnp.argmax(logits, axis=-1), labels) * label_mask + + # summarize metrics + metrics = {"loss": loss.sum(), "accuracy": accuracy.sum(), "normalizer": label_mask.sum()} + metrics = jax.lax.psum(metrics, axis_name="batch") + + return metrics + + p_eval_step = jax.pmap(eval_step, "batch", donate_argnums=(0,)) + + # Replicate the train state on each device + state = jax_utils.replicate(state) + + train_time = 0 + epochs = tqdm(range(num_epochs), desc=f"Epoch ... (1/{num_epochs})", position=0) + for epoch in epochs: + # ======================== Training ================================ + train_start = time.time() + train_metrics = [] + + # Create sampling rng + rng, input_rng = jax.random.split(rng) + + # Generate an epoch by shuffling sampling indices from the train dataset + num_train_samples = len(tokenized_datasets["train"]) + train_samples_idx = jax.random.permutation(input_rng, jnp.arange(num_train_samples)) + train_batch_idx = generate_batch_splits(train_samples_idx, train_batch_size) + + # Gather the indexes for creating the batch and do a training step + for step, batch_idx in enumerate(tqdm(train_batch_idx, desc="Training...", position=1,initial=resume_step)): + samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx] + model_inputs = data_collator(samples, pad_to_multiple_of=16) + + # Model forward + model_inputs = shard(model_inputs.data) + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + train_metrics.append(train_metric) + + cur_step = epoch * (num_train_samples // train_batch_size) + step + if cur_step < resume_step: + continue + + if cur_step % training_args.logging_steps == 0 and cur_step > 0: + # Save metrics + train_metric = jax_utils.unreplicate(train_metric) + train_time += time.time() - train_start + if has_tensorboard and jax.process_index() == 0: + write_train_metric(summary_writer, train_metrics, train_time, cur_step) + + if has_wandb and jax.process_index() == 0 and ("wandb" in training_args.report_to): + # TODO: add accumulation of metrics + _metrics = {k if k=="learning_rate" else f"train_{k}":mb_item(v.mean()) for k, v in train_metric.items()} + wandb.log({"training_step":cur_step, **_metrics}, commit=True) + + epochs.write( + f"Step... ({cur_step} | Loss: {train_metric['loss']}, Learning Rate: {train_metric['learning_rate']})" + ) + + train_metrics = [] + + if cur_step % training_args.eval_steps == 0 and cur_step > 0: + # ======================== Evaluating ============================== + num_eval_samples = len(tokenized_datasets["validation"]) + eval_samples_idx = jnp.arange(num_eval_samples) + eval_batch_idx = generate_batch_splits(eval_samples_idx, eval_batch_size) + + eval_metrics = [] + for i, batch_idx in enumerate(tqdm(eval_batch_idx, desc="Evaluating ...", position=2)): + samples = [tokenized_datasets["validation"][int(idx)] for idx in batch_idx] + model_inputs = data_collator(samples, pad_to_multiple_of=16) + + # Model forward + model_inputs = shard(model_inputs.data) + metrics = p_eval_step(state.params, model_inputs) + eval_metrics.append(metrics) + + # normalize eval metrics + eval_metrics = get_metrics(eval_metrics) + eval_metrics = jax.tree_map(jnp.sum, eval_metrics) + eval_normalizer = eval_metrics.pop("normalizer") + eval_metrics = jax.tree_map(lambda x: x / eval_normalizer, eval_metrics) + + # Update progress bar + epochs.desc = f"Step... ({cur_step} | Loss: {eval_metrics['loss']}, Acc: {eval_metrics['accuracy']})" + + # Save metrics + if has_tensorboard and jax.process_index() == 0: + write_eval_metric(summary_writer, eval_metrics, cur_step) + if has_wandb and jax.process_index() == 0 and ("wandb" in training_args.report_to): + _metrics = {f"eval_{k}":mb_item(v) for k, v in eval_metrics.items()} + wandb.log({"eval_step":cur_step, **_metrics}) + + if cur_step % training_args.save_steps == 0 and cur_step > 0: + # save checkpoint after each epoch and push checkpoint to the hub + if jax.process_index() == 0: + save_checkpoint(training_args.output_dir, jax_utils.unreplicate(state), cur_step, keep=training_args.save_total_limit, overwrite=True) + params = jax.device_get(jax.tree_map(lambda x: x[0], state.params)) + model.save_pretrained( + training_args.output_dir, + params=params, + push_to_hub=training_args.push_to_hub, + commit_message=f"Saving weights and logs of step {cur_step}", + ) + if training_args.save_total_limit is not None: + rotate_checkpoints(training_args.output_dir, training_args.save_total_limit) + if jax.process_index() == 0: + params = jax.device_get(jax.tree_map(lambda x: x[0], state.params)) + model.save_pretrained( + training_args.output_dir, + params=params, + push_to_hub=training_args.push_to_hub, + commit_message=f"Saving weights and logs of step {cur_step}", + ) \ No newline at end of file diff --git a/save_tokenized_data.py b/save_tokenized_data.py new file mode 100644 index 0000000000000000000000000000000000000000..7163c6ed2d9c8f36782e4042084cda23063089ca --- /dev/null +++ b/save_tokenized_data.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2021 The HuggingFace Team All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fine-tuning the library models for masked language modeling (BERT, ALBERT, RoBERTa...) with whole word masking on a +text file or a dataset. + +Here is the full list of checkpoints on the hub that can be fine-tuned by this script: +https://huggingface.co/models?filter=masked-lm +""" +import logging +import os +import sys +import time +from dataclasses import dataclass, field + +# You can also adapt this script on your own masked language modeling task. Pointers for this are left as comments. +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +from datasets import load_dataset +from tqdm import tqdm + +import flax +import jax +import jax.numpy as jnp +import optax +from flax import jax_utils, traverse_util +from flax.training import train_state +from flax.training.common_utils import get_metrics, onehot, shard +from transformers import ( + CONFIG_MAPPING, + FLAX_MODEL_FOR_MASKED_LM_MAPPING, + AutoConfig, + AutoTokenizer, + FlaxAutoModelForMaskedLM, + HfArgumentParser, + PreTrainedTokenizerBase, + TensorType, + TrainingArguments, + is_tensorboard_available, + set_seed, +) +import json +from flax.training import checkpoints +from flax.jax_utils import unreplicate +from flax.training.checkpoints import save_checkpoint, restore_checkpoint +from importlib.util import find_spec + + +MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys()) +MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. + """ + + model_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": "The model checkpoint for weights initialization." + "Don't set if you want to train a model from scratch." + }, + ) + model_type: Optional[str] = field( + default=None, + metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)}, + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + cache_dir: Optional[str] = field( + default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"} + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + dtype: Optional[str] = field( + default="float32", + metadata={ + "help": "Floating-point format in which the model weights should be initialized and trained. Choose one of `[float32, float16, bfloat16]`." + }, + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + dataset_name: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + dataset_config_name: Optional[str] = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + train_ref_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input train ref data file for whole word masking in Chinese."}, + ) + validation_ref_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input validation ref data file for whole word masking in Chinese."}, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + ) + + + + validation_split_percentage: Optional[int] = field( + default=5, + metadata={ + "help": "The percentage of the train set used as validation set in case there's no validation split" + }, + ) + max_seq_length: Optional[int] = field( + default=None, + metadata={ + "help": "The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated. Default to the max input length of the model." + }, + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + mlm_probability: float = field( + default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"} + ) + pad_to_max_length: bool = field( + default=False, + metadata={ + "help": "Whether to pad all samples to `max_seq_length`. " + "If False, will pad the samples dynamically when batching to the maximum length in the batch." + }, + ) + line_by_line: bool = field( + default=False, + metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."}, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + }, + ) + + + + + + +@flax.struct.dataclass +class FlaxDataCollatorForLanguageModeling: + """ + Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they + are not all of the same length. + + Args: + tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`): + The tokenizer used for encoding the data. + mlm_probability (:obj:`float`, `optional`, defaults to 0.15): + The probability with which to (randomly) mask tokens in the input. + + .. note:: + + For best performance, this data collator should be used with a dataset having items that are dictionaries or + BatchEncoding, with the :obj:`"special_tokens_mask"` key, as returned by a + :class:`~transformers.PreTrainedTokenizer` or a :class:`~transformers.PreTrainedTokenizerFast` with the + argument :obj:`return_special_tokens_mask=True`. + """ + + tokenizer: PreTrainedTokenizerBase + mlm_probability: float = 0.15 + + def __post_init__(self): + if self.tokenizer.mask_token is None: + raise ValueError( + "This tokenizer does not have a mask token which is necessary for masked language modeling. " + "You should pass `mlm=False` to train on causal language modeling instead." + ) + + def __call__(self, examples: List[Dict[str, np.ndarray]], pad_to_multiple_of: int) -> Dict[str, np.ndarray]: + # Handle dict or lists with proper padding and conversion to tensor. + batch = self.tokenizer.pad(examples, pad_to_multiple_of=pad_to_multiple_of, return_tensors=TensorType.NUMPY) + + # If special token mask has been preprocessed, pop it from the dict. + special_tokens_mask = batch.pop("special_tokens_mask", None) + + batch["input_ids"], batch["labels"] = self.mask_tokens( + batch["input_ids"], special_tokens_mask=special_tokens_mask + ) + return batch + + def mask_tokens( + self, inputs: np.ndarray, special_tokens_mask: Optional[np.ndarray] + ) -> Tuple[jnp.ndarray, jnp.ndarray]: + """ + Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. + """ + labels = inputs.copy() + # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`) + probability_matrix = np.full(labels.shape, self.mlm_probability) + special_tokens_mask = special_tokens_mask.astype("bool") + + probability_matrix[special_tokens_mask] = 0.0 + masked_indices = np.random.binomial(1, probability_matrix).astype("bool") + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = np.random.binomial(1, np.full(labels.shape, 0.8)).astype("bool") & masked_indices + inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token) + + # 10% of the time, we replace masked input tokens with random word + indices_random = np.random.binomial(1, np.full(labels.shape, 0.5)).astype("bool") + indices_random &= masked_indices & ~indices_replaced + + random_words = np.random.randint(self.tokenizer.vocab_size, size=labels.shape, dtype="i4") + inputs[indices_random] = random_words[indices_random] + + # The rest of the time (10% of the time) we keep the masked input tokens unchanged + return inputs, labels + + +def generate_batch_splits(samples_idx: jnp.ndarray, batch_size: int) -> jnp.ndarray: + num_samples = len(samples_idx) + samples_to_remove = num_samples % batch_size + + if samples_to_remove != 0: + samples_idx = samples_idx[:-samples_to_remove] + sections_split = num_samples // batch_size + batch_idx = np.split(samples_idx, sections_split) + return batch_idx + + +def write_train_metric(summary_writer, train_metrics, train_time, step): + summary_writer.scalar("train_time", train_time, step) + + train_metrics = get_metrics(train_metrics) + for key, vals in train_metrics.items(): + tag = f"train_{key}" + for i, val in enumerate(vals): + summary_writer.scalar(tag, val, step - len(vals) + i + 1) + + +def write_eval_metric(summary_writer, eval_metrics, step): + for metric_name, value in eval_metrics.items(): + summary_writer.scalar(f"eval_{metric_name}", value, step) + + +if __name__ == "__main__": + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + if ( + os.path.exists(training_args.output_dir) + and os.listdir(training_args.output_dir) + and training_args.do_train + and not training_args.overwrite_output_dir + ): + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty." + "Use --overwrite_output_dir to overcome." + ) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + level="NOTSET", + datefmt="[%X]", + ) + + # Log on each process the small summary: + logger = logging.getLogger(__name__) + + # Set the verbosity to info of the Transformers logger (on main process only): + logger.info(f"Training/evaluation parameters {training_args}") + + # Set seed before initializing model. + set_seed(training_args.seed) + + # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below) + # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/ + # (the dataset will be downloaded automatically from the datasets Hub). + # + # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called + # 'text' is found. You can easily tweak this behavior (see below). + # + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if data_args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir) + + if "validation" not in datasets.keys(): + datasets["validation"] = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + split=f"train[:{data_args.validation_split_percentage}%]", + cache_dir=model_args.cache_dir, + ) + datasets["train"] = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + split=f"train[{data_args.validation_split_percentage}%:]", + cache_dir=model_args.cache_dir, + ) + else: + import glob + import random + data_files = [] + def add_jsonlines_dir(path, filespec): + global data_files + data_files += glob.glob(f"{path}/{filespec}") + data_files = list(set(data_files)) + print(f"Number of files {len(data_files)} after adding {path} glob {filespec}") + #add_jsonlines_dir(f"/data/c4_cleaned2", "*.gz") + #add_jsonlines_dir(f"/data/nrc_uniq_cleaned_20210223", "*.gz") + add_jsonlines_dir(f"/data/nu_uniq_cleaned_20210225", "*.gz") + random.Random(42).shuffle(data_files) + total = len(data_files) + print(total) + perc = 0.05 + val_size = int(perc * total) + train_size = total - val_size + train = data_files[5:8] + val = data_files[1:3] + print(f"Got {len(train)} training files and {perc * 100} % {len(val)} validation files") + assert list(set(train) & set(val)) == [], "Train overlaps with test" + datasets = load_dataset('json', data_files={'train': train, 'validation': val},cache_dir="/home/dat/.cache/huggingface/datasets/json/default-9add402b38836560/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723") + + #from datasets import Dataset + + #dataset = Dataset.from_file("/home/dat/.cache/huggingface/datasets/json/default-9add402b38836560/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723/json-train.arrow") + #dataset = Dataset.from_file("/home/dat/.cache/huggingface/datasets/json/default-9add402b38836560/0.0.0/f92a4de297ac644ad9781979b79064b0e222b3af766f8ea3bee32390dca23723/json-validation.arrow") + + + def mb_item(x): + return x.item() if hasattr(x, "item") else x + + # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at + # https://huggingface.co/docs/datasets/loading_datasets.html. + + # Load pretrained model and tokenizer + + # Distributed training: + # The .from_pretrained methods guarantee that only one local process can concurrently + # download model & vocab. + if model_args.config_name: + config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir) + elif model_args.model_name_or_path: + config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir) + else: + config = CONFIG_MAPPING[model_args.model_type]() + logger.warning("You are instantiating a new config instance from scratch.") + + if model_args.tokenizer_name: + tokenizer = AutoTokenizer.from_pretrained( + model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer + ) + elif model_args.model_name_or_path: + tokenizer = AutoTokenizer.from_pretrained( + model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer + ) + else: + raise ValueError( + "You are instantiating a new tokenizer from scratch. This is not supported by this script." + "You can do it from another script, save it, and load it from here, using --tokenizer_name." + ) + + # Preprocessing the datasets. + # First we tokenize all the texts. + if training_args.do_train: + column_names = datasets["train"].column_names + else: + column_names = datasets["validation"].column_names + text_column_name = "text" if "text" in column_names else column_names[0] + + max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length) + + if data_args.line_by_line: + # When using line_by_line, we just tokenize each nonempty line. + padding = "max_length" if data_args.pad_to_max_length else False + + def tokenize_function(examples): + # Remove empty lines + examples = [line for line in examples if len(line) > 0 and not line.isspace()] + return tokenizer( + examples, + return_special_tokens_mask=True, + padding=padding, + truncation=True, + max_length=max_seq_length, + ) + + tokenized_datasets = datasets.map( + tokenize_function, + input_columns=[text_column_name], + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + ) + tokenized_datasets.save_to_disk("/data/tokenized_data") + print ("save data") + else: + # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts. + # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more + # efficient when it receives the `special_tokens_mask`. + def tokenize_function(examples): + return tokenizer(examples[text_column_name], return_special_tokens_mask=True) + + tokenized_datasets = datasets.map( + tokenize_function, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + ) + + # Main data processing function that will concatenate all texts from our dataset and generate chunks of + # max_seq_length. + def group_texts(examples): + # Concatenate all texts. + concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()} + total_length = len(concatenated_examples[list(examples.keys())[0]]) + # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can + # customize this part to your needs. + if total_length >= max_seq_length: + total_length = (total_length // max_seq_length) * max_seq_length + # Split by chunks of max_len. + result = { + k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)] + for k, t in concatenated_examples.items() + } + return result + + # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a + # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value + # might be slower to preprocess. + # + # To speed up this part, we use multiprocessing. See the documentation of the map method for more information: + # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map + tokenized_datasets = tokenized_datasets.map( + group_texts, + batched=True, + num_proc=data_args.preprocessing_num_workers, + load_from_cache_file=not data_args.overwrite_cache, + ) + + tokenized_datasets.save_to_disk("/data/tokenized_data") + print ("save data") \ No newline at end of file diff --git a/train_tokenizer.py b/train_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..00b06bbffc31724fcf796821d1ce5031b42beb2f --- /dev/null +++ b/train_tokenizer.py @@ -0,0 +1,43 @@ +import glob +import random +from tokenizers import ByteLevelBPETokenizer +from datasets import load_dataset + +data_files = [] +def add_jsonlines_dir(path, filespec): + global data_files + data_files += glob.glob(f"{path}/{filespec}") + data_files = list(set(data_files)) + print(f"Number of files {len(data_files)} after adding {path} glob {filespec}") +add_jsonlines_dir(f"/data/c4_cleaned2", "*.gz") +add_jsonlines_dir(f"/data/nrc_uniq_cleaned_20210223", "*.gz") +add_jsonlines_dir(f"/data/nu_uniq_cleaned_20210225", "*.gz") +random.Random(42).shuffle(data_files) +total = len(data_files) +print(total) +perc = 0.05 +val_size = int(perc * total) +train_size = total - val_size +train = data_files[:train_size] +val = data_files[train_size:] +print(f"Got {len(train)} training files and {perc * 100} % {len(val)} validation files") +assert list(set(train) & set(val)) == [], "Train overlaps with test" +datasets = load_dataset('json', data_files={'train': train, 'validation': val}) + + + +tokenizer = ByteLevelBPETokenizer() + +def batch_iterator(batch_size=1000): + for i in range(0, len(datasets), batch_size): + yield datasets["train"][i: i + batch_size]["text"] + +tokenizer.train_from_iterator(batch_iterator(), vocab_size=50358, min_frequency=2, special_tokens=[ + "", + "", + "", + "", + "", +]) + +tokenizer.save("tokenizer.json") \ No newline at end of file diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log index 2b3455adfdced3d5bfad519431b7e6901107b6db..11a2afd3ffbd86e068001cb9041aab10a41c8aea 120000 --- a/wandb/debug-internal.log +++ b/wandb/debug-internal.log @@ -1 +1 @@ -run-20210713_010630-14xhiyhf/logs/debug-internal.log \ No newline at end of file +run-20210714_210351-1msvb4w4/logs/debug-internal.log \ No newline at end of file diff --git a/wandb/debug.log b/wandb/debug.log index b34fbe7c8cfad4b9a7da87cf70216a6681f54b7d..c35b8951058d9950a92d8419960e3802e5422767 120000 --- a/wandb/debug.log +++ b/wandb/debug.log @@ -1 +1 @@ -run-20210713_010630-14xhiyhf/logs/debug.log \ No newline at end of file +run-20210714_210351-1msvb4w4/logs/debug.log \ No newline at end of file diff --git a/wandb/latest-run b/wandb/latest-run index 434d995608a84fcf2694fa89772356a018b794dc..a172d3a1000db660da42d95053ca3a0c0469229e 120000 --- a/wandb/latest-run +++ b/wandb/latest-run @@ -1 +1 @@ -run-20210713_010630-14xhiyhf \ No newline at end of file +run-20210714_210351-1msvb4w4 \ No newline at end of file diff --git a/wandb/run-20210713_010630-14xhiyhf/files/output.log b/wandb/run-20210713_010630-14xhiyhf/files/output.log index 053c21e505ead7674b9ac465cfb46dcde0b2283f..c33e9e44c4109ac021155bbc04c21a4cd212ca7b 100644 --- a/wandb/run-20210713_010630-14xhiyhf/files/output.log +++ b/wandb/run-20210713_010630-14xhiyhf/files/output.log @@ -16222,3 +16222,12 @@ Training...: 64%|████████████▊ | 59500/92767 [9 Training...: 65%|████████████▉ | 60000/92767 [9:35:07<5:11:39, 1.75it/s] git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)92767 [9:35:07<5:11:39, 1.75it/s] +[10:43:30] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo +[10:44:08] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (3/3), 1.0 GB | 43 MB/s, done. +[10:44:09] - INFO - absl - Saving checkpoint at step: 60000 +tcmalloc: large alloc 1363968000 bytes == 0x2ed6e2000 @ 0x7f170bb8c680 0x7f170bbacbdd 0x7f143fe0e20d 0x7f143fe1c340 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe1be87 0x7f143fe17bd3 0x7f143fe181fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56fb87 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x68cdc7 +[10:44:13] - INFO - absl - Saved checkpoint at checkpoint_60000 + + + + diff --git a/wandb/run-20210713_010630-14xhiyhf/logs/debug-internal.log b/wandb/run-20210713_010630-14xhiyhf/logs/debug-internal.log index af1829e9195b69049dabaaae921e84d07ff66653..dcd62cde41ffeb9404da5c56b68a8270138998e7 100644 --- a/wandb/run-20210713_010630-14xhiyhf/logs/debug-internal.log +++ b/wandb/run-20210713_010630-14xhiyhf/logs/debug-internal.log @@ -22396,3 +22396,27 @@ 2021-07-13 10:43:28,960 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/wandb-summary.json 2021-07-13 10:43:29,961 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log 2021-07-13 10:43:31,962 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 10:43:36,601 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 10:43:36,601 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 10:43:51,734 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 10:43:51,734 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 10:43:55,447 DEBUG SenderThread:332390 [sender.py:send():179] send: stats +2021-07-13 10:44:06,865 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 10:44:06,866 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 10:44:09,977 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 10:44:14,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 10:44:16,979 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 10:44:18,980 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 10:44:20,981 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 10:44:22,005 DEBUG HandlerThread:332390 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 10:44:22,005 DEBUG SenderThread:332390 [sender.py:send_request():193] send_request: stop_status +2021-07-13 10:44:22,982 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 10:44:23,482 WARNING MainThread:332390 [internal.py:wandb_internal():147] Internal process interrupt: 1 +2021-07-13 10:44:24,702 WARNING MainThread:332390 [internal.py:wandb_internal():147] Internal process interrupt: 2 +2021-07-13 10:44:24,703 ERROR MainThread:332390 [internal.py:wandb_internal():150] Internal process interrupted. +2021-07-13 10:44:24,982 INFO Thread-8 :332390 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/files/output.log +2021-07-13 10:44:25,021 INFO SenderThread:332390 [sender.py:finish():945] shutting down sender +2021-07-13 10:44:25,022 INFO SenderThread:332390 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 10:44:25,022 INFO WriterThread:332390 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_010630-14xhiyhf/run-14xhiyhf.wandb +2021-07-13 10:44:25,022 INFO HandlerThread:332390 [handler.py:finish():638] shutting down handler +2021-07-13 10:44:25,103 INFO MainThread:332390 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_010630-14xhiyhf/logs/debug.log b/wandb/run-20210713_010630-14xhiyhf/logs/debug.log index d159fc3838857560bb6905273c2f5b9171a19a5c..0c56613f35f9469650ae15346d61e644f9ed2da4 100644 --- a/wandb/run-20210713_010630-14xhiyhf/logs/debug.log +++ b/wandb/run-20210713_010630-14xhiyhf/logs/debug.log @@ -23,3 +23,5 @@ config: {} 2021-07-13 01:06:32,711 INFO MainThread:330819 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_01-05-41_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 92768, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} 2021-07-13 01:06:32,712 INFO MainThread:330819 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'} 2021-07-13 01:06:32,714 INFO MainThread:330819 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 10:44:23,634 INFO MainThread:330819 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255 +2021-07-13 10:44:23,634 INFO MainThread:330819 [wandb_run.py:_restore():1565] restore diff --git a/wandb/run-20210713_010630-14xhiyhf/run-14xhiyhf.wandb b/wandb/run-20210713_010630-14xhiyhf/run-14xhiyhf.wandb index c8c0943e496dd816b83be69274cf4d6156fd6632..9140cf26fbb2b0b129841115ae4bc40eab4c0a38 100644 Binary files a/wandb/run-20210713_010630-14xhiyhf/run-14xhiyhf.wandb and b/wandb/run-20210713_010630-14xhiyhf/run-14xhiyhf.wandb differ diff --git a/wandb/run-20210713_104745-1rl2j7or/files/config.yaml b/wandb/run-20210713_104745-1rl2j7or/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a4b33a04865a069f172d072b1f5f4c1c313c43c --- /dev/null +++ b/wandb/run-20210713_104745-1rl2j7or/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: false +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 2 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_10-47-16_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 50 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_104745-1rl2j7or/files/output.log b/wandb/run-20210713_104745-1rl2j7or/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..fcba0e050ba596f63ab0b30d2fdcc5722969470f --- /dev/null +++ b/wandb/run-20210713_104745-1rl2j7or/files/output.log @@ -0,0 +1,57 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3114: UserWarning: Explicitly requested dtype requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1647, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 637, in xla_pmap_impl + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Attempting to reserve 12.60G at the bottom of memory. That was not possible. There are 12.15G free, 0B reserved, and 12.13G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 712, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +RuntimeError: Resource exhausted: Attempting to reserve 12.60G at the bottom of memory. That was not possible. There are 12.15G free, 0B reserved, and 12.13G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). \ No newline at end of file diff --git a/wandb/run-20210713_110212-594z6oo0/files/requirements.txt b/wandb/run-20210713_110212-594z6oo0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_110212-594z6oo0/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_110212-594z6oo0/files/wandb-metadata.json b/wandb/run-20210713_110212-594z6oo0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bd47497f53f5c1a32fc3f7ee4be03697ce81e5fd --- /dev/null +++ b/wandb/run-20210713_110212-594z6oo0/files/wandb-metadata.json @@ -0,0 +1,44 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T11:02:14.868640", + "startedAt": "2021-07-13T11:02:12.800310", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=2" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_110212-594z6oo0/files/wandb-summary.json b/wandb/run-20210713_110212-594z6oo0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210713_110212-594z6oo0/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210713_110212-594z6oo0/logs/debug-internal.log b/wandb/run-20210713_110212-594z6oo0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e30f56ff069f206d2bce78236bed26fe607a8a37 --- /dev/null +++ b/wandb/run-20210713_110212-594z6oo0/logs/debug-internal.log @@ -0,0 +1,173 @@ +2021-07-13 11:02:13,518 INFO MainThread:345480 [internal.py:wandb_internal():88] W&B internal server running at pid: 345480, started at: 2021-07-13 11:02:13.518424 +2021-07-13 11:02:13,520 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 11:02:13,520 INFO WriterThread:345480 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/run-594z6oo0.wandb +2021-07-13 11:02:13,521 DEBUG SenderThread:345480 [sender.py:send():179] send: header +2021-07-13 11:02:13,522 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: check_version +2021-07-13 11:02:13,558 DEBUG SenderThread:345480 [sender.py:send():179] send: run +2021-07-13 11:02:13,741 INFO SenderThread:345480 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files +2021-07-13 11:02:13,742 INFO SenderThread:345480 [sender.py:_start_run_threads():716] run started: 594z6oo0 with start time 1626174132 +2021-07-13 11:02:13,742 DEBUG SenderThread:345480 [sender.py:send():179] send: summary +2021-07-13 11:02:13,742 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 11:02:13,743 INFO SenderThread:345480 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:02:14,743 INFO Thread-8 :345480 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/wandb-summary.json +2021-07-13 11:02:14,868 DEBUG HandlerThread:345480 [meta.py:__init__():39] meta init +2021-07-13 11:02:14,868 DEBUG HandlerThread:345480 [meta.py:__init__():53] meta init done +2021-07-13 11:02:14,868 DEBUG HandlerThread:345480 [meta.py:probe():210] probe +2021-07-13 11:02:14,869 DEBUG HandlerThread:345480 [meta.py:_setup_git():200] setup git +2021-07-13 11:02:14,899 DEBUG HandlerThread:345480 [meta.py:_setup_git():207] setup git done +2021-07-13 11:02:14,899 DEBUG HandlerThread:345480 [meta.py:_save_pip():57] save pip +2021-07-13 11:02:14,899 DEBUG HandlerThread:345480 [meta.py:_save_pip():71] save pip done +2021-07-13 11:02:14,899 DEBUG HandlerThread:345480 [meta.py:probe():252] probe done +2021-07-13 11:02:14,902 DEBUG SenderThread:345480 [sender.py:send():179] send: files +2021-07-13 11:02:14,902 INFO SenderThread:345480 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 11:02:14,910 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:02:14,910 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:02:15,037 DEBUG SenderThread:345480 [sender.py:send():179] send: config +2021-07-13 11:02:15,038 DEBUG SenderThread:345480 [sender.py:send():179] send: config +2021-07-13 11:02:15,038 DEBUG SenderThread:345480 [sender.py:send():179] send: config +2021-07-13 11:02:15,378 INFO Thread-11 :345480 [upload_job.py:push():137] Uploaded file /tmp/tmpy185wdfywandb/22iauf0r-wandb-metadata.json +2021-07-13 11:02:15,743 INFO Thread-8 :345480 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/wandb-metadata.json +2021-07-13 11:02:15,743 INFO Thread-8 :345480 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/requirements.txt +2021-07-13 11:02:15,743 INFO Thread-8 :345480 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/output.log +2021-07-13 11:02:29,748 INFO Thread-8 :345480 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/output.log +2021-07-13 11:02:30,039 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:02:30,040 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:02:31,749 INFO Thread-8 :345480 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/output.log +2021-07-13 11:02:42,950 DEBUG SenderThread:345480 [sender.py:send():179] send: stats +2021-07-13 11:02:44,755 INFO Thread-8 :345480 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/config.yaml +2021-07-13 11:02:45,176 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:02:45,176 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:03:00,308 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:03:00,308 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:03:13,026 DEBUG SenderThread:345480 [sender.py:send():179] send: stats +2021-07-13 11:03:15,445 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:03:15,445 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:03:30,574 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:03:30,575 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:03:43,084 DEBUG SenderThread:345480 [sender.py:send():179] send: stats +2021-07-13 11:03:45,707 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:03:45,708 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:04:00,845 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:04:00,845 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:04:01,785 INFO Thread-8 :345480 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/output.log +2021-07-13 11:04:13,156 DEBUG SenderThread:345480 [sender.py:send():179] send: stats +2021-07-13 11:04:15,994 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:04:15,994 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:04:31,136 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:04:31,136 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:04:43,236 DEBUG SenderThread:345480 [sender.py:send():179] send: stats +2021-07-13 11:04:46,269 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:04:46,270 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:05:01,401 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:05:01,401 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:05:13,316 DEBUG SenderThread:345480 [sender.py:send():179] send: stats +2021-07-13 11:05:16,535 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:05:16,536 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:05:27,829 INFO Thread-8 :345480 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/output.log +2021-07-13 11:05:28,324 DEBUG SenderThread:345480 [sender.py:send():179] send: telemetry +2021-07-13 11:05:28,325 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:28,325 DEBUG SenderThread:345480 [sender.py:send():179] send: exit +2021-07-13 11:05:28,325 INFO SenderThread:345480 [sender.py:send_exit():287] handling exit code: 1 +2021-07-13 11:05:28,325 INFO SenderThread:345480 [sender.py:send_exit():295] send defer +2021-07-13 11:05:28,325 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:28,326 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:28,326 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-13 11:05:28,326 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:28,326 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-13 11:05:28,326 INFO SenderThread:345480 [sender.py:transition_state():308] send defer: 1 +2021-07-13 11:05:28,327 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:28,327 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-13 11:05:28,355 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:28,355 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-13 11:05:28,355 INFO SenderThread:345480 [sender.py:transition_state():308] send defer: 2 +2021-07-13 11:05:28,355 DEBUG SenderThread:345480 [sender.py:send():179] send: stats +2021-07-13 11:05:28,355 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:28,355 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-13 11:05:28,355 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:28,355 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-13 11:05:28,355 INFO SenderThread:345480 [sender.py:transition_state():308] send defer: 3 +2021-07-13 11:05:28,356 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:28,356 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-13 11:05:28,356 DEBUG SenderThread:345480 [sender.py:send():179] send: summary +2021-07-13 11:05:28,356 INFO SenderThread:345480 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:05:28,356 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:28,356 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-13 11:05:28,356 INFO SenderThread:345480 [sender.py:transition_state():308] send defer: 4 +2021-07-13 11:05:28,357 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:28,357 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-13 11:05:28,357 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:28,357 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-13 11:05:28,428 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:28,533 INFO SenderThread:345480 [sender.py:transition_state():308] send defer: 5 +2021-07-13 11:05:28,533 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:28,534 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:28,534 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-13 11:05:28,534 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:28,534 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-13 11:05:28,535 INFO SenderThread:345480 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 11:05:28,636 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:28,829 INFO Thread-8 :345480 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/wandb-summary.json +2021-07-13 11:05:28,830 INFO SenderThread:345480 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/config.yaml +2021-07-13 11:05:28,830 INFO SenderThread:345480 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/output.log +2021-07-13 11:05:28,830 INFO SenderThread:345480 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files +2021-07-13 11:05:28,830 INFO SenderThread:345480 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/requirements.txt requirements.txt +2021-07-13 11:05:28,830 INFO SenderThread:345480 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/output.log output.log +2021-07-13 11:05:28,831 INFO SenderThread:345480 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/wandb-metadata.json wandb-metadata.json +2021-07-13 11:05:28,831 INFO SenderThread:345480 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/config.yaml config.yaml +2021-07-13 11:05:28,831 INFO SenderThread:345480 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/wandb-summary.json wandb-summary.json +2021-07-13 11:05:28,831 INFO SenderThread:345480 [sender.py:transition_state():308] send defer: 6 +2021-07-13 11:05:28,831 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:28,837 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:28,837 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-13 11:05:28,837 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:28,844 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-13 11:05:28,844 INFO SenderThread:345480 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 11:05:28,933 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:28,934 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,036 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:29,036 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,139 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:29,139 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,241 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:29,241 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,280 INFO Thread-15 :345480 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/wandb-summary.json +2021-07-13 11:05:29,323 INFO Thread-13 :345480 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/output.log +2021-07-13 11:05:29,343 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:29,343 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,367 INFO Thread-12 :345480 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/requirements.txt +2021-07-13 11:05:29,389 INFO Thread-14 :345480 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/files/config.yaml +2021-07-13 11:05:29,445 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:29,445 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,548 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:29,548 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,589 INFO Thread-7 :345480 [sender.py:transition_state():308] send defer: 7 +2021-07-13 11:05:29,590 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:29,590 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-13 11:05:29,590 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:29,590 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-13 11:05:29,650 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:29,884 INFO SenderThread:345480 [sender.py:transition_state():308] send defer: 8 +2021-07-13 11:05:29,884 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,885 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:29,885 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-13 11:05:29,885 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:29,885 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-13 11:05:29,885 INFO SenderThread:345480 [sender.py:transition_state():308] send defer: 9 +2021-07-13 11:05:29,886 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:05:29,886 INFO HandlerThread:345480 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-13 11:05:29,886 DEBUG SenderThread:345480 [sender.py:send():179] send: final +2021-07-13 11:05:29,886 DEBUG SenderThread:345480 [sender.py:send():179] send: footer +2021-07-13 11:05:29,886 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: defer +2021-07-13 11:05:29,886 INFO SenderThread:345480 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-13 11:05:29,987 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:05:29,987 DEBUG SenderThread:345480 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:05:29,987 INFO SenderThread:345480 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 11:05:29,989 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: get_summary +2021-07-13 11:05:29,990 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-13 11:05:29,991 DEBUG HandlerThread:345480 [handler.py:handle_request():124] handle_request: shutdown +2021-07-13 11:05:29,991 INFO HandlerThread:345480 [handler.py:finish():638] shutting down handler +2021-07-13 11:05:30,886 INFO WriterThread:345480 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/run-594z6oo0.wandb +2021-07-13 11:05:30,987 INFO SenderThread:345480 [sender.py:finish():945] shutting down sender +2021-07-13 11:05:30,988 INFO SenderThread:345480 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 11:05:30,988 INFO SenderThread:345480 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 11:05:30,990 INFO MainThread:345480 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_110212-594z6oo0/logs/debug.log b/wandb/run-20210713_110212-594z6oo0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4b195387621345f35b68986f7d742e89df58e435 --- /dev/null +++ b/wandb/run-20210713_110212-594z6oo0/logs/debug.log @@ -0,0 +1,127 @@ +2021-07-13 11:02:12,801 INFO MainThread:343920 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 11:02:12,802 INFO MainThread:343920 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 11:02:12,802 INFO MainThread:343920 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/logs/debug.log +2021-07-13 11:02:12,802 INFO MainThread:343920 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_110212-594z6oo0/logs/debug-internal.log +2021-07-13 11:02:12,802 INFO MainThread:343920 [wandb_init.py:init():370] calling init triggers +2021-07-13 11:02:12,802 INFO MainThread:343920 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 11:02:12,802 INFO MainThread:343920 [wandb_init.py:init():419] starting backend +2021-07-13 11:02:12,802 INFO MainThread:343920 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 11:02:12,860 INFO MainThread:343920 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 11:02:12,916 INFO MainThread:343920 [backend.py:ensure_launched():139] started backend process with pid: 345480 +2021-07-13 11:02:12,918 INFO MainThread:343920 [wandb_init.py:init():424] backend started and connected +2021-07-13 11:02:12,921 INFO MainThread:343920 [wandb_init.py:init():472] updated telemetry +2021-07-13 11:02:12,922 INFO MainThread:343920 [wandb_init.py:init():491] communicating current version +2021-07-13 11:02:13,556 INFO MainThread:343920 [wandb_init.py:init():496] got version response +2021-07-13 11:02:13,556 INFO MainThread:343920 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 11:02:13,741 INFO MainThread:343920 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 11:02:14,906 INFO MainThread:343920 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 11:02:14,907 INFO MainThread:343920 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 11:02:14,907 INFO MainThread:343920 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 11:02:14,909 INFO MainThread:343920 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 11:02:14,909 INFO MainThread:343920 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 11:02:14,917 INFO MainThread:343920 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_11-01-24_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 11:02:14,919 INFO MainThread:343920 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 11:02:14,920 INFO MainThread:343920 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 11:05:26,001 INFO MainThread:343920 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-13 11:05:26,003 INFO MainThread:343920 [wandb_run.py:_restore():1565] restore +2021-07-13 11:05:28,327 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 11:05:28,534 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 11:05:28,832 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 4 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 11229 +} + +2021-07-13 11:05:28,935 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 11231 +} + +2021-07-13 11:05:29,037 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:05:29,140 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:05:29,242 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:05:29,344 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:05:29,446 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:05:29,549 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:05:29,885 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:05:29,988 INFO MainThread:343920 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:05:31,287 INFO MainThread:343920 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210713_110212-594z6oo0/run-594z6oo0.wandb b/wandb/run-20210713_110212-594z6oo0/run-594z6oo0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..019619a00341bc6ec0ca85774f0d6e4057bf4788 Binary files /dev/null and b/wandb/run-20210713_110212-594z6oo0/run-594z6oo0.wandb differ diff --git a/wandb/run-20210713_111111-3cwslju0/files/config.yaml b/wandb/run-20210713_111111-3cwslju0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4544a18667411ebee64c3b4f98010130621c97f --- /dev/null +++ b/wandb/run-20210713_111111-3cwslju0/files/config.yaml @@ -0,0 +1,307 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: false +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 4 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_11-10-23_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_111111-3cwslju0/files/output.log b/wandb/run-20210713_111111-3cwslju0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3eafae4861659ada24e9b74f365d523e12ac457c --- /dev/null +++ b/wandb/run-20210713_111111-3cwslju0/files/output.log @@ -0,0 +1,39 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3114: UserWarning: Explicitly requested dtype requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1647, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 637, in xla_pmap_impl + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Attempting to reserve 12.60G at the bottom of memory. That was not possible. There are 12.15G free, 0B reserved, and 12.13G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 712, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +RuntimeError: Resource exhausted: Attempting to reserve 12.60G at the bottom of memory. That was not possible. There are 12.15G free, 0B reserved, and 12.13G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). \ No newline at end of file diff --git a/wandb/run-20210713_111111-3cwslju0/files/requirements.txt b/wandb/run-20210713_111111-3cwslju0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_111111-3cwslju0/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_111111-3cwslju0/files/wandb-metadata.json b/wandb/run-20210713_111111-3cwslju0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7a38a54302446b8cb0db651870be6ce8b911d424 --- /dev/null +++ b/wandb/run-20210713_111111-3cwslju0/files/wandb-metadata.json @@ -0,0 +1,44 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T11:11:14.000891", + "startedAt": "2021-07-13T11:11:11.728152", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=4" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_111111-3cwslju0/files/wandb-summary.json b/wandb/run-20210713_111111-3cwslju0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210713_111111-3cwslju0/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210713_111111-3cwslju0/logs/debug-internal.log b/wandb/run-20210713_111111-3cwslju0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5d5fd33a3d376794908ae73b756942f06cd674d3 --- /dev/null +++ b/wandb/run-20210713_111111-3cwslju0/logs/debug-internal.log @@ -0,0 +1,175 @@ +2021-07-13 11:11:12,514 INFO MainThread:348072 [internal.py:wandb_internal():88] W&B internal server running at pid: 348072, started at: 2021-07-13 11:11:12.514575 +2021-07-13 11:11:12,516 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 11:11:12,517 INFO WriterThread:348072 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/run-3cwslju0.wandb +2021-07-13 11:11:12,518 DEBUG SenderThread:348072 [sender.py:send():179] send: header +2021-07-13 11:11:12,518 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: check_version +2021-07-13 11:11:12,553 DEBUG SenderThread:348072 [sender.py:send():179] send: run +2021-07-13 11:11:12,741 INFO SenderThread:348072 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files +2021-07-13 11:11:12,741 INFO SenderThread:348072 [sender.py:_start_run_threads():716] run started: 3cwslju0 with start time 1626174671 +2021-07-13 11:11:12,741 DEBUG SenderThread:348072 [sender.py:send():179] send: summary +2021-07-13 11:11:12,742 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 11:11:12,743 INFO SenderThread:348072 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:11:13,745 INFO Thread-8 :348072 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/wandb-summary.json +2021-07-13 11:11:14,000 DEBUG HandlerThread:348072 [meta.py:__init__():39] meta init +2021-07-13 11:11:14,000 DEBUG HandlerThread:348072 [meta.py:__init__():53] meta init done +2021-07-13 11:11:14,000 DEBUG HandlerThread:348072 [meta.py:probe():210] probe +2021-07-13 11:11:14,001 DEBUG HandlerThread:348072 [meta.py:_setup_git():200] setup git +2021-07-13 11:11:14,031 DEBUG HandlerThread:348072 [meta.py:_setup_git():207] setup git done +2021-07-13 11:11:14,031 DEBUG HandlerThread:348072 [meta.py:_save_pip():57] save pip +2021-07-13 11:11:14,031 DEBUG HandlerThread:348072 [meta.py:_save_pip():71] save pip done +2021-07-13 11:11:14,031 DEBUG HandlerThread:348072 [meta.py:probe():252] probe done +2021-07-13 11:11:14,035 DEBUG SenderThread:348072 [sender.py:send():179] send: files +2021-07-13 11:11:14,035 INFO SenderThread:348072 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 11:11:14,043 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:11:14,043 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:11:14,170 DEBUG SenderThread:348072 [sender.py:send():179] send: config +2021-07-13 11:11:14,170 DEBUG SenderThread:348072 [sender.py:send():179] send: config +2021-07-13 11:11:14,170 DEBUG SenderThread:348072 [sender.py:send():179] send: config +2021-07-13 11:11:14,515 INFO Thread-11 :348072 [upload_job.py:push():137] Uploaded file /tmp/tmp64u985tqwandb/1u3tchat-wandb-metadata.json +2021-07-13 11:11:14,745 INFO Thread-8 :348072 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/wandb-metadata.json +2021-07-13 11:11:14,745 INFO Thread-8 :348072 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/output.log +2021-07-13 11:11:14,745 INFO Thread-8 :348072 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/requirements.txt +2021-07-13 11:11:28,753 INFO Thread-8 :348072 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/output.log +2021-07-13 11:11:29,172 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:11:29,172 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:11:30,754 INFO Thread-8 :348072 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/output.log +2021-07-13 11:11:42,078 DEBUG SenderThread:348072 [sender.py:send():179] send: stats +2021-07-13 11:11:43,760 INFO Thread-8 :348072 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/config.yaml +2021-07-13 11:11:44,303 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:11:44,304 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:11:59,435 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:11:59,436 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:12:12,149 DEBUG SenderThread:348072 [sender.py:send():179] send: stats +2021-07-13 11:12:14,568 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:12:14,569 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:12:29,703 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:12:29,704 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:12:42,213 DEBUG SenderThread:348072 [sender.py:send():179] send: stats +2021-07-13 11:12:44,838 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:12:44,839 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:12:59,973 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:12:59,974 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:13:06,802 INFO Thread-8 :348072 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/output.log +2021-07-13 11:13:12,281 DEBUG SenderThread:348072 [sender.py:send():179] send: stats +2021-07-13 11:13:15,129 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:13:15,129 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:13:30,277 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:13:30,277 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:13:42,360 DEBUG SenderThread:348072 [sender.py:send():179] send: stats +2021-07-13 11:13:45,408 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:13:45,409 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:14:00,541 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:14:00,555 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:14:12,440 DEBUG SenderThread:348072 [sender.py:send():179] send: stats +2021-07-13 11:14:15,687 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:14:15,687 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:14:26,833 INFO Thread-8 :348072 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/output.log +2021-07-13 11:14:27,502 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:27,503 DEBUG SenderThread:348072 [sender.py:send():179] send: telemetry +2021-07-13 11:14:27,503 DEBUG SenderThread:348072 [sender.py:send():179] send: exit +2021-07-13 11:14:27,503 INFO SenderThread:348072 [sender.py:send_exit():287] handling exit code: 1 +2021-07-13 11:14:27,503 INFO SenderThread:348072 [sender.py:send_exit():295] send defer +2021-07-13 11:14:27,504 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:27,504 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:27,504 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-13 11:14:27,504 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:27,505 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-13 11:14:27,505 INFO SenderThread:348072 [sender.py:transition_state():308] send defer: 1 +2021-07-13 11:14:27,505 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:27,505 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-13 11:14:27,574 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:27,574 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-13 11:14:27,574 INFO SenderThread:348072 [sender.py:transition_state():308] send defer: 2 +2021-07-13 11:14:27,575 DEBUG SenderThread:348072 [sender.py:send():179] send: stats +2021-07-13 11:14:27,575 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:27,575 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-13 11:14:27,576 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:27,576 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-13 11:14:27,576 INFO SenderThread:348072 [sender.py:transition_state():308] send defer: 3 +2021-07-13 11:14:27,576 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:27,576 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-13 11:14:27,576 DEBUG SenderThread:348072 [sender.py:send():179] send: summary +2021-07-13 11:14:27,606 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:27,704 INFO SenderThread:348072 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:14:27,705 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:27,705 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-13 11:14:27,705 INFO SenderThread:348072 [sender.py:transition_state():308] send defer: 4 +2021-07-13 11:14:27,705 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:27,706 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:27,706 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-13 11:14:27,706 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:27,706 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-13 11:14:27,807 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:27,834 INFO Thread-8 :348072 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/output.log +2021-07-13 11:14:27,834 INFO Thread-8 :348072 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/wandb-summary.json +2021-07-13 11:14:27,901 INFO SenderThread:348072 [sender.py:transition_state():308] send defer: 5 +2021-07-13 11:14:27,901 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:27,902 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:27,902 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-13 11:14:27,902 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:27,902 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-13 11:14:27,902 INFO SenderThread:348072 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 11:14:28,004 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:28,834 INFO Thread-8 :348072 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/config.yaml +2021-07-13 11:14:28,836 INFO SenderThread:348072 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files +2021-07-13 11:14:28,836 INFO SenderThread:348072 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/requirements.txt requirements.txt +2021-07-13 11:14:28,836 INFO SenderThread:348072 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/output.log output.log +2021-07-13 11:14:28,836 INFO SenderThread:348072 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/wandb-metadata.json wandb-metadata.json +2021-07-13 11:14:28,836 INFO SenderThread:348072 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/config.yaml config.yaml +2021-07-13 11:14:28,836 INFO SenderThread:348072 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/wandb-summary.json wandb-summary.json +2021-07-13 11:14:28,837 INFO SenderThread:348072 [sender.py:transition_state():308] send defer: 6 +2021-07-13 11:14:28,837 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:28,840 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:28,840 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-13 11:14:28,843 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:28,844 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-13 11:14:28,844 INFO SenderThread:348072 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 11:14:28,943 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:28,943 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:29,046 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:29,046 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:29,148 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:29,149 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:29,251 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:29,251 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:29,300 INFO Thread-14 :348072 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/config.yaml +2021-07-13 11:14:29,309 INFO Thread-12 :348072 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/requirements.txt +2021-07-13 11:14:29,332 INFO Thread-15 :348072 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/wandb-summary.json +2021-07-13 11:14:29,353 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:29,354 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:29,401 INFO Thread-13 :348072 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/files/output.log +2021-07-13 11:14:29,456 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:29,479 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:29,581 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:29,582 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:29,679 INFO Thread-7 :348072 [sender.py:transition_state():308] send defer: 7 +2021-07-13 11:14:29,680 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:29,680 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-13 11:14:29,680 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:29,680 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-13 11:14:29,684 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:29,949 INFO SenderThread:348072 [sender.py:transition_state():308] send defer: 8 +2021-07-13 11:14:29,949 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:29,950 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:29,950 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-13 11:14:29,950 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:29,950 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-13 11:14:29,950 INFO SenderThread:348072 [sender.py:transition_state():308] send defer: 9 +2021-07-13 11:14:29,951 DEBUG SenderThread:348072 [sender.py:send():179] send: final +2021-07-13 11:14:29,951 DEBUG SenderThread:348072 [sender.py:send():179] send: footer +2021-07-13 11:14:29,951 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:14:29,951 INFO HandlerThread:348072 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-13 11:14:29,951 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: defer +2021-07-13 11:14:29,951 INFO SenderThread:348072 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-13 11:14:30,051 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:14:30,052 DEBUG SenderThread:348072 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:14:30,052 INFO SenderThread:348072 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 11:14:30,054 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: get_summary +2021-07-13 11:14:30,055 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-13 11:14:30,055 DEBUG HandlerThread:348072 [handler.py:handle_request():124] handle_request: shutdown +2021-07-13 11:14:30,056 INFO HandlerThread:348072 [handler.py:finish():638] shutting down handler +2021-07-13 11:14:30,951 INFO WriterThread:348072 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/run-3cwslju0.wandb +2021-07-13 11:14:31,052 INFO SenderThread:348072 [sender.py:finish():945] shutting down sender +2021-07-13 11:14:31,052 INFO SenderThread:348072 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 11:14:31,052 INFO SenderThread:348072 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 11:14:31,054 INFO MainThread:348072 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_111111-3cwslju0/logs/debug.log b/wandb/run-20210713_111111-3cwslju0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b860f783637b752eefa8075d9c1d724d041b5a7e --- /dev/null +++ b/wandb/run-20210713_111111-3cwslju0/logs/debug.log @@ -0,0 +1,135 @@ +2021-07-13 11:11:11,770 INFO MainThread:346512 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 11:11:11,770 INFO MainThread:346512 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 11:11:11,770 INFO MainThread:346512 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/logs/debug.log +2021-07-13 11:11:11,770 INFO MainThread:346512 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_111111-3cwslju0/logs/debug-internal.log +2021-07-13 11:11:11,770 INFO MainThread:346512 [wandb_init.py:init():370] calling init triggers +2021-07-13 11:11:11,770 INFO MainThread:346512 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 11:11:11,770 INFO MainThread:346512 [wandb_init.py:init():419] starting backend +2021-07-13 11:11:11,771 INFO MainThread:346512 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 11:11:11,846 INFO MainThread:346512 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 11:11:11,894 INFO MainThread:346512 [backend.py:ensure_launched():139] started backend process with pid: 348072 +2021-07-13 11:11:11,896 INFO MainThread:346512 [wandb_init.py:init():424] backend started and connected +2021-07-13 11:11:11,899 INFO MainThread:346512 [wandb_init.py:init():472] updated telemetry +2021-07-13 11:11:11,900 INFO MainThread:346512 [wandb_init.py:init():491] communicating current version +2021-07-13 11:11:12,551 INFO MainThread:346512 [wandb_init.py:init():496] got version response +2021-07-13 11:11:12,551 INFO MainThread:346512 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 11:11:12,741 INFO MainThread:346512 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 11:11:14,038 INFO MainThread:346512 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 11:11:14,039 INFO MainThread:346512 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 11:11:14,039 INFO MainThread:346512 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 11:11:14,041 INFO MainThread:346512 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 11:11:14,041 INFO MainThread:346512 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 11:11:14,047 INFO MainThread:346512 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_11-10-23_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 11:11:14,049 INFO MainThread:346512 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 11:11:14,050 INFO MainThread:346512 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 11:14:25,148 INFO MainThread:346512 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-13 11:14:25,149 INFO MainThread:346512 [wandb_run.py:_restore():1565] restore +2021-07-13 11:14:27,505 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 11:14:27,706 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 11:14:27,902 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 11:14:28,841 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 3 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 6624 +} + +2021-07-13 11:14:28,944 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 11231 +} + +2021-07-13 11:14:29,047 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:14:29,150 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:14:29,252 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:14:29,354 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:14:29,480 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:14:29,583 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:14:29,950 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:14:30,053 INFO MainThread:346512 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11231 + total_bytes: 11231 +} + +2021-07-13 11:14:31,336 INFO MainThread:346512 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210713_111111-3cwslju0/run-3cwslju0.wandb b/wandb/run-20210713_111111-3cwslju0/run-3cwslju0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ddc0fcfd305aefdd2426bb71ef744dafa3ecca1f Binary files /dev/null and b/wandb/run-20210713_111111-3cwslju0/run-3cwslju0.wandb differ diff --git a/wandb/run-20210713_112038-qbdrt0c0/files/config.yaml b/wandb/run-20210713_112038-qbdrt0c0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bff6bd752899ed8c72d20165b0e0c729ebf24c8 --- /dev/null +++ b/wandb/run-20210713_112038-qbdrt0c0/files/config.yaml @@ -0,0 +1,307 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: false +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 8 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_11-19-43_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_112038-qbdrt0c0/files/output.log b/wandb/run-20210713_112038-qbdrt0c0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ffc46b0e24f184474214471f1d5ba912086bb8b9 --- /dev/null +++ b/wandb/run-20210713_112038-qbdrt0c0/files/output.log @@ -0,0 +1,39 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3114: UserWarning: Explicitly requested dtype requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1647, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 637, in xla_pmap_impl + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Attempting to reserve 12.60G at the bottom of memory. That was not possible. There are 12.15G free, 0B reserved, and 12.13G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 712, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +RuntimeError: Resource exhausted: Attempting to reserve 12.60G at the bottom of memory. That was not possible. There are 12.15G free, 0B reserved, and 12.13G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). \ No newline at end of file diff --git a/wandb/run-20210713_112038-qbdrt0c0/files/requirements.txt b/wandb/run-20210713_112038-qbdrt0c0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_112038-qbdrt0c0/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_112038-qbdrt0c0/files/wandb-metadata.json b/wandb/run-20210713_112038-qbdrt0c0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5505a8d0c10f646546533c1719ed8a0a8355c5ad --- /dev/null +++ b/wandb/run-20210713_112038-qbdrt0c0/files/wandb-metadata.json @@ -0,0 +1,44 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T11:20:40.845183", + "startedAt": "2021-07-13T11:20:38.780227", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=8" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_112038-qbdrt0c0/files/wandb-summary.json b/wandb/run-20210713_112038-qbdrt0c0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210713_112038-qbdrt0c0/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210713_112038-qbdrt0c0/logs/debug-internal.log b/wandb/run-20210713_112038-qbdrt0c0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c8b8bcd3b547d16be2772a235254bd7522cb018a --- /dev/null +++ b/wandb/run-20210713_112038-qbdrt0c0/logs/debug-internal.log @@ -0,0 +1,171 @@ +2021-07-13 11:20:39,477 INFO MainThread:350932 [internal.py:wandb_internal():88] W&B internal server running at pid: 350932, started at: 2021-07-13 11:20:39.476738 +2021-07-13 11:20:39,479 INFO WriterThread:350932 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/run-qbdrt0c0.wandb +2021-07-13 11:20:39,479 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 11:20:39,480 DEBUG SenderThread:350932 [sender.py:send():179] send: header +2021-07-13 11:20:39,480 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: check_version +2021-07-13 11:20:39,519 DEBUG SenderThread:350932 [sender.py:send():179] send: run +2021-07-13 11:20:39,698 INFO SenderThread:350932 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files +2021-07-13 11:20:39,698 INFO SenderThread:350932 [sender.py:_start_run_threads():716] run started: qbdrt0c0 with start time 1626175238 +2021-07-13 11:20:39,698 DEBUG SenderThread:350932 [sender.py:send():179] send: summary +2021-07-13 11:20:39,698 INFO SenderThread:350932 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:20:39,699 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 11:20:40,702 INFO Thread-8 :350932 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/wandb-summary.json +2021-07-13 11:20:40,844 DEBUG HandlerThread:350932 [meta.py:__init__():39] meta init +2021-07-13 11:20:40,845 DEBUG HandlerThread:350932 [meta.py:__init__():53] meta init done +2021-07-13 11:20:40,845 DEBUG HandlerThread:350932 [meta.py:probe():210] probe +2021-07-13 11:20:40,846 DEBUG HandlerThread:350932 [meta.py:_setup_git():200] setup git +2021-07-13 11:20:40,874 DEBUG HandlerThread:350932 [meta.py:_setup_git():207] setup git done +2021-07-13 11:20:40,874 DEBUG HandlerThread:350932 [meta.py:_save_pip():57] save pip +2021-07-13 11:20:40,875 DEBUG HandlerThread:350932 [meta.py:_save_pip():71] save pip done +2021-07-13 11:20:40,875 DEBUG HandlerThread:350932 [meta.py:probe():252] probe done +2021-07-13 11:20:40,878 DEBUG SenderThread:350932 [sender.py:send():179] send: files +2021-07-13 11:20:40,878 INFO SenderThread:350932 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 11:20:40,885 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:20:40,886 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:20:41,013 DEBUG SenderThread:350932 [sender.py:send():179] send: config +2021-07-13 11:20:41,014 DEBUG SenderThread:350932 [sender.py:send():179] send: config +2021-07-13 11:20:41,014 DEBUG SenderThread:350932 [sender.py:send():179] send: config +2021-07-13 11:20:41,368 INFO Thread-11 :350932 [upload_job.py:push():137] Uploaded file /tmp/tmp_o2vfehlwandb/wusrh9hx-wandb-metadata.json +2021-07-13 11:20:41,700 INFO Thread-8 :350932 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/output.log +2021-07-13 11:20:41,700 INFO Thread-8 :350932 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/wandb-metadata.json +2021-07-13 11:20:41,700 INFO Thread-8 :350932 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/requirements.txt +2021-07-13 11:20:55,705 INFO Thread-8 :350932 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/output.log +2021-07-13 11:20:56,015 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:20:56,016 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:20:57,706 INFO Thread-8 :350932 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/output.log +2021-07-13 11:21:08,929 DEBUG SenderThread:350932 [sender.py:send():179] send: stats +2021-07-13 11:21:10,712 INFO Thread-8 :350932 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/config.yaml +2021-07-13 11:21:11,150 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:21:11,150 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:21:26,282 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:21:26,282 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:21:39,007 DEBUG SenderThread:350932 [sender.py:send():179] send: stats +2021-07-13 11:21:41,413 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:21:41,414 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:21:56,566 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:21:56,566 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:22:09,085 DEBUG SenderThread:350932 [sender.py:send():179] send: stats +2021-07-13 11:22:11,700 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:22:11,700 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:22:26,842 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:22:26,842 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:22:27,737 INFO Thread-8 :350932 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/output.log +2021-07-13 11:22:39,163 DEBUG SenderThread:350932 [sender.py:send():179] send: stats +2021-07-13 11:22:41,997 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:22:41,997 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:22:57,140 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:22:57,140 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:23:09,248 DEBUG SenderThread:350932 [sender.py:send():179] send: stats +2021-07-13 11:23:12,271 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:23:12,272 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:23:27,405 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:23:27,405 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:23:39,327 DEBUG SenderThread:350932 [sender.py:send():179] send: stats +2021-07-13 11:23:42,540 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:23:42,540 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:23:47,766 INFO Thread-8 :350932 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/output.log +2021-07-13 11:23:49,172 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:49,172 DEBUG SenderThread:350932 [sender.py:send():179] send: telemetry +2021-07-13 11:23:49,172 DEBUG SenderThread:350932 [sender.py:send():179] send: exit +2021-07-13 11:23:49,172 INFO SenderThread:350932 [sender.py:send_exit():287] handling exit code: 1 +2021-07-13 11:23:49,173 INFO SenderThread:350932 [sender.py:send_exit():295] send defer +2021-07-13 11:23:49,173 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:49,173 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:49,174 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-13 11:23:49,174 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:49,174 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-13 11:23:49,174 INFO SenderThread:350932 [sender.py:transition_state():308] send defer: 1 +2021-07-13 11:23:49,175 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:49,175 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-13 11:23:49,253 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:49,254 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-13 11:23:49,254 INFO SenderThread:350932 [sender.py:transition_state():308] send defer: 2 +2021-07-13 11:23:49,254 DEBUG SenderThread:350932 [sender.py:send():179] send: stats +2021-07-13 11:23:49,254 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:49,254 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-13 11:23:49,255 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:49,255 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-13 11:23:49,255 INFO SenderThread:350932 [sender.py:transition_state():308] send defer: 3 +2021-07-13 11:23:49,255 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:49,255 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-13 11:23:49,255 DEBUG SenderThread:350932 [sender.py:send():179] send: summary +2021-07-13 11:23:49,256 INFO SenderThread:350932 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:23:49,256 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:49,256 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-13 11:23:49,256 INFO SenderThread:350932 [sender.py:transition_state():308] send defer: 4 +2021-07-13 11:23:49,256 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:49,256 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-13 11:23:49,256 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:49,256 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-13 11:23:49,275 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:49,445 INFO SenderThread:350932 [sender.py:transition_state():308] send defer: 5 +2021-07-13 11:23:49,445 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:49,445 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:49,445 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-13 11:23:49,446 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:49,446 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-13 11:23:49,446 INFO SenderThread:350932 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 11:23:49,547 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:49,766 INFO Thread-8 :350932 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/config.yaml +2021-07-13 11:23:49,767 INFO SenderThread:350932 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/output.log +2021-07-13 11:23:49,767 INFO SenderThread:350932 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/wandb-summary.json +2021-07-13 11:23:49,767 INFO SenderThread:350932 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files +2021-07-13 11:23:49,767 INFO SenderThread:350932 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/requirements.txt requirements.txt +2021-07-13 11:23:49,767 INFO SenderThread:350932 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/output.log output.log +2021-07-13 11:23:49,767 INFO SenderThread:350932 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/wandb-metadata.json wandb-metadata.json +2021-07-13 11:23:49,767 INFO SenderThread:350932 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/config.yaml config.yaml +2021-07-13 11:23:49,767 INFO SenderThread:350932 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/wandb-summary.json wandb-summary.json +2021-07-13 11:23:49,768 INFO SenderThread:350932 [sender.py:transition_state():308] send defer: 6 +2021-07-13 11:23:49,768 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:49,771 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:49,771 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-13 11:23:49,771 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:49,772 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-13 11:23:49,772 INFO SenderThread:350932 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 11:23:49,873 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:49,873 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:49,975 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:49,976 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:50,078 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:50,078 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:50,180 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:50,180 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:50,215 INFO Thread-15 :350932 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/wandb-summary.json +2021-07-13 11:23:50,216 INFO Thread-14 :350932 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/config.yaml +2021-07-13 11:23:50,221 INFO Thread-12 :350932 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/requirements.txt +2021-07-13 11:23:50,225 INFO Thread-13 :350932 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/files/output.log +2021-07-13 11:23:50,282 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:50,282 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:50,384 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:50,384 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:50,425 INFO Thread-7 :350932 [sender.py:transition_state():308] send defer: 7 +2021-07-13 11:23:50,426 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:50,426 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-13 11:23:50,426 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:50,426 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-13 11:23:50,486 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:50,554 INFO SenderThread:350932 [sender.py:transition_state():308] send defer: 8 +2021-07-13 11:23:50,554 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:50,555 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:50,555 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-13 11:23:50,555 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:50,555 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-13 11:23:50,555 INFO SenderThread:350932 [sender.py:transition_state():308] send defer: 9 +2021-07-13 11:23:50,556 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: defer +2021-07-13 11:23:50,556 INFO HandlerThread:350932 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-13 11:23:50,556 DEBUG SenderThread:350932 [sender.py:send():179] send: final +2021-07-13 11:23:50,556 DEBUG SenderThread:350932 [sender.py:send():179] send: footer +2021-07-13 11:23:50,556 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: defer +2021-07-13 11:23:50,556 INFO SenderThread:350932 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-13 11:23:50,656 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 11:23:50,656 DEBUG SenderThread:350932 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 11:23:50,657 INFO SenderThread:350932 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 11:23:50,658 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: get_summary +2021-07-13 11:23:50,659 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-13 11:23:50,659 DEBUG HandlerThread:350932 [handler.py:handle_request():124] handle_request: shutdown +2021-07-13 11:23:50,660 INFO HandlerThread:350932 [handler.py:finish():638] shutting down handler +2021-07-13 11:23:51,557 INFO WriterThread:350932 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/run-qbdrt0c0.wandb +2021-07-13 11:23:51,657 INFO SenderThread:350932 [sender.py:finish():945] shutting down sender +2021-07-13 11:23:51,657 INFO SenderThread:350932 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 11:23:51,657 INFO SenderThread:350932 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 11:23:51,659 INFO MainThread:350932 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_112038-qbdrt0c0/logs/debug.log b/wandb/run-20210713_112038-qbdrt0c0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9abbb8d281eeb3b599aeafdc77c7c4db8127bc59 --- /dev/null +++ b/wandb/run-20210713_112038-qbdrt0c0/logs/debug.log @@ -0,0 +1,119 @@ +2021-07-13 11:20:38,781 INFO MainThread:349243 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 11:20:38,781 INFO MainThread:349243 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 11:20:38,782 INFO MainThread:349243 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/logs/debug.log +2021-07-13 11:20:38,782 INFO MainThread:349243 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_112038-qbdrt0c0/logs/debug-internal.log +2021-07-13 11:20:38,782 INFO MainThread:349243 [wandb_init.py:init():370] calling init triggers +2021-07-13 11:20:38,782 INFO MainThread:349243 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 11:20:38,782 INFO MainThread:349243 [wandb_init.py:init():419] starting backend +2021-07-13 11:20:38,782 INFO MainThread:349243 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 11:20:38,835 INFO MainThread:349243 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 11:20:38,885 INFO MainThread:349243 [backend.py:ensure_launched():139] started backend process with pid: 350932 +2021-07-13 11:20:38,887 INFO MainThread:349243 [wandb_init.py:init():424] backend started and connected +2021-07-13 11:20:38,890 INFO MainThread:349243 [wandb_init.py:init():472] updated telemetry +2021-07-13 11:20:38,891 INFO MainThread:349243 [wandb_init.py:init():491] communicating current version +2021-07-13 11:20:39,518 INFO MainThread:349243 [wandb_init.py:init():496] got version response +2021-07-13 11:20:39,518 INFO MainThread:349243 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 11:20:39,698 INFO MainThread:349243 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 11:20:40,881 INFO MainThread:349243 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 11:20:40,882 INFO MainThread:349243 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 11:20:40,882 INFO MainThread:349243 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 11:20:40,884 INFO MainThread:349243 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 11:20:40,884 INFO MainThread:349243 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 11:20:40,890 INFO MainThread:349243 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_11-19-43_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 11:20:40,892 INFO MainThread:349243 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 11:20:40,893 INFO MainThread:349243 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 11:23:46,878 INFO MainThread:349243 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-13 11:23:46,879 INFO MainThread:349243 [wandb_run.py:_restore():1565] restore +2021-07-13 11:23:49,174 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 11:23:49,446 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 11:23:49,772 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 4 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 11217 +} + +2021-07-13 11:23:49,874 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 11219 +} + +2021-07-13 11:23:49,976 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11219 + total_bytes: 11219 +} + +2021-07-13 11:23:50,079 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11219 + total_bytes: 11219 +} + +2021-07-13 11:23:50,181 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11219 + total_bytes: 11219 +} + +2021-07-13 11:23:50,283 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11219 + total_bytes: 11219 +} + +2021-07-13 11:23:50,385 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11219 + total_bytes: 11219 +} + +2021-07-13 11:23:50,555 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11219 + total_bytes: 11219 +} + +2021-07-13 11:23:50,657 INFO MainThread:349243 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11219 + total_bytes: 11219 +} + +2021-07-13 11:23:52,001 INFO MainThread:349243 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210713_112038-qbdrt0c0/run-qbdrt0c0.wandb b/wandb/run-20210713_112038-qbdrt0c0/run-qbdrt0c0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d0e73d0d7f3c5d5c104633a94aac518744ac660e Binary files /dev/null and b/wandb/run-20210713_112038-qbdrt0c0/run-qbdrt0c0.wandb differ diff --git a/wandb/run-20210713_113435-1vxvoloi/files/config.yaml b/wandb/run-20210713_113435-1vxvoloi/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76ddee8737cdc60bf5dd7b05c6441139e1420b91 --- /dev/null +++ b/wandb/run-20210713_113435-1vxvoloi/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 4 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_11-27-24_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_113435-1vxvoloi/files/output.log b/wandb/run-20210713_113435-1vxvoloi/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8472bc7d573431f30a66163aa9ca20dd6b93dda5 --- /dev/null +++ b/wandb/run-20210713_113435-1vxvoloi/files/output.log @@ -0,0 +1,1139 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3114: UserWarning: Explicitly requested dtype requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) diff --git a/wandb/run-20210713_113435-1vxvoloi/files/requirements.txt b/wandb/run-20210713_113435-1vxvoloi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_113435-1vxvoloi/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_113435-1vxvoloi/files/wandb-metadata.json b/wandb/run-20210713_113435-1vxvoloi/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d19f60f54d96123f9a18794d2b00444d27d1b17e --- /dev/null +++ b/wandb/run-20210713_113435-1vxvoloi/files/wandb-metadata.json @@ -0,0 +1,45 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T11:34:37.732696", + "startedAt": "2021-07-13T11:34:35.614288", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=4", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json b/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7733d314f68af04e40daa75018682f5522013408 --- /dev/null +++ b/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json @@ -0,0 +1 @@ +{"training_step": 3000, "learning_rate": 4.968088251189329e-05, "train_loss": 9.284842491149902, "_runtime": 2307, "_timestamp": 1626178382, "_step": 5} \ No newline at end of file diff --git a/wandb/run-20210713_113435-1vxvoloi/logs/debug-internal.log b/wandb/run-20210713_113435-1vxvoloi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f1047ceab29d5bba7729fbdca5bf13c44139a2a8 --- /dev/null +++ b/wandb/run-20210713_113435-1vxvoloi/logs/debug-internal.log @@ -0,0 +1,1591 @@ +2021-07-13 11:34:36,371 INFO MainThread:353216 [internal.py:wandb_internal():88] W&B internal server running at pid: 353216, started at: 2021-07-13 11:34:36.371423 +2021-07-13 11:34:36,374 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 11:34:36,374 DEBUG SenderThread:353216 [sender.py:send():179] send: header +2021-07-13 11:34:36,374 INFO WriterThread:353216 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/run-1vxvoloi.wandb +2021-07-13 11:34:36,375 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: check_version +2021-07-13 11:34:36,412 DEBUG SenderThread:353216 [sender.py:send():179] send: run +2021-07-13 11:34:36,584 INFO SenderThread:353216 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files +2021-07-13 11:34:36,585 INFO SenderThread:353216 [sender.py:_start_run_threads():716] run started: 1vxvoloi with start time 1626176075 +2021-07-13 11:34:36,585 DEBUG SenderThread:353216 [sender.py:send():179] send: summary +2021-07-13 11:34:36,585 INFO SenderThread:353216 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:34:36,585 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 11:34:37,598 INFO Thread-8 :353216 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json +2021-07-13 11:34:37,732 DEBUG HandlerThread:353216 [meta.py:__init__():39] meta init +2021-07-13 11:34:37,732 DEBUG HandlerThread:353216 [meta.py:__init__():53] meta init done +2021-07-13 11:34:37,732 DEBUG HandlerThread:353216 [meta.py:probe():210] probe +2021-07-13 11:34:37,733 DEBUG HandlerThread:353216 [meta.py:_setup_git():200] setup git +2021-07-13 11:34:37,764 DEBUG HandlerThread:353216 [meta.py:_setup_git():207] setup git done +2021-07-13 11:34:37,764 DEBUG HandlerThread:353216 [meta.py:_save_pip():57] save pip +2021-07-13 11:34:37,765 DEBUG HandlerThread:353216 [meta.py:_save_pip():71] save pip done +2021-07-13 11:34:37,765 DEBUG HandlerThread:353216 [meta.py:probe():252] probe done +2021-07-13 11:34:37,769 DEBUG SenderThread:353216 [sender.py:send():179] send: files +2021-07-13 11:34:37,769 INFO SenderThread:353216 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 11:34:37,775 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:34:37,776 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:34:37,906 DEBUG SenderThread:353216 [sender.py:send():179] send: config +2021-07-13 11:34:37,906 DEBUG SenderThread:353216 [sender.py:send():179] send: config +2021-07-13 11:34:37,906 DEBUG SenderThread:353216 [sender.py:send():179] send: config +2021-07-13 11:34:38,222 INFO Thread-11 :353216 [upload_job.py:push():137] Uploaded file /tmp/tmpokklr1agwandb/32ipfyng-wandb-metadata.json +2021-07-13 11:34:38,597 INFO Thread-8 :353216 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/requirements.txt +2021-07-13 11:34:38,597 INFO Thread-8 :353216 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/wandb-metadata.json +2021-07-13 11:34:38,598 INFO Thread-8 :353216 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:34:52,602 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:34:52,909 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:34:52,909 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:34:54,603 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:35:05,813 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:35:07,608 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/config.yaml +2021-07-13 11:35:08,039 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:35:08,039 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:35:23,172 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:35:23,173 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:35:35,884 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:35:38,310 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:35:38,310 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:35:53,444 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:35:53,444 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:36:05,948 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:36:08,575 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:36:08,575 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:36:22,634 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:36:23,746 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:36:23,747 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:36:36,019 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:36:38,888 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:36:38,889 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:36:54,028 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:36:54,029 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:37:06,096 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:37:09,466 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:37:09,467 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:37:24,600 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:37:24,600 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:37:36,175 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:37:39,736 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:37:39,736 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:37:54,868 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:37:54,868 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:37:56,667 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:37:58,668 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:00,669 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:02,669 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:04,670 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:06,251 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:38:06,671 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:08,672 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:10,000 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:38:10,000 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:38:10,672 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:12,673 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:14,674 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:16,675 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:18,676 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:20,676 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:22,677 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:24,678 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:25,132 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:38:25,132 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:38:26,679 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:28,679 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:30,680 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:32,681 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:34,682 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:36,327 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:38:36,682 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:38,683 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:40,286 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:38:40,286 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:38:40,684 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:42,684 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:44,685 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:46,686 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:48,687 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:50,687 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:52,688 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:54,689 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:55,418 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:38:55,418 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:38:56,690 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:38:58,691 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:00,691 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:02,692 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:04,693 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:06,402 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:39:06,694 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:08,694 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:10,549 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:39:10,549 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:39:10,695 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:12,696 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:14,696 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:16,698 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:18,698 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:20,699 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:22,700 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:24,701 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:25,682 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:39:25,682 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:39:26,701 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:28,702 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:30,702 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:32,703 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:34,704 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:36,479 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:39:36,705 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:38,706 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:40,706 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:40,814 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:39:40,814 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:39:42,707 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:44,708 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:46,708 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:48,709 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:50,710 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:53,711 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:55,712 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:55,957 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:39:55,957 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:39:57,713 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:39:59,714 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:01,714 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:03,715 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:05,716 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:06,556 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:40:07,717 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:09,718 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:11,098 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:40:11,099 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:40:11,718 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:13,719 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:15,720 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:17,721 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:19,722 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:21,723 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:23,723 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:25,724 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:26,232 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:40:26,233 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:40:27,725 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:29,726 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:31,727 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:33,728 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:35,728 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:36,634 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:40:37,729 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:39,730 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:41,362 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:40:41,362 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:40:41,731 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:43,732 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:45,732 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:47,733 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:49,734 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:51,735 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:53,736 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:55,737 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:56,493 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:40:56,494 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:40:57,738 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:40:59,739 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:01,739 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:03,740 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:05,741 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:06,712 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:41:07,742 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:09,743 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:11,648 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:41:11,648 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:41:11,744 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:13,745 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:15,746 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:17,747 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:19,748 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:21,749 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:23,750 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:25,750 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:26,781 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:41:26,781 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:41:27,751 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:29,752 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:31,753 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:33,754 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:35,754 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:36,789 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:41:37,755 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:39,756 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:41,757 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:41,912 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:41:41,912 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:41:43,757 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:45,758 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:47,759 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:49,760 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:51,760 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:53,761 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:55,762 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:57,041 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:41:57,042 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:41:57,762 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:41:59,763 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:01,764 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:03,765 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:05,765 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:06,883 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:42:07,766 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:09,767 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:11,767 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:12,179 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:42:12,180 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:42:13,768 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:15,769 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:17,770 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:19,770 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:21,771 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:23,771 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:25,772 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:27,320 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:42:27,320 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:42:27,773 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:29,773 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:31,774 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:33,775 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:35,776 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:36,960 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:42:37,776 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:39,777 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:41,778 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:42,460 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:42:42,460 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:42:43,779 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:45,779 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:47,780 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:49,781 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:51,781 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:53,782 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:55,783 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:57,600 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:42:57,601 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:42:57,783 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:42:59,784 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:01,785 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:03,785 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:05,786 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:07,038 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:43:07,787 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:09,787 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:11,788 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:12,730 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:43:12,730 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:43:13,789 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:15,790 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:17,790 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:19,791 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:21,792 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:23,793 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:25,793 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:27,860 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:43:27,860 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:43:37,114 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:43:42,995 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:43:42,995 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:43:47,346 DEBUG SenderThread:353216 [sender.py:send():179] send: history +2021-07-13 11:43:47,346 DEBUG SenderThread:353216 [sender.py:send():179] send: summary +2021-07-13 11:43:47,347 INFO SenderThread:353216 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:43:47,801 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json +2021-07-13 11:43:47,801 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:49,802 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:50,802 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:52,803 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:54,803 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:56,804 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:43:58,127 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:43:58,127 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:43:58,805 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:00,805 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:02,806 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:04,807 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:06,808 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:07,193 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:44:08,808 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:10,809 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:12,810 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:13,273 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:44:13,273 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:44:14,811 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:16,811 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:18,812 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:20,813 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:22,813 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:24,814 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:26,815 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:28,406 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:44:28,406 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:44:28,816 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:30,816 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:32,817 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:34,818 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:36,819 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:37,272 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:44:38,820 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:40,820 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:42,821 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:43,539 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:44:43,539 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:44:44,822 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:46,823 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:48,824 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:50,824 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:52,825 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:54,826 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:56,827 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:44:58,671 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:44:58,671 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:44:58,827 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:00,828 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:02,829 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:04,830 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:06,831 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:07,349 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:45:08,832 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:10,833 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:12,833 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:13,803 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:45:13,803 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:45:14,834 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:16,835 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:18,836 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:20,837 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:22,838 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:24,838 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:26,839 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:28,840 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:28,938 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:45:28,938 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:45:30,840 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:32,841 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:34,842 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:36,843 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:37,427 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:45:38,844 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:40,844 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:42,845 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:44,070 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:45:44,071 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:45:44,846 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:46,847 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:48,848 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:50,848 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:52,849 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:54,850 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:56,851 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:58,852 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:45:59,201 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:45:59,201 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:46:00,852 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:02,853 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:04,854 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:06,855 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:07,505 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:46:08,855 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:10,856 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:12,857 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:14,334 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:46:14,334 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:46:14,858 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:16,859 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:18,860 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:20,861 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:22,862 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:24,862 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:26,863 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:28,864 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:29,468 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:46:29,468 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:46:30,865 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:32,865 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:34,866 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:36,867 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:37,583 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:46:38,868 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:40,869 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:42,870 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:44,601 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:46:44,602 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:46:44,870 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:46,871 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:48,872 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:50,873 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:52,874 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:54,874 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:56,875 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:58,876 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:46:59,732 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:46:59,733 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:47:00,877 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:02,878 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:04,879 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:06,880 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:07,661 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:47:08,881 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:10,881 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:13,882 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:14,866 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:47:14,866 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:47:15,883 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:17,884 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:19,885 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:21,886 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:23,887 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:25,887 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:27,888 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:29,889 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:30,009 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:47:30,009 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:47:31,889 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:33,890 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:35,891 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:37,741 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:47:37,892 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:39,892 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:41,893 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:43,894 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:45,150 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:47:45,150 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:47:45,895 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:47,895 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:49,896 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:51,897 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:53,897 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:55,898 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:57,899 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:47:59,899 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:00,283 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:48:00,284 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:48:01,900 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:03,901 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:05,902 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:07,817 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:48:07,902 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:09,903 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:11,904 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:13,904 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:15,414 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:48:15,414 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:48:15,905 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:17,906 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:19,907 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:21,907 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:23,908 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:25,908 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:27,909 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:29,910 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:30,544 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:48:30,545 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:48:31,910 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:33,911 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:35,912 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:37,895 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:48:37,913 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:39,914 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:41,914 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:43,915 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:45,687 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:48:45,687 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:48:45,916 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:47,916 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:49,917 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:51,918 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:53,918 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:55,919 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:57,920 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:48:59,920 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:00,825 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:49:00,826 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:49:01,921 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:03,922 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:05,923 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:07,924 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:07,971 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:49:09,924 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:11,925 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:13,926 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:15,926 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:15,957 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:49:15,957 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:49:31,088 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:49:31,089 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:49:38,049 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:49:38,378 DEBUG SenderThread:353216 [sender.py:send():179] send: history +2021-07-13 11:49:38,378 DEBUG SenderThread:353216 [sender.py:send():179] send: summary +2021-07-13 11:49:38,379 INFO SenderThread:353216 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:49:38,935 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json +2021-07-13 11:49:39,936 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:40,936 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:41,937 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:43,938 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:45,939 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:46,219 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:49:46,219 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:49:47,940 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:49,940 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:51,941 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:53,942 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:55,943 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:57,944 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:49:59,945 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:01,352 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:50:01,352 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:50:01,945 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:03,946 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:05,947 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:07,948 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:08,127 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:50:09,948 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:11,949 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:13,950 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:15,951 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:16,484 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:50:16,484 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:50:17,952 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:19,952 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:21,953 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:23,954 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:25,955 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:27,956 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:29,957 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:31,618 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:50:31,618 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:50:31,957 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:34,959 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:36,960 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:38,205 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:50:38,961 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:40,962 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:42,963 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:44,963 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:46,747 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:50:46,748 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:50:46,964 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:48,965 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:50,966 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:52,967 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:54,968 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:56,969 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:50:58,970 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:00,970 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:01,878 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:51:01,878 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:51:02,971 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:04,972 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:06,973 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:08,281 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:51:08,974 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:10,975 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:12,976 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:14,977 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:16,978 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:17,021 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:51:17,021 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:51:18,980 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:20,980 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:22,981 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:24,982 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:26,983 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:28,984 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:30,985 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:32,159 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:51:32,159 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:51:32,986 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:34,987 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:36,988 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:38,360 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:51:38,989 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:40,990 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:42,991 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:44,992 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:46,992 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:47,293 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:51:47,293 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:51:48,993 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:50,994 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:52,995 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:54,996 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:56,997 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:51:58,997 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:00,998 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:02,426 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:52:02,427 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:52:02,999 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:05,000 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:07,001 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:08,437 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:52:09,002 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:11,003 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:13,003 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:15,004 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:17,005 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:17,560 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:52:17,560 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:52:19,006 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:21,007 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:23,007 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:25,008 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:27,009 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:29,010 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:31,010 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:32,696 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:52:32,696 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:52:33,011 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:35,012 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:37,013 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:38,515 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:52:39,013 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:41,014 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:43,015 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:45,015 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:47,016 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:47,827 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:52:47,827 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:52:49,017 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:51,017 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:53,018 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:55,019 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:57,019 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:52:59,020 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:01,020 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:02,961 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:53:02,962 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:53:03,021 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:05,022 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:07,022 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:08,593 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:53:09,023 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:11,024 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:13,024 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:15,025 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:17,026 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:18,093 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:53:18,093 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:53:19,027 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:21,027 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:23,028 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:25,029 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:27,030 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:29,030 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:31,031 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:33,032 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:33,226 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:53:33,226 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:53:35,032 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:37,033 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:38,672 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:53:39,033 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:41,034 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:43,035 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:45,035 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:47,036 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:48,358 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:53:48,359 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:53:49,037 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:51,038 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:53,038 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:55,039 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:57,040 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:53:59,041 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:01,041 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:03,042 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:03,503 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:54:03,503 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:54:05,043 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:07,044 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:08,748 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:54:09,045 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:11,045 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:13,046 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:15,047 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:17,048 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:18,648 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:54:18,649 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:54:19,049 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:21,049 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:23,050 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:25,051 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:27,052 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:29,053 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:31,054 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:33,055 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:33,812 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:54:33,812 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:54:35,056 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:37,057 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:38,827 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:54:39,058 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:41,059 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:43,059 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:45,060 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:47,061 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:48,944 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:54:48,945 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:54:49,062 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:51,063 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:53,064 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:55,065 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:57,065 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:54:59,066 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:01,067 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:03,068 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:04,077 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:55:04,077 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:55:05,069 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:07,070 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:08,904 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:55:19,209 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:55:19,209 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:55:29,385 DEBUG SenderThread:353216 [sender.py:send():179] send: history +2021-07-13 11:55:29,385 DEBUG SenderThread:353216 [sender.py:send():179] send: summary +2021-07-13 11:55:29,386 INFO SenderThread:353216 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 11:55:30,079 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json +2021-07-13 11:55:31,079 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:33,080 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:34,343 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:55:34,344 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:55:35,081 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:37,082 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:38,981 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:55:39,083 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:41,083 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:43,084 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:45,085 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:47,086 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:49,087 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:49,528 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:55:49,528 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:55:51,088 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:53,088 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:55,089 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:57,090 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:55:59,091 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:01,092 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:03,093 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:04,677 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:56:04,678 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:56:05,094 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:07,095 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:09,060 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:56:09,095 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:12,097 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:14,098 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:16,098 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:18,099 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:19,807 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:56:19,807 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:56:20,100 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:22,101 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:24,102 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:26,103 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:28,104 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:30,104 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:32,106 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:34,106 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:34,941 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:56:34,941 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:56:36,107 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:38,108 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:39,136 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:56:40,109 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:42,110 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:44,111 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:46,112 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:48,113 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:50,074 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:56:50,074 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:56:50,114 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:52,114 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:54,115 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:56,116 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:56:58,117 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:00,118 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:02,119 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:04,120 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:05,206 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:57:05,206 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:57:06,121 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:08,122 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:09,215 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:57:10,123 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:12,124 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:14,125 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:16,126 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:18,127 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:20,127 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:20,338 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:57:20,338 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:57:22,128 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:24,129 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:26,130 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:28,131 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:30,131 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:32,132 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:34,133 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:35,468 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:57:35,469 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:57:36,134 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:38,135 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:39,293 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:57:40,136 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:42,137 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:44,138 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:46,139 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:48,140 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:50,141 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:50,600 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:57:50,600 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:57:52,142 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:54,143 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:56,144 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:57:58,145 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:00,146 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:02,147 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:04,148 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:05,736 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:58:05,736 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:58:06,149 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:08,149 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:09,364 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:58:10,151 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:12,151 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:14,152 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:16,154 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:18,154 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:20,155 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:20,876 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:58:20,876 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:58:22,156 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:24,157 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:26,158 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:28,159 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:30,159 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:32,160 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:34,161 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:36,010 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:58:36,011 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:58:36,162 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:38,162 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:39,435 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:58:40,163 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:42,164 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:44,165 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:46,165 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:48,166 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:50,167 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:51,146 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:58:51,147 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:58:52,168 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:54,169 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:56,169 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:58:58,170 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:00,171 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:02,172 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:04,173 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:06,173 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:06,278 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:59:06,278 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:59:08,174 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:09,508 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:59:10,175 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:12,176 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:14,176 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:16,177 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:18,178 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:20,179 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:21,407 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:59:21,407 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:59:22,180 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:24,180 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:26,181 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:28,182 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:30,183 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:32,184 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:34,185 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:36,185 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:36,559 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:59:36,560 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:59:38,186 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:39,586 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 11:59:40,187 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:42,188 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:44,189 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:46,190 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:48,190 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:50,191 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:51,712 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 11:59:51,713 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 11:59:52,192 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:54,193 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:56,193 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 11:59:58,194 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:00,195 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:03,196 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:05,197 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:06,846 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:00:06,847 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:00:07,198 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:09,199 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:09,660 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:00:11,200 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:13,200 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:15,201 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:17,202 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:19,203 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:21,203 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:21,978 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:00:21,978 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:00:23,204 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:25,205 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:27,206 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:29,206 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:31,207 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:33,208 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:35,209 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:37,111 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:00:37,111 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:00:37,210 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:39,211 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:39,735 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:00:41,211 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:43,212 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:45,213 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:47,214 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:49,215 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:51,216 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:52,245 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:00:52,245 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:00:53,216 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:55,217 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:57,218 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:00:59,219 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:07,378 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:01:07,379 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:01:09,812 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:01:20,445 DEBUG SenderThread:353216 [sender.py:send():179] send: history +2021-07-13 12:01:20,446 DEBUG SenderThread:353216 [sender.py:send():179] send: summary +2021-07-13 12:01:20,446 INFO SenderThread:353216 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 12:01:21,226 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json +2021-07-13 12:01:22,226 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:22,523 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:01:22,524 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:01:23,227 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:25,228 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:27,228 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:29,229 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:31,230 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:33,231 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:35,232 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:37,233 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:37,665 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:01:37,665 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:01:39,234 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:39,889 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:01:41,235 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:43,236 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:45,237 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:47,238 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:49,239 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:51,240 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:52,795 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:01:52,796 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:01:53,241 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:55,242 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:57,243 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:01:59,244 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:01,245 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:03,246 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:05,247 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:07,248 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:07,929 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:02:07,930 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:02:09,249 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:09,967 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:02:11,250 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:13,251 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:15,252 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:17,253 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:19,254 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:21,255 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:23,062 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:02:23,062 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:02:23,256 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:25,257 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:27,258 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:29,259 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:31,260 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:33,261 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:35,262 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:37,262 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:38,194 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:02:38,194 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:02:39,263 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:40,044 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:02:41,265 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:43,266 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:45,267 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:47,268 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:49,269 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:51,270 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:53,271 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:53,326 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:02:53,327 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:02:55,272 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:57,273 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:02:59,274 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:01,275 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:03,276 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:05,277 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:07,278 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:08,458 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:03:08,458 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:03:09,279 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:10,122 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:03:11,280 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:13,281 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:15,282 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:17,283 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:19,284 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:21,285 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:23,286 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:23,591 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:03:23,591 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:03:25,287 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:27,288 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:29,289 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:31,290 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:33,291 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:35,292 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:37,293 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:38,720 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:03:38,720 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:03:39,294 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:40,201 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:03:41,295 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:43,296 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:45,297 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:47,298 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:49,299 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:51,300 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:53,301 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:53,852 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:03:53,852 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:03:55,302 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:57,303 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:03:59,304 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:01,305 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:03,306 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:05,307 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:07,308 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:08,985 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:04:08,986 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:04:09,309 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:10,279 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:04:11,310 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:13,311 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:15,312 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:17,313 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:20,314 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:22,315 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:24,119 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:04:24,120 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:04:24,316 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:26,317 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:28,318 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:30,319 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:32,320 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:34,321 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:36,322 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:38,323 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:39,266 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:04:39,266 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:04:40,324 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:40,357 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:04:42,325 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:44,326 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:46,327 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:48,328 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:50,329 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:52,330 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:54,331 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:54,399 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:04:54,400 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:04:56,332 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:04:58,333 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:00,334 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:02,335 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:04,336 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:06,337 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:08,339 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:09,530 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:05:09,530 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:05:10,339 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:10,438 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:05:12,340 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:14,342 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:16,343 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:18,344 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:20,345 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:22,346 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:24,347 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:24,661 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:05:24,661 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:05:26,348 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:28,349 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:30,350 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:32,351 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:34,352 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:36,353 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:38,354 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:39,803 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:05:39,804 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:05:40,355 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:40,517 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:05:42,356 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:44,357 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:46,358 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:48,359 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:50,360 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:52,361 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:54,362 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:54,994 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:05:54,995 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:05:56,363 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:05:58,364 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:00,365 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:02,366 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:04,366 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:06,368 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:08,368 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:10,133 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:06:10,133 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:06:10,369 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:10,594 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:06:12,370 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:14,371 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:16,372 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:18,373 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:20,374 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:22,374 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:24,375 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:25,266 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:06:25,267 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:06:26,376 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:28,377 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:30,377 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:32,378 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:34,379 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:36,380 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:38,381 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:40,381 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:40,397 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:06:40,397 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:06:40,671 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:06:42,382 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:44,383 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:46,384 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:48,384 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:50,385 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:06:55,527 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:06:55,527 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:07:10,658 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:07:10,658 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:07:10,787 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:07:11,468 DEBUG SenderThread:353216 [sender.py:send():179] send: history +2021-07-13 12:07:11,468 DEBUG SenderThread:353216 [sender.py:send():179] send: summary +2021-07-13 12:07:11,469 INFO SenderThread:353216 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 12:07:12,393 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json +2021-07-13 12:07:12,393 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:13,394 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:14,394 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:16,395 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:18,396 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:20,397 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:22,398 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:24,398 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:25,789 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:07:25,790 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:07:26,399 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:28,400 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:30,401 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:32,402 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:34,403 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:36,403 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:38,404 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:40,405 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:40,828 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:07:40,920 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:07:40,920 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:07:42,406 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:44,407 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:46,408 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:48,408 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:50,409 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:52,410 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:54,411 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:56,056 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:07:56,056 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:07:56,412 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:07:58,413 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:00,414 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:02,415 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:04,415 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:06,416 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:09,417 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:10,904 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:08:11,259 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:08:11,259 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:08:11,418 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:13,419 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:15,420 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:17,421 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:19,422 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:21,423 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:23,423 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:25,424 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:26,390 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:08:26,390 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:08:27,425 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:29,426 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:31,427 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:33,427 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:35,428 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:37,429 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:39,430 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:40,967 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:08:41,431 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:41,523 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:08:41,523 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:08:43,431 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:45,432 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:47,433 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:49,434 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:51,435 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:53,436 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:55,436 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:56,655 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:08:56,655 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:08:57,437 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:08:59,438 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:01,439 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:03,440 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:05,441 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:07,442 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:09,443 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:11,033 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:09:11,444 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:11,806 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:09:11,806 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:09:13,445 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:15,446 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:17,447 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:19,448 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:21,448 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:23,449 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:25,450 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:26,945 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:09:26,945 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:09:27,451 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:29,452 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:31,453 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:33,454 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:35,455 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:37,456 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:39,456 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:41,102 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:09:41,457 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:42,142 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:09:42,142 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:09:43,458 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:45,459 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:47,460 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:49,461 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:51,462 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:53,463 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:55,464 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:57,274 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:09:57,274 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:09:57,464 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:09:59,465 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:01,466 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:03,467 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:05,468 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:07,469 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:09,470 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:11,173 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:10:11,471 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:12,413 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:10:12,414 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:10:13,472 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:15,473 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:17,474 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:19,475 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:21,476 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:23,476 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:25,477 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:27,478 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:27,548 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:10:27,548 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:10:29,479 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:31,480 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:33,481 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:35,482 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:37,483 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:39,484 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:41,245 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:10:41,485 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:42,679 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:10:42,680 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:10:43,486 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:45,487 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:47,487 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:49,488 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:51,489 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:53,490 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:55,491 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:57,492 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:10:57,812 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:10:57,812 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:10:59,493 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:01,494 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:03,495 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:05,496 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:07,497 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:09,497 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:11,317 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:11:11,498 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:12,942 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:11:12,942 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:11:13,499 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:15,500 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:17,501 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:19,501 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:21,502 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:23,503 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:25,504 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:27,505 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:28,163 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:11:28,164 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:11:29,506 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:31,507 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:33,508 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:35,509 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:37,510 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:39,510 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:41,392 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:11:41,511 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:43,299 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:11:43,299 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:11:43,512 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:45,513 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:47,514 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:49,515 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:51,516 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:53,517 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:55,517 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:57,518 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:11:58,429 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:11:58,430 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:11:59,519 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:01,520 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:03,521 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:05,522 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:07,523 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:09,524 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:11,468 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:12:11,525 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:13,526 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:13,840 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:12:13,840 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:12:15,527 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:17,527 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:20,529 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:22,530 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:24,531 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:26,532 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:28,533 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:28,970 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:12:28,971 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:12:30,534 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:32,535 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:34,535 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:36,536 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:38,537 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:40,538 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:12:41,538 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:12:44,115 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:12:44,116 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:12:59,247 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:12:59,248 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:13:02,477 DEBUG SenderThread:353216 [sender.py:send():179] send: history +2021-07-13 12:13:02,477 DEBUG SenderThread:353216 [sender.py:send():179] send: summary +2021-07-13 12:13:02,478 INFO SenderThread:353216 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 12:13:02,549 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/wandb-summary.json +2021-07-13 12:13:03,549 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:04,550 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:06,551 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:08,552 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:10,552 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:11,616 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:13:12,553 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:14,473 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:13:14,474 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:13:14,554 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:16,555 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:18,556 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:20,557 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:22,558 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:24,559 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:26,560 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:28,561 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:29,632 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:13:29,632 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:13:30,561 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:32,562 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:34,563 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:36,564 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:38,565 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:40,565 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:41,684 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:13:42,566 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:44,567 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:44,764 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:13:44,764 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:13:46,568 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:48,569 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:50,570 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:52,571 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:54,572 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:56,573 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:58,574 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:13:59,895 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:13:59,896 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:14:00,575 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:02,576 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:04,577 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:06,577 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:08,578 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:10,579 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:11,756 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:14:12,580 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:14,581 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:15,058 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:14:15,058 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:14:16,581 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:18,582 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:20,583 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:22,584 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:24,585 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:26,585 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:28,586 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:30,189 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:14:30,190 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:14:30,587 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:32,588 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:34,589 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:36,589 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:38,590 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:40,591 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:41,831 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:14:42,592 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:44,593 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:45,328 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:14:45,328 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:14:46,593 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:48,594 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:50,595 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:52,596 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:54,597 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:56,598 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:14:58,598 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:00,465 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:15:00,465 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:15:00,599 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:02,600 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:04,601 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:06,601 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:08,602 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:10,603 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:11,905 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:15:12,604 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:14,605 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:15,598 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:15:15,598 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:15:16,606 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:18,606 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:20,607 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:22,608 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:24,609 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:26,610 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:28,611 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:30,611 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:30,729 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:15:30,730 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:15:32,612 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:34,613 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:36,613 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:38,614 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:40,615 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:41,980 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:15:42,616 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:44,616 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:45,872 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:15:45,872 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:15:46,617 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:48,618 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:50,619 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:52,620 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:54,620 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:56,621 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:15:58,622 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:00,623 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:01,008 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:16:01,008 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:16:02,623 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:04,624 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:06,625 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:08,626 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:10,626 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:12,058 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:16:12,627 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:14,628 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:16,148 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:16:16,148 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:16:16,629 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:19,630 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:21,631 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:23,632 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:25,633 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:27,633 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:29,634 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:31,287 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:16:31,287 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:16:31,635 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:33,636 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:35,637 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:37,637 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:39,638 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:41,639 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:42,135 DEBUG SenderThread:353216 [sender.py:send():179] send: stats +2021-07-13 12:16:43,640 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:45,640 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:46,418 DEBUG HandlerThread:353216 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:16:46,419 DEBUG SenderThread:353216 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:16:47,641 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:49,642 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:51,643 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:53,644 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:55,488 WARNING MainThread:353216 [internal.py:wandb_internal():147] Internal process interrupt: 1 +2021-07-13 12:16:55,645 INFO Thread-8 :353216 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/files/output.log +2021-07-13 12:16:55,695 WARNING MainThread:353216 [internal.py:wandb_internal():147] Internal process interrupt: 2 +2021-07-13 12:16:55,695 ERROR MainThread:353216 [internal.py:wandb_internal():150] Internal process interrupted. +2021-07-13 12:16:55,767 INFO SenderThread:353216 [sender.py:finish():945] shutting down sender +2021-07-13 12:16:55,767 INFO SenderThread:353216 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 12:16:55,768 INFO WriterThread:353216 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/run-1vxvoloi.wandb +2021-07-13 12:16:55,768 INFO HandlerThread:353216 [handler.py:finish():638] shutting down handler +2021-07-13 12:16:55,990 INFO MainThread:353216 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_113435-1vxvoloi/logs/debug.log b/wandb/run-20210713_113435-1vxvoloi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..761b35acfd128ecd6ac30c1dca375b469b029b22 --- /dev/null +++ b/wandb/run-20210713_113435-1vxvoloi/logs/debug.log @@ -0,0 +1,27 @@ +2021-07-13 11:34:35,615 INFO MainThread:351681 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 11:34:35,615 INFO MainThread:351681 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 11:34:35,616 INFO MainThread:351681 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/logs/debug.log +2021-07-13 11:34:35,616 INFO MainThread:351681 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_113435-1vxvoloi/logs/debug-internal.log +2021-07-13 11:34:35,616 INFO MainThread:351681 [wandb_init.py:init():370] calling init triggers +2021-07-13 11:34:35,616 INFO MainThread:351681 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 11:34:35,616 INFO MainThread:351681 [wandb_init.py:init():419] starting backend +2021-07-13 11:34:35,616 INFO MainThread:351681 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 11:34:35,672 INFO MainThread:351681 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 11:34:35,726 INFO MainThread:351681 [backend.py:ensure_launched():139] started backend process with pid: 353216 +2021-07-13 11:34:35,728 INFO MainThread:351681 [wandb_init.py:init():424] backend started and connected +2021-07-13 11:34:35,731 INFO MainThread:351681 [wandb_init.py:init():472] updated telemetry +2021-07-13 11:34:35,732 INFO MainThread:351681 [wandb_init.py:init():491] communicating current version +2021-07-13 11:34:36,411 INFO MainThread:351681 [wandb_init.py:init():496] got version response +2021-07-13 11:34:36,411 INFO MainThread:351681 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 11:34:36,584 INFO MainThread:351681 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 11:34:37,772 INFO MainThread:351681 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 11:34:37,773 INFO MainThread:351681 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 11:34:37,774 INFO MainThread:351681 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 11:34:37,775 INFO MainThread:351681 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 11:34:37,776 INFO MainThread:351681 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 11:34:37,781 INFO MainThread:351681 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_11-27-24_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 11:34:37,783 INFO MainThread:351681 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 11:34:37,785 INFO MainThread:351681 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 12:16:55,997 INFO MainThread:351681 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255 +2021-07-13 12:16:55,997 INFO MainThread:351681 [wandb_run.py:_restore():1565] restore diff --git a/wandb/run-20210713_113435-1vxvoloi/run-1vxvoloi.wandb b/wandb/run-20210713_113435-1vxvoloi/run-1vxvoloi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5d42217ab11f5e876b4003d3559613a89881d290 Binary files /dev/null and b/wandb/run-20210713_113435-1vxvoloi/run-1vxvoloi.wandb differ diff --git a/wandb/run-20210713_124748-vubzdnhu/files/config.yaml b/wandb/run-20210713_124748-vubzdnhu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27092f2d261895e2f6fb39efeaa4cd5f57ef7a35 --- /dev/null +++ b/wandb/run-20210713_124748-vubzdnhu/files/config.yaml @@ -0,0 +1,307 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 4 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_12-47-00_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_124748-vubzdnhu/files/output.log b/wandb/run-20210713_124748-vubzdnhu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1d7dc66e638922cc583f4c7fe01d3a01ded1e71a --- /dev/null +++ b/wandb/run-20210713_124748-vubzdnhu/files/output.log @@ -0,0 +1,328 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1647, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 624, in xla_pmap_impl + compiled_fun, fingerprint = parallel_callable(fun, backend, axis_name, axis_size, + File "/home/dat/pino/lib/python3.8/site-packages/jax/linear_util.py", line 262, in memoized_fun + ans = call(fun, *args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 899, in parallel_callable + compiled = xla.backend_compile(backend, built, compile_options) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile + return backend.compile(built_c, compile_options=options) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Ran out of memory in memory space hbm. Used 35.77G of 15.48G hbm. Exceeded hbm capacity by 20.29G. +Total hbm usage >= 36.29G: + reserved 530.00M + program 35.77G + arguments 0B +Output size 0B; shares 0B with arguments. +Program hbm requirement 35.77G: + global 692.0K + scoped 253.0K + HLO temp 35.77G (97.6% utilization: Unpadded (34.82G) Padded (35.67G), 0.3% fragmentation (105.77M)) + Largest program allocations in hbm: + 1. Size: 6.15G + Operator: op_type="dot_general" op_name="pmap(train_step)/dot_general[ dimension_numbers=(((2,), (0,)), ((), ()))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/pino/lib/python3.8/site-packages/flax/linen/linear.py" source_line=175 + Shape: f32[8,4096,50358]{1,2,0:T(8,128)} + Unpadded size: 6.15G + Extra memory due to padding: 256.0K (1.0x expansion) + XLA label: %fusion.1737.remat4 = f32[8,4096,50358]{1,2,0:T(8,128)} fusion(f32[50358]{0:T(1024)} %get-tuple-element.23321, f32[768,50358,1]{0,1,2:T(8,128)} %bitcast.5512, f32[768]{0:T(1024)} %get-tuple-element.23322, f32[768]{0:T(1024)} %get-tuple-element.23323, f32[8... + Allocation type: HLO temp + ========================== + 2. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1805.remat6 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[8,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 3. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13201 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2412, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9402, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 4. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1805.remat6 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[8,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 5. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13202 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2411, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9401, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 6. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1814 = bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2416, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9406, f32[8,12,28,128,128]{3,4,2,1,0:T(8,128)} %get-tuple-element.20627, f32[8,12,28,128,384]{... + Allocation type: HLO temp + ========================== + 7. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13199 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2414, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9404, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 8. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13200 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2413, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9403, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 9. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1816.remat = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2418, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9408, f32[8,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 10. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1815.remat = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2417, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9407, f32[8,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 11. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13203 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2410, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9400, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 12. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13204 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2409, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9399, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 13. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13205 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2408, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9398, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 14. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11557 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25239, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28505.remat_uncompressed, f32[8,12... + Allocation type: HLO temp + ========================== + 15. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11549 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25240, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28470.remat_uncompressed.remat, f3... + Allocation type: HLO temp + ========================== + 16. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11469 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.20990, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28115, f32[8,12,32,128,64]{3,2,4,1... + Allocation type: HLO temp + ========================== + 17. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11477 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.20989, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28151, f32[8,12,32,128,64]{3,2,4,1... + Allocation type: HLO temp + ========================== + 18. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11541 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25236, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28435.remat_uncompressed, f32[8,12... + Allocation type: HLO temp + ========================== + 19. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=584 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.2085.remat5.1.remat = f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} fusion(f32[8,28,128,384]{2,3,1,0:T(8,128)} %get-tuple-element.20992, bf16[8,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2473.remat_uncompressed, f32[8,12,32,128,64]{3,2,4,1,0:T(8,128... + Allocation type: HLO temp + ========================== + 20. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11533 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25238, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28400.remat_uncompressed, f32[8,12... + Allocation type: HLO temp + ========================== +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 725, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile + return backend.compile(built_c, compile_options=options) +RuntimeError: Resource exhausted: Ran out of memory in memory space hbm. Used 35.77G of 15.48G hbm. Exceeded hbm capacity by 20.29G. +Total hbm usage >= 36.29G: + reserved 530.00M + program 35.77G + arguments 0B +Output size 0B; shares 0B with arguments. +Program hbm requirement 35.77G: + global 692.0K + scoped 253.0K + HLO temp 35.77G (97.6% utilization: Unpadded (34.82G) Padded (35.67G), 0.3% fragmentation (105.77M)) + Largest program allocations in hbm: + 1. Size: 6.15G + Operator: op_type="dot_general" op_name="pmap(train_step)/dot_general[ dimension_numbers=(((2,), (0,)), ((), ()))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/pino/lib/python3.8/site-packages/flax/linen/linear.py" source_line=175 + Shape: f32[8,4096,50358]{1,2,0:T(8,128)} + Unpadded size: 6.15G + Extra memory due to padding: 256.0K (1.0x expansion) + XLA label: %fusion.1737.remat4 = f32[8,4096,50358]{1,2,0:T(8,128)} fusion(f32[50358]{0:T(1024)} %get-tuple-element.23321, f32[768,50358,1]{0,1,2:T(8,128)} %bitcast.5512, f32[768]{0:T(1024)} %get-tuple-element.23322, f32[768]{0:T(1024)} %get-tuple-element.23323, f32[8... + Allocation type: HLO temp + ========================== + 2. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1805.remat6 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[8,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 3. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13201 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2412, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9402, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 4. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1805.remat6 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[8,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 5. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13202 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2411, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9401, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 6. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1814 = bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2416, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9406, f32[8,12,28,128,128]{3,4,2,1,0:T(8,128)} %get-tuple-element.20627, f32[8,12,28,128,384]{... + Allocation type: HLO temp + ========================== + 7. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13199 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2414, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9404, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 8. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13200 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2413, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9403, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 9. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1816.remat = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2418, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9408, f32[8,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 10. Size: 672.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.1815.remat = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2417, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9407, f32[8,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 11. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13203 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2410, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9400, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 12. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13204 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2409, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9399, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 13. Size: 672.00M + Shape: bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 672.00M + XLA label: %fusion.13205 = (bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[8,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.2408, f32[8,12,28,128]{3,2,1,0:T(8,128)} %fusion.9398, f32[8,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 14. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11557 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25239, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28505.remat_uncompressed, f32[8,12... + Allocation type: HLO temp + ========================== + 15. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11549 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25240, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28470.remat_uncompressed.remat, f3... + Allocation type: HLO temp + ========================== + 16. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11469 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.20990, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28115, f32[8,12,32,128,64]{3,2,4,1... + Allocation type: HLO temp + ========================== + 17. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11477 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.20989, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28151, f32[8,12,32,128,64]{3,2,4,1... + Allocation type: HLO temp + ========================== + 18. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11541 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25236, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28435.remat_uncompressed, f32[8,12... + Allocation type: HLO temp + ========================== + 19. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=584 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.2085.remat5.1.remat = f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} fusion(f32[8,28,128,384]{2,3,1,0:T(8,128)} %get-tuple-element.20992, bf16[8,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2473.remat_uncompressed, f32[8,12,32,128,64]{3,2,4,1,0:T(8,128... + Allocation type: HLO temp + ========================== + 20. Size: 504.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 504.00M + XLA label: %fusion.11533 = (f32[8,12,28,128]{3,2,1,0:T(8,128)}, f32[8,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[8,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25238, bf16[8,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28400.remat_uncompressed, f32[8,12... + Allocation type: HLO temp + ========================== \ No newline at end of file diff --git a/wandb/run-20210713_124748-vubzdnhu/files/requirements.txt b/wandb/run-20210713_124748-vubzdnhu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_124748-vubzdnhu/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_124748-vubzdnhu/files/wandb-metadata.json b/wandb/run-20210713_124748-vubzdnhu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..77ab9d96013388c98404f2646bf83789d69f0838 --- /dev/null +++ b/wandb/run-20210713_124748-vubzdnhu/files/wandb-metadata.json @@ -0,0 +1,45 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T12:47:50.855421", + "startedAt": "2021-07-13T12:47:48.810573", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=4", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_124748-vubzdnhu/files/wandb-summary.json b/wandb/run-20210713_124748-vubzdnhu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210713_124748-vubzdnhu/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210713_124748-vubzdnhu/logs/debug-internal.log b/wandb/run-20210713_124748-vubzdnhu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2e83ba7ca1608b3b0886db7ba81d34e6bb4e6c65 --- /dev/null +++ b/wandb/run-20210713_124748-vubzdnhu/logs/debug-internal.log @@ -0,0 +1,164 @@ +2021-07-13 12:47:49,474 INFO MainThread:355550 [internal.py:wandb_internal():88] W&B internal server running at pid: 355550, started at: 2021-07-13 12:47:49.474151 +2021-07-13 12:47:49,476 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 12:47:49,477 INFO WriterThread:355550 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/run-vubzdnhu.wandb +2021-07-13 12:47:49,477 DEBUG SenderThread:355550 [sender.py:send():179] send: header +2021-07-13 12:47:49,477 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: check_version +2021-07-13 12:47:49,517 DEBUG SenderThread:355550 [sender.py:send():179] send: run +2021-07-13 12:47:49,718 INFO SenderThread:355550 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files +2021-07-13 12:47:49,718 INFO SenderThread:355550 [sender.py:_start_run_threads():716] run started: vubzdnhu with start time 1626180468 +2021-07-13 12:47:49,718 DEBUG SenderThread:355550 [sender.py:send():179] send: summary +2021-07-13 12:47:49,718 INFO SenderThread:355550 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 12:47:49,719 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 12:47:50,719 INFO Thread-8 :355550 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/wandb-summary.json +2021-07-13 12:47:50,855 DEBUG HandlerThread:355550 [meta.py:__init__():39] meta init +2021-07-13 12:47:50,855 DEBUG HandlerThread:355550 [meta.py:__init__():53] meta init done +2021-07-13 12:47:50,855 DEBUG HandlerThread:355550 [meta.py:probe():210] probe +2021-07-13 12:47:50,856 DEBUG HandlerThread:355550 [meta.py:_setup_git():200] setup git +2021-07-13 12:47:50,887 DEBUG HandlerThread:355550 [meta.py:_setup_git():207] setup git done +2021-07-13 12:47:50,888 DEBUG HandlerThread:355550 [meta.py:_save_pip():57] save pip +2021-07-13 12:47:50,888 DEBUG HandlerThread:355550 [meta.py:_save_pip():71] save pip done +2021-07-13 12:47:50,888 DEBUG HandlerThread:355550 [meta.py:probe():252] probe done +2021-07-13 12:47:50,892 DEBUG SenderThread:355550 [sender.py:send():179] send: files +2021-07-13 12:47:50,892 INFO SenderThread:355550 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 12:47:50,899 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:47:50,899 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:47:51,026 DEBUG SenderThread:355550 [sender.py:send():179] send: config +2021-07-13 12:47:51,027 DEBUG SenderThread:355550 [sender.py:send():179] send: config +2021-07-13 12:47:51,027 DEBUG SenderThread:355550 [sender.py:send():179] send: config +2021-07-13 12:47:51,378 INFO Thread-11 :355550 [upload_job.py:push():137] Uploaded file /tmp/tmpy_8wrz9rwandb/2yxmm3l6-wandb-metadata.json +2021-07-13 12:47:51,719 INFO Thread-8 :355550 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/requirements.txt +2021-07-13 12:47:51,719 INFO Thread-8 :355550 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/output.log +2021-07-13 12:47:51,719 INFO Thread-8 :355550 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/wandb-metadata.json +2021-07-13 12:48:05,724 INFO Thread-8 :355550 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/output.log +2021-07-13 12:48:06,028 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:48:06,029 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:48:07,724 INFO Thread-8 :355550 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/output.log +2021-07-13 12:48:18,943 DEBUG SenderThread:355550 [sender.py:send():179] send: stats +2021-07-13 12:48:20,728 INFO Thread-8 :355550 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/config.yaml +2021-07-13 12:48:21,161 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:48:21,161 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:48:35,733 INFO Thread-8 :355550 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/output.log +2021-07-13 12:48:36,292 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:48:36,292 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:48:49,027 DEBUG SenderThread:355550 [sender.py:send():179] send: stats +2021-07-13 12:48:51,451 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:48:51,451 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:49:06,603 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:49:06,604 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:49:19,111 DEBUG SenderThread:355550 [sender.py:send():179] send: stats +2021-07-13 12:49:21,736 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:49:21,737 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:49:36,867 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:49:36,868 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:49:49,193 DEBUG SenderThread:355550 [sender.py:send():179] send: stats +2021-07-13 12:49:51,999 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:49:52,000 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:50:07,132 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:50:07,132 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:50:19,272 DEBUG SenderThread:355550 [sender.py:send():179] send: stats +2021-07-13 12:50:19,770 INFO Thread-8 :355550 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/output.log +2021-07-13 12:50:20,563 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:20,564 DEBUG SenderThread:355550 [sender.py:send():179] send: telemetry +2021-07-13 12:50:20,564 DEBUG SenderThread:355550 [sender.py:send():179] send: exit +2021-07-13 12:50:20,564 INFO SenderThread:355550 [sender.py:send_exit():287] handling exit code: 1 +2021-07-13 12:50:20,565 INFO SenderThread:355550 [sender.py:send_exit():295] send defer +2021-07-13 12:50:20,565 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:20,566 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:20,566 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-13 12:50:20,566 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:20,567 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-13 12:50:20,567 INFO SenderThread:355550 [sender.py:transition_state():308] send defer: 1 +2021-07-13 12:50:20,567 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:20,567 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-13 12:50:20,576 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:20,576 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-13 12:50:20,576 INFO SenderThread:355550 [sender.py:transition_state():308] send defer: 2 +2021-07-13 12:50:20,576 DEBUG SenderThread:355550 [sender.py:send():179] send: stats +2021-07-13 12:50:20,577 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:20,577 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-13 12:50:20,577 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:20,577 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-13 12:50:20,577 INFO SenderThread:355550 [sender.py:transition_state():308] send defer: 3 +2021-07-13 12:50:20,577 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:20,577 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-13 12:50:20,578 DEBUG SenderThread:355550 [sender.py:send():179] send: summary +2021-07-13 12:50:20,578 INFO SenderThread:355550 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 12:50:20,578 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:20,578 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-13 12:50:20,578 INFO SenderThread:355550 [sender.py:transition_state():308] send defer: 4 +2021-07-13 12:50:20,579 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:20,579 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-13 12:50:20,579 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:20,579 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-13 12:50:20,668 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:20,746 INFO SenderThread:355550 [sender.py:transition_state():308] send defer: 5 +2021-07-13 12:50:20,746 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:20,746 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:20,746 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-13 12:50:20,747 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:20,747 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-13 12:50:20,747 INFO SenderThread:355550 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 12:50:20,771 INFO Thread-8 :355550 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/config.yaml +2021-07-13 12:50:20,771 INFO SenderThread:355550 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/output.log +2021-07-13 12:50:20,771 INFO SenderThread:355550 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/wandb-summary.json +2021-07-13 12:50:20,772 INFO SenderThread:355550 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files +2021-07-13 12:50:20,772 INFO SenderThread:355550 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/requirements.txt requirements.txt +2021-07-13 12:50:20,772 INFO SenderThread:355550 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/output.log output.log +2021-07-13 12:50:20,772 INFO SenderThread:355550 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/wandb-metadata.json wandb-metadata.json +2021-07-13 12:50:20,772 INFO SenderThread:355550 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/config.yaml config.yaml +2021-07-13 12:50:20,772 INFO SenderThread:355550 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/wandb-summary.json wandb-summary.json +2021-07-13 12:50:20,775 INFO SenderThread:355550 [sender.py:transition_state():308] send defer: 6 +2021-07-13 12:50:20,777 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:20,778 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-13 12:50:20,784 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:20,784 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-13 12:50:20,784 INFO SenderThread:355550 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 12:50:20,848 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:20,848 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:20,950 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:20,950 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:21,052 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:21,053 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:21,154 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:21,155 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:21,218 INFO Thread-15 :355550 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/wandb-summary.json +2021-07-13 12:50:21,219 INFO Thread-13 :355550 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/output.log +2021-07-13 12:50:21,224 INFO Thread-14 :355550 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/config.yaml +2021-07-13 12:50:21,256 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:21,257 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:21,271 INFO Thread-12 :355550 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/files/requirements.txt +2021-07-13 12:50:21,359 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:21,359 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:21,461 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:21,461 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:21,471 INFO Thread-7 :355550 [sender.py:transition_state():308] send defer: 7 +2021-07-13 12:50:21,471 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:21,471 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-13 12:50:21,472 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:21,472 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-13 12:50:21,562 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:21,867 INFO SenderThread:355550 [sender.py:transition_state():308] send defer: 8 +2021-07-13 12:50:21,867 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:21,868 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:21,868 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-13 12:50:21,868 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:21,868 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-13 12:50:21,868 INFO SenderThread:355550 [sender.py:transition_state():308] send defer: 9 +2021-07-13 12:50:21,869 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:50:21,869 INFO HandlerThread:355550 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-13 12:50:21,869 DEBUG SenderThread:355550 [sender.py:send():179] send: final +2021-07-13 12:50:21,869 DEBUG SenderThread:355550 [sender.py:send():179] send: footer +2021-07-13 12:50:21,869 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: defer +2021-07-13 12:50:21,869 INFO SenderThread:355550 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-13 12:50:21,969 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:50:21,970 DEBUG SenderThread:355550 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:50:21,970 INFO SenderThread:355550 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 12:50:21,971 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: get_summary +2021-07-13 12:50:21,972 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-13 12:50:21,973 DEBUG HandlerThread:355550 [handler.py:handle_request():124] handle_request: shutdown +2021-07-13 12:50:21,973 INFO HandlerThread:355550 [handler.py:finish():638] shutting down handler +2021-07-13 12:50:22,870 INFO WriterThread:355550 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/run-vubzdnhu.wandb +2021-07-13 12:50:22,970 INFO SenderThread:355550 [sender.py:finish():945] shutting down sender +2021-07-13 12:50:22,970 INFO SenderThread:355550 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 12:50:22,970 INFO SenderThread:355550 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 12:50:22,973 INFO MainThread:355550 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_124748-vubzdnhu/logs/debug.log b/wandb/run-20210713_124748-vubzdnhu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2284042df95b19ccc3fb1e676ea3cb377f6e05d0 --- /dev/null +++ b/wandb/run-20210713_124748-vubzdnhu/logs/debug.log @@ -0,0 +1,119 @@ +2021-07-13 12:47:48,812 INFO MainThread:354027 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 12:47:48,812 INFO MainThread:354027 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 12:47:48,812 INFO MainThread:354027 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/logs/debug.log +2021-07-13 12:47:48,812 INFO MainThread:354027 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_124748-vubzdnhu/logs/debug-internal.log +2021-07-13 12:47:48,812 INFO MainThread:354027 [wandb_init.py:init():370] calling init triggers +2021-07-13 12:47:48,812 INFO MainThread:354027 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 12:47:48,812 INFO MainThread:354027 [wandb_init.py:init():419] starting backend +2021-07-13 12:47:48,812 INFO MainThread:354027 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 12:47:48,868 INFO MainThread:354027 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 12:47:48,922 INFO MainThread:354027 [backend.py:ensure_launched():139] started backend process with pid: 355550 +2021-07-13 12:47:48,924 INFO MainThread:354027 [wandb_init.py:init():424] backend started and connected +2021-07-13 12:47:48,927 INFO MainThread:354027 [wandb_init.py:init():472] updated telemetry +2021-07-13 12:47:48,928 INFO MainThread:354027 [wandb_init.py:init():491] communicating current version +2021-07-13 12:47:49,516 INFO MainThread:354027 [wandb_init.py:init():496] got version response +2021-07-13 12:47:49,516 INFO MainThread:354027 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 12:47:49,718 INFO MainThread:354027 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 12:47:50,895 INFO MainThread:354027 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 12:47:50,896 INFO MainThread:354027 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 12:47:50,896 INFO MainThread:354027 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 12:47:50,898 INFO MainThread:354027 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 12:47:50,898 INFO MainThread:354027 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 12:47:50,904 INFO MainThread:354027 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_12-47-00_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 12:47:50,906 INFO MainThread:354027 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 12:47:50,908 INFO MainThread:354027 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 12:50:18,269 INFO MainThread:354027 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-13 12:50:18,270 INFO MainThread:354027 [wandb_run.py:_restore():1565] restore +2021-07-13 12:50:20,567 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 1388 +} + +2021-07-13 12:50:20,746 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 1388 +} + +2021-07-13 12:50:20,849 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 37457 +} + +2021-07-13 12:50:20,951 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 37457 + total_bytes: 37457 +} + +2021-07-13 12:50:21,053 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 37457 + total_bytes: 37457 +} + +2021-07-13 12:50:21,155 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 37457 + total_bytes: 37457 +} + +2021-07-13 12:50:21,257 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 37457 + total_bytes: 37457 +} + +2021-07-13 12:50:21,359 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 37457 + total_bytes: 37457 +} + +2021-07-13 12:50:21,461 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 37457 + total_bytes: 37457 +} + +2021-07-13 12:50:21,868 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 37457 + total_bytes: 37457 +} + +2021-07-13 12:50:21,970 INFO MainThread:354027 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 37457 + total_bytes: 37457 +} + +2021-07-13 12:50:23,279 INFO MainThread:354027 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210713_124748-vubzdnhu/run-vubzdnhu.wandb b/wandb/run-20210713_124748-vubzdnhu/run-vubzdnhu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b935ee0f979e0a2d6204b9be5e7ac099ba4421fe Binary files /dev/null and b/wandb/run-20210713_124748-vubzdnhu/run-vubzdnhu.wandb differ diff --git a/wandb/run-20210713_125231-3e8clw7k/files/config.yaml b/wandb/run-20210713_125231-3e8clw7k/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b032f2c446596ec6147b257a756509bdd3bd6b2a --- /dev/null +++ b/wandb/run-20210713_125231-3e8clw7k/files/config.yaml @@ -0,0 +1,307 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 8 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_12-51-41_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_125231-3e8clw7k/files/output.log b/wandb/run-20210713_125231-3e8clw7k/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c40cd5dfcbd6cf6bc0df75be166e8f4143416fd1 --- /dev/null +++ b/wandb/run-20210713_125231-3e8clw7k/files/output.log @@ -0,0 +1,322 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1647, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 624, in xla_pmap_impl + compiled_fun, fingerprint = parallel_callable(fun, backend, axis_name, axis_size, + File "/home/dat/pino/lib/python3.8/site-packages/jax/linear_util.py", line 262, in memoized_fun + ans = call(fun, *args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 899, in parallel_callable + compiled = xla.backend_compile(backend, built, compile_options) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile + return backend.compile(built_c, compile_options=options) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Ran out of memory in memory space hbm. Used 77.35G of 15.48G hbm. Exceeded hbm capacity by 61.87G. +Total hbm usage >= 77.87G: + reserved 530.00M + program 77.35G + arguments 0B +Output size 0B; shares 0B with arguments. +Program hbm requirement 77.35G: + global 692.0K + scoped 253.0K + HLO temp 77.35G (97.9% utilization: Unpadded (75.71G) Padded (77.34G), 0.0% fragmentation (10.14M)) + Largest program allocations in hbm: + 1. Size: 12.29G + Operator: op_type="dot_general" op_name="pmap(train_step)/dot_general[ dimension_numbers=(((2,), (0,)), ((), ()))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/pino/lib/python3.8/site-packages/flax/linen/linear.py" source_line=175 + Shape: f32[16,4096,50358]{1,2,0:T(8,128)} + Unpadded size: 12.29G + Extra memory due to padding: 512.0K (1.0x expansion) + XLA label: %fusion.1737.remat7 = f32[16,4096,50358]{1,2,0:T(8,128)} fusion(f32[50358]{0:T(1024)} %get-tuple-element.23314, f32[768,50358,1]{0,1,2:T(8,128)} %bitcast.5719, f32[768]{0:T(1024)} %get-tuple-element.23315, f32[768]{0:T(1024)} %get-tuple-element.23316, f32[... + Allocation type: HLO temp + ========================== + 2. Size: 2.62G + Operator: op_type="mul" op_name="pmap(train_step)/mul" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} + Unpadded size: 2.62G + XLA label: %fusion.1803 = f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} fusion(f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} %fusion.1984, f32[16,12,28,128]{3,2,1,0:T(8,128)} %negate.34, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:T(8,128)} ... + Allocation type: HLO temp + ========================== + 3. Size: 2.62G + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(transpose(jvp(_einsum)))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=626 + Shape: f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} + Unpadded size: 2.62G + XLA label: %fusion.1984 = f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} fusion(f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} %fusion.1986, pred[]{:T(1024)} %fusion.9668, bf16[16,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2490.remat_uncompressed.remat, f32[16,12,32,128,... + Allocation type: HLO temp + ========================== + 4. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1816.remat2 = bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2418, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9170, f32[16,12,28,128,128]{3,4,2,1,0:T(8,128)} %fusion.3004.remat3, f32[16,12,28,12... + Allocation type: HLO temp + ========================== + 5. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1805.remat6 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:... + Allocation type: HLO temp + ========================== + 6. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1805.remat6 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:... + Allocation type: HLO temp + ========================== + 7. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13419 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2417, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9169, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 8. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13420 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2416, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9168, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 9. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13421 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2415, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9167, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 10. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13422 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2414, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9166, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 11. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13423 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2413, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9165, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 12. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13424 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2412, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9164, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 13. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13425 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2411, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9163, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 14. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13426 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2410, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9162, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 15. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13427 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2409, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9161, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 16. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13428 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2408, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9160, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 17. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11600 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25240, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28186.remat_uncompressed, f32[... + Allocation type: HLO temp + ========================== + 18. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=584 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.2061.remat6 = f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} fusion(f32[16,28,128,384]{2,3,1,0:T(8,128)} %get-tuple-element.20979, bf16[16,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2489.remat_uncompressed, f32[16,12,32,128,64]{3,2,4,1,0:T(8,128)} %... + Allocation type: HLO temp + ========================== + 19. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11584 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25199, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28115, f32[16,12,32,128,64]{3,... + Allocation type: HLO temp + ========================== + 20. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11592 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25220, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28151.remat_uncompressed, f32[... + Allocation type: HLO temp + ========================== +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 725, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile + return backend.compile(built_c, compile_options=options) +RuntimeError: Resource exhausted: Ran out of memory in memory space hbm. Used 77.35G of 15.48G hbm. Exceeded hbm capacity by 61.87G. +Total hbm usage >= 77.87G: + reserved 530.00M + program 77.35G + arguments 0B +Output size 0B; shares 0B with arguments. +Program hbm requirement 77.35G: + global 692.0K + scoped 253.0K + HLO temp 77.35G (97.9% utilization: Unpadded (75.71G) Padded (77.34G), 0.0% fragmentation (10.14M)) + Largest program allocations in hbm: + 1. Size: 12.29G + Operator: op_type="dot_general" op_name="pmap(train_step)/dot_general[ dimension_numbers=(((2,), (0,)), ((), ()))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/pino/lib/python3.8/site-packages/flax/linen/linear.py" source_line=175 + Shape: f32[16,4096,50358]{1,2,0:T(8,128)} + Unpadded size: 12.29G + Extra memory due to padding: 512.0K (1.0x expansion) + XLA label: %fusion.1737.remat7 = f32[16,4096,50358]{1,2,0:T(8,128)} fusion(f32[50358]{0:T(1024)} %get-tuple-element.23314, f32[768,50358,1]{0,1,2:T(8,128)} %bitcast.5719, f32[768]{0:T(1024)} %get-tuple-element.23315, f32[768]{0:T(1024)} %get-tuple-element.23316, f32[... + Allocation type: HLO temp + ========================== + 2. Size: 2.62G + Operator: op_type="mul" op_name="pmap(train_step)/mul" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} + Unpadded size: 2.62G + XLA label: %fusion.1803 = f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} fusion(f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} %fusion.1984, f32[16,12,28,128]{3,2,1,0:T(8,128)} %negate.34, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:T(8,128)} ... + Allocation type: HLO temp + ========================== + 3. Size: 2.62G + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(transpose(jvp(_einsum)))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=626 + Shape: f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} + Unpadded size: 2.62G + XLA label: %fusion.1984 = f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} fusion(f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} %fusion.1986, pred[]{:T(1024)} %fusion.9668, bf16[16,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2490.remat_uncompressed.remat, f32[16,12,32,128,... + Allocation type: HLO temp + ========================== + 4. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1816.remat2 = bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2418, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9170, f32[16,12,28,128,128]{3,4,2,1,0:T(8,128)} %fusion.3004.remat3, f32[16,12,28,12... + Allocation type: HLO temp + ========================== + 5. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1805.remat6 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:... + Allocation type: HLO temp + ========================== + 6. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1805.remat6 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:... + Allocation type: HLO temp + ========================== + 7. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13419 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2417, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9169, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 8. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13420 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2416, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9168, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 9. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13421 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2415, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9167, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 10. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13422 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2414, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9166, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 11. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13423 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2413, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9165, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 12. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13424 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2412, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9164, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 13. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13425 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2411, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9163, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 14. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13426 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2410, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9162, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 15. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13427 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2409, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9161, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 16. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13428 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2408, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9160, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 17. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11600 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25240, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28186.remat_uncompressed, f32[... + Allocation type: HLO temp + ========================== + 18. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=584 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.2061.remat6 = f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} fusion(f32[16,28,128,384]{2,3,1,0:T(8,128)} %get-tuple-element.20979, bf16[16,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2489.remat_uncompressed, f32[16,12,32,128,64]{3,2,4,1,0:T(8,128)} %... + Allocation type: HLO temp + ========================== + 19. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11584 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25199, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28115, f32[16,12,32,128,64]{3,... + Allocation type: HLO temp + ========================== + 20. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11592 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25220, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28151.remat_uncompressed, f32[... + Allocation type: HLO temp + ========================== \ No newline at end of file diff --git a/wandb/run-20210713_125231-3e8clw7k/files/requirements.txt b/wandb/run-20210713_125231-3e8clw7k/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_125231-3e8clw7k/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_125231-3e8clw7k/files/wandb-metadata.json b/wandb/run-20210713_125231-3e8clw7k/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3219c34024ee184072e5626e5df61b529e8060aa --- /dev/null +++ b/wandb/run-20210713_125231-3e8clw7k/files/wandb-metadata.json @@ -0,0 +1,45 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T12:52:33.640751", + "startedAt": "2021-07-13T12:52:31.556081", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=8", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_125231-3e8clw7k/files/wandb-summary.json b/wandb/run-20210713_125231-3e8clw7k/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210713_125231-3e8clw7k/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210713_125231-3e8clw7k/logs/debug-internal.log b/wandb/run-20210713_125231-3e8clw7k/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..042615117346311d53a352c210a2881c1f44df03 --- /dev/null +++ b/wandb/run-20210713_125231-3e8clw7k/logs/debug-internal.log @@ -0,0 +1,162 @@ +2021-07-13 12:52:32,293 INFO MainThread:357371 [internal.py:wandb_internal():88] W&B internal server running at pid: 357371, started at: 2021-07-13 12:52:32.292806 +2021-07-13 12:52:32,295 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 12:52:32,295 INFO WriterThread:357371 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/run-3e8clw7k.wandb +2021-07-13 12:52:32,296 DEBUG SenderThread:357371 [sender.py:send():179] send: header +2021-07-13 12:52:32,296 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: check_version +2021-07-13 12:52:32,336 DEBUG SenderThread:357371 [sender.py:send():179] send: run +2021-07-13 12:52:32,510 INFO SenderThread:357371 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files +2021-07-13 12:52:32,510 INFO SenderThread:357371 [sender.py:_start_run_threads():716] run started: 3e8clw7k with start time 1626180751 +2021-07-13 12:52:32,510 DEBUG SenderThread:357371 [sender.py:send():179] send: summary +2021-07-13 12:52:32,510 INFO SenderThread:357371 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 12:52:32,511 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 12:52:33,512 INFO Thread-8 :357371 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/wandb-summary.json +2021-07-13 12:52:33,640 DEBUG HandlerThread:357371 [meta.py:__init__():39] meta init +2021-07-13 12:52:33,640 DEBUG HandlerThread:357371 [meta.py:__init__():53] meta init done +2021-07-13 12:52:33,640 DEBUG HandlerThread:357371 [meta.py:probe():210] probe +2021-07-13 12:52:33,641 DEBUG HandlerThread:357371 [meta.py:_setup_git():200] setup git +2021-07-13 12:52:33,672 DEBUG HandlerThread:357371 [meta.py:_setup_git():207] setup git done +2021-07-13 12:52:33,673 DEBUG HandlerThread:357371 [meta.py:_save_pip():57] save pip +2021-07-13 12:52:33,673 DEBUG HandlerThread:357371 [meta.py:_save_pip():71] save pip done +2021-07-13 12:52:33,673 DEBUG HandlerThread:357371 [meta.py:probe():252] probe done +2021-07-13 12:52:33,676 DEBUG SenderThread:357371 [sender.py:send():179] send: files +2021-07-13 12:52:33,677 INFO SenderThread:357371 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 12:52:33,684 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:52:33,684 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:52:33,819 DEBUG SenderThread:357371 [sender.py:send():179] send: config +2021-07-13 12:52:33,819 DEBUG SenderThread:357371 [sender.py:send():179] send: config +2021-07-13 12:52:33,819 DEBUG SenderThread:357371 [sender.py:send():179] send: config +2021-07-13 12:52:34,121 INFO Thread-11 :357371 [upload_job.py:push():137] Uploaded file /tmp/tmpf08p42k1wandb/2dh5gbl6-wandb-metadata.json +2021-07-13 12:52:34,512 INFO Thread-8 :357371 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/wandb-metadata.json +2021-07-13 12:52:34,512 INFO Thread-8 :357371 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/output.log +2021-07-13 12:52:34,512 INFO Thread-8 :357371 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/requirements.txt +2021-07-13 12:52:48,517 INFO Thread-8 :357371 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/output.log +2021-07-13 12:52:48,821 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:52:48,822 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:52:50,518 INFO Thread-8 :357371 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/output.log +2021-07-13 12:53:01,734 DEBUG SenderThread:357371 [sender.py:send():179] send: stats +2021-07-13 12:53:03,523 INFO Thread-8 :357371 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/config.yaml +2021-07-13 12:53:03,954 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:53:03,954 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:53:06,524 INFO Thread-8 :357371 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/output.log +2021-07-13 12:53:19,100 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:53:19,100 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:53:31,817 DEBUG SenderThread:357371 [sender.py:send():179] send: stats +2021-07-13 12:53:34,247 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:53:34,247 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:53:49,380 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:53:49,380 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:54:01,895 DEBUG SenderThread:357371 [sender.py:send():179] send: stats +2021-07-13 12:54:04,520 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:54:04,520 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:54:19,658 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:54:19,658 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:54:31,974 DEBUG SenderThread:357371 [sender.py:send():179] send: stats +2021-07-13 12:54:34,800 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:54:34,800 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:54:49,937 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 12:54:49,938 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: stop_status +2021-07-13 12:54:54,269 DEBUG SenderThread:357371 [sender.py:send():179] send: telemetry +2021-07-13 12:54:54,269 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:54,269 DEBUG SenderThread:357371 [sender.py:send():179] send: exit +2021-07-13 12:54:54,270 INFO SenderThread:357371 [sender.py:send_exit():287] handling exit code: 1 +2021-07-13 12:54:54,270 INFO SenderThread:357371 [sender.py:send_exit():295] send defer +2021-07-13 12:54:54,270 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:54,271 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:54,271 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-13 12:54:54,271 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:54,271 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-13 12:54:54,271 INFO SenderThread:357371 [sender.py:transition_state():308] send defer: 1 +2021-07-13 12:54:54,271 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:54,271 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-13 12:54:54,333 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:54,333 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-13 12:54:54,333 INFO SenderThread:357371 [sender.py:transition_state():308] send defer: 2 +2021-07-13 12:54:54,333 DEBUG SenderThread:357371 [sender.py:send():179] send: stats +2021-07-13 12:54:54,334 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:54,334 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-13 12:54:54,334 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:54,334 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-13 12:54:54,334 INFO SenderThread:357371 [sender.py:transition_state():308] send defer: 3 +2021-07-13 12:54:54,334 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:54,334 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-13 12:54:54,334 DEBUG SenderThread:357371 [sender.py:send():179] send: summary +2021-07-13 12:54:54,335 INFO SenderThread:357371 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 12:54:54,335 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:54,335 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-13 12:54:54,335 INFO SenderThread:357371 [sender.py:transition_state():308] send defer: 4 +2021-07-13 12:54:54,335 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:54,335 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-13 12:54:54,336 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:54,336 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-13 12:54:54,372 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:54,508 INFO SenderThread:357371 [sender.py:transition_state():308] send defer: 5 +2021-07-13 12:54:54,508 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:54,509 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:54,509 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-13 12:54:54,509 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:54,509 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-13 12:54:54,509 INFO SenderThread:357371 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 12:54:54,551 INFO SenderThread:357371 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/wandb-summary.json +2021-07-13 12:54:54,551 INFO SenderThread:357371 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/config.yaml +2021-07-13 12:54:54,551 INFO SenderThread:357371 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/output.log +2021-07-13 12:54:54,552 INFO SenderThread:357371 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files +2021-07-13 12:54:54,552 INFO SenderThread:357371 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/requirements.txt requirements.txt +2021-07-13 12:54:54,552 INFO SenderThread:357371 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/output.log output.log +2021-07-13 12:54:54,552 INFO SenderThread:357371 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/wandb-metadata.json wandb-metadata.json +2021-07-13 12:54:54,552 INFO SenderThread:357371 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/config.yaml config.yaml +2021-07-13 12:54:54,552 INFO SenderThread:357371 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/wandb-summary.json wandb-summary.json +2021-07-13 12:54:54,552 INFO SenderThread:357371 [sender.py:transition_state():308] send defer: 6 +2021-07-13 12:54:54,553 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:54,553 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-13 12:54:54,556 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:54,556 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-13 12:54:54,556 INFO SenderThread:357371 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 12:54:54,610 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:54,611 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:54,713 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:54,713 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:54,815 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:54,815 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:54,917 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:54,917 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:55,002 INFO Thread-14 :357371 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/config.yaml +2021-07-13 12:54:55,019 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:55,020 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:55,024 INFO Thread-12 :357371 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/requirements.txt +2021-07-13 12:54:55,026 INFO Thread-13 :357371 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/output.log +2021-07-13 12:54:55,027 INFO Thread-15 :357371 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/files/wandb-summary.json +2021-07-13 12:54:55,122 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:55,122 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:55,224 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:55,224 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:55,228 INFO Thread-7 :357371 [sender.py:transition_state():308] send defer: 7 +2021-07-13 12:54:55,229 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:55,229 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-13 12:54:55,229 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:55,229 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-13 12:54:55,326 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:55,508 INFO SenderThread:357371 [sender.py:transition_state():308] send defer: 8 +2021-07-13 12:54:55,509 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:55,509 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:55,509 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-13 12:54:55,509 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:55,509 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-13 12:54:55,510 INFO SenderThread:357371 [sender.py:transition_state():308] send defer: 9 +2021-07-13 12:54:55,510 DEBUG SenderThread:357371 [sender.py:send():179] send: final +2021-07-13 12:54:55,510 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: defer +2021-07-13 12:54:55,510 DEBUG SenderThread:357371 [sender.py:send():179] send: footer +2021-07-13 12:54:55,510 INFO HandlerThread:357371 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-13 12:54:55,511 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: defer +2021-07-13 12:54:55,511 INFO SenderThread:357371 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-13 12:54:55,611 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 12:54:55,611 DEBUG SenderThread:357371 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 12:54:55,611 INFO SenderThread:357371 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 12:54:55,612 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: get_summary +2021-07-13 12:54:55,613 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-13 12:54:55,614 DEBUG HandlerThread:357371 [handler.py:handle_request():124] handle_request: shutdown +2021-07-13 12:54:55,614 INFO HandlerThread:357371 [handler.py:finish():638] shutting down handler +2021-07-13 12:54:56,510 INFO WriterThread:357371 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/run-3e8clw7k.wandb +2021-07-13 12:54:56,611 INFO SenderThread:357371 [sender.py:finish():945] shutting down sender +2021-07-13 12:54:56,611 INFO SenderThread:357371 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 12:54:56,611 INFO SenderThread:357371 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 12:54:56,614 INFO MainThread:357371 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_125231-3e8clw7k/logs/debug.log b/wandb/run-20210713_125231-3e8clw7k/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ca816c7693e6173058cd3c722165f2dd72552bf4 --- /dev/null +++ b/wandb/run-20210713_125231-3e8clw7k/logs/debug.log @@ -0,0 +1,119 @@ +2021-07-13 12:52:31,557 INFO MainThread:355855 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 12:52:31,557 INFO MainThread:355855 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 12:52:31,557 INFO MainThread:355855 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/logs/debug.log +2021-07-13 12:52:31,557 INFO MainThread:355855 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_125231-3e8clw7k/logs/debug-internal.log +2021-07-13 12:52:31,558 INFO MainThread:355855 [wandb_init.py:init():370] calling init triggers +2021-07-13 12:52:31,558 INFO MainThread:355855 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 12:52:31,558 INFO MainThread:355855 [wandb_init.py:init():419] starting backend +2021-07-13 12:52:31,558 INFO MainThread:355855 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 12:52:31,613 INFO MainThread:355855 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 12:52:31,689 INFO MainThread:355855 [backend.py:ensure_launched():139] started backend process with pid: 357371 +2021-07-13 12:52:31,691 INFO MainThread:355855 [wandb_init.py:init():424] backend started and connected +2021-07-13 12:52:31,694 INFO MainThread:355855 [wandb_init.py:init():472] updated telemetry +2021-07-13 12:52:31,695 INFO MainThread:355855 [wandb_init.py:init():491] communicating current version +2021-07-13 12:52:32,334 INFO MainThread:355855 [wandb_init.py:init():496] got version response +2021-07-13 12:52:32,334 INFO MainThread:355855 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 12:52:32,510 INFO MainThread:355855 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 12:52:33,680 INFO MainThread:355855 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 12:52:33,681 INFO MainThread:355855 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 12:52:33,681 INFO MainThread:355855 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 12:52:33,683 INFO MainThread:355855 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 12:52:33,683 INFO MainThread:355855 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 12:52:33,689 INFO MainThread:355855 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_12-51-41_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 12:52:33,691 INFO MainThread:355855 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 12:52:33,693 INFO MainThread:355855 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 12:54:52,164 INFO MainThread:355855 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-13 12:54:52,166 INFO MainThread:355855 [wandb_run.py:_restore():1565] restore +2021-07-13 12:54:54,271 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 1388 +} + +2021-07-13 12:54:54,509 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 1388 +} + +2021-07-13 12:54:54,611 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 35507 +} + +2021-07-13 12:54:54,714 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 3000 + total_bytes: 35507 +} + +2021-07-13 12:54:54,816 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 12:54:54,918 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 12:54:55,020 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 12:54:55,123 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 12:54:55,225 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 12:54:55,509 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 12:54:55,612 INFO MainThread:355855 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 12:54:56,938 INFO MainThread:355855 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210713_125231-3e8clw7k/run-3e8clw7k.wandb b/wandb/run-20210713_125231-3e8clw7k/run-3e8clw7k.wandb new file mode 100644 index 0000000000000000000000000000000000000000..353c5e8f9054ce8fefdbf90d02e0c6c40deec32f Binary files /dev/null and b/wandb/run-20210713_125231-3e8clw7k/run-3e8clw7k.wandb differ diff --git a/wandb/run-20210713_130641-2gq51wyk/files/config.yaml b/wandb/run-20210713_130641-2gq51wyk/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..823b6d9e7465f4024b075f3efc34bba5305c38d0 --- /dev/null +++ b/wandb/run-20210713_130641-2gq51wyk/files/config.yaml @@ -0,0 +1,307 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 8 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_13-05-54_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_130641-2gq51wyk/files/output.log b/wandb/run-20210713_130641-2gq51wyk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b2806aa813f756e6da787e8a9ab803e1f3a8eee5 --- /dev/null +++ b/wandb/run-20210713_130641-2gq51wyk/files/output.log @@ -0,0 +1,322 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1647, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 624, in xla_pmap_impl + compiled_fun, fingerprint = parallel_callable(fun, backend, axis_name, axis_size, + File "/home/dat/pino/lib/python3.8/site-packages/jax/linear_util.py", line 262, in memoized_fun + ans = call(fun, *args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 899, in parallel_callable + compiled = xla.backend_compile(backend, built, compile_options) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile + return backend.compile(built_c, compile_options=options) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Ran out of memory in memory space hbm. Used 77.35G of 15.48G hbm. Exceeded hbm capacity by 61.87G. +Total hbm usage >= 77.87G: + reserved 530.00M + program 77.35G + arguments 0B +Output size 0B; shares 0B with arguments. +Program hbm requirement 77.35G: + global 692.0K + scoped 253.0K + HLO temp 77.35G (97.9% utilization: Unpadded (75.71G) Padded (77.34G), 0.0% fragmentation (10.14M)) + Largest program allocations in hbm: + 1. Size: 12.29G + Operator: op_type="dot_general" op_name="pmap(train_step)/dot_general[ dimension_numbers=(((2,), (0,)), ((), ()))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/pino/lib/python3.8/site-packages/flax/linen/linear.py" source_line=175 + Shape: f32[16,4096,50358]{1,2,0:T(8,128)} + Unpadded size: 12.29G + Extra memory due to padding: 512.0K (1.0x expansion) + XLA label: %fusion.1737.remat7 = f32[16,4096,50358]{1,2,0:T(8,128)} fusion(f32[50358]{0:T(1024)} %get-tuple-element.23314, f32[768,50358,1]{0,1,2:T(8,128)} %bitcast.5719, f32[768]{0:T(1024)} %get-tuple-element.23315, f32[768]{0:T(1024)} %get-tuple-element.23316, f32[... + Allocation type: HLO temp + ========================== + 2. Size: 2.62G + Operator: op_type="mul" op_name="pmap(train_step)/mul" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} + Unpadded size: 2.62G + XLA label: %fusion.1803 = f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} fusion(f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} %fusion.1984, f32[16,12,28,128]{3,2,1,0:T(8,128)} %negate.34, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:T(8,128)} ... + Allocation type: HLO temp + ========================== + 3. Size: 2.62G + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(transpose(jvp(_einsum)))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=626 + Shape: f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} + Unpadded size: 2.62G + XLA label: %fusion.1984 = f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} fusion(f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} %fusion.1986, pred[]{:T(1024)} %fusion.9668, bf16[16,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2490.remat_uncompressed.remat, f32[16,12,32,128,... + Allocation type: HLO temp + ========================== + 4. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1816.remat2 = bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2418, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9170, f32[16,12,28,128,128]{3,4,2,1,0:T(8,128)} %fusion.3004.remat3, f32[16,12,28,12... + Allocation type: HLO temp + ========================== + 5. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1805.remat6 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:... + Allocation type: HLO temp + ========================== + 6. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1805.remat6 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:... + Allocation type: HLO temp + ========================== + 7. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13419 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2417, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9169, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 8. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13420 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2416, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9168, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 9. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13421 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2415, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9167, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 10. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13422 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2414, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9166, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 11. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13423 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2413, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9165, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 12. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13424 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2412, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9164, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 13. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13425 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2411, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9163, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 14. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13426 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2410, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9162, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 15. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13427 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2409, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9161, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 16. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13428 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2408, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9160, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 17. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11600 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25240, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28186.remat_uncompressed, f32[... + Allocation type: HLO temp + ========================== + 18. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=584 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.2061.remat6 = f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} fusion(f32[16,28,128,384]{2,3,1,0:T(8,128)} %get-tuple-element.20979, bf16[16,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2489.remat_uncompressed, f32[16,12,32,128,64]{3,2,4,1,0:T(8,128)} %... + Allocation type: HLO temp + ========================== + 19. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11584 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25199, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28115, f32[16,12,32,128,64]{3,... + Allocation type: HLO temp + ========================== + 20. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11592 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25220, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28151.remat_uncompressed, f32[... + Allocation type: HLO temp + ========================== +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 725, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile + return backend.compile(built_c, compile_options=options) +RuntimeError: Resource exhausted: Ran out of memory in memory space hbm. Used 77.35G of 15.48G hbm. Exceeded hbm capacity by 61.87G. +Total hbm usage >= 77.87G: + reserved 530.00M + program 77.35G + arguments 0B +Output size 0B; shares 0B with arguments. +Program hbm requirement 77.35G: + global 692.0K + scoped 253.0K + HLO temp 77.35G (97.9% utilization: Unpadded (75.71G) Padded (77.34G), 0.0% fragmentation (10.14M)) + Largest program allocations in hbm: + 1. Size: 12.29G + Operator: op_type="dot_general" op_name="pmap(train_step)/dot_general[ dimension_numbers=(((2,), (0,)), ((), ()))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/pino/lib/python3.8/site-packages/flax/linen/linear.py" source_line=175 + Shape: f32[16,4096,50358]{1,2,0:T(8,128)} + Unpadded size: 12.29G + Extra memory due to padding: 512.0K (1.0x expansion) + XLA label: %fusion.1737.remat7 = f32[16,4096,50358]{1,2,0:T(8,128)} fusion(f32[50358]{0:T(1024)} %get-tuple-element.23314, f32[768,50358,1]{0,1,2:T(8,128)} %bitcast.5719, f32[768]{0:T(1024)} %get-tuple-element.23315, f32[768]{0:T(1024)} %get-tuple-element.23316, f32[... + Allocation type: HLO temp + ========================== + 2. Size: 2.62G + Operator: op_type="mul" op_name="pmap(train_step)/mul" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} + Unpadded size: 2.62G + XLA label: %fusion.1803 = f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} fusion(f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} %fusion.1984, f32[16,12,28,128]{3,2,1,0:T(8,128)} %negate.34, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:T(8,128)} ... + Allocation type: HLO temp + ========================== + 3. Size: 2.62G + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(transpose(jvp(_einsum)))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=626 + Shape: f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} + Unpadded size: 2.62G + XLA label: %fusion.1984 = f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} fusion(f32[16,12,28,128,1024]{3,4,2,1,0:T(8,128)} %fusion.1986, pred[]{:T(1024)} %fusion.9668, bf16[16,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2490.remat_uncompressed.remat, f32[16,12,32,128,... + Allocation type: HLO temp + ========================== + 4. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1816.remat2 = bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2418, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9170, f32[16,12,28,128,128]{3,4,2,1,0:T(8,128)} %fusion.3004.remat3, f32[16,12,28,12... + Allocation type: HLO temp + ========================== + 5. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1805.remat6 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:... + Allocation type: HLO temp + ========================== + 6. Size: 1.31G + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.1805.remat6 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2407, f32[16,12,28,128]{3,2,1,0:... + Allocation type: HLO temp + ========================== + 7. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13419 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2417, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9169, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 8. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13420 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2416, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9168, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 9. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13421 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2415, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9167, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 10. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13422 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2414, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9166, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 11. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13423 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2413, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9165, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 12. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13424 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2412, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9164, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 13. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13425 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2411, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9163, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 14. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13426 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2410, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9162, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 15. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13427 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2409, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9161, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 16. Size: 1.31G + Shape: bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 1.31G + XLA label: %fusion.13428 = (bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[16,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.2408, f32[16,12,28,128]{3,2,1,0:T(8,128)} %fusion.9160, f32[16,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 17. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11600 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25240, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28186.remat_uncompressed, f32[... + Allocation type: HLO temp + ========================== + 18. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=584 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.2061.remat6 = f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} fusion(f32[16,28,128,384]{2,3,1,0:T(8,128)} %get-tuple-element.20979, bf16[16,12,28,384,64]{3,2,4,1,0:T(8,128)(2,1)} %fusion.2489.remat_uncompressed, f32[16,12,32,128,64]{3,2,4,1,0:T(8,128)} %... + Allocation type: HLO temp + ========================== + 19. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11584 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25199, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28115, f32[16,12,32,128,64]{3,... + Allocation type: HLO temp + ========================== + 20. Size: 1008.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 1008.00M + XLA label: %fusion.11592 = (f32[16,12,28,128]{3,2,1,0:T(8,128)}, f32[16,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[16,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.25220, bf16[16,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28151.remat_uncompressed, f32[... + Allocation type: HLO temp + ========================== \ No newline at end of file diff --git a/wandb/run-20210713_130641-2gq51wyk/files/requirements.txt b/wandb/run-20210713_130641-2gq51wyk/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_130641-2gq51wyk/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_130641-2gq51wyk/files/wandb-metadata.json b/wandb/run-20210713_130641-2gq51wyk/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0085b3eae8e6a20c6c6b74743f063b9cf4835ae0 --- /dev/null +++ b/wandb/run-20210713_130641-2gq51wyk/files/wandb-metadata.json @@ -0,0 +1,45 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T13:06:43.367075", + "startedAt": "2021-07-13T13:06:41.423626", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=8", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_130641-2gq51wyk/files/wandb-summary.json b/wandb/run-20210713_130641-2gq51wyk/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210713_130641-2gq51wyk/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210713_130641-2gq51wyk/logs/debug-internal.log b/wandb/run-20210713_130641-2gq51wyk/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..346e1ee1b3c7e0e101eddbc8f6d9404a4d5d30b9 --- /dev/null +++ b/wandb/run-20210713_130641-2gq51wyk/logs/debug-internal.log @@ -0,0 +1,161 @@ +2021-07-13 13:06:42,101 INFO MainThread:359333 [internal.py:wandb_internal():88] W&B internal server running at pid: 359333, started at: 2021-07-13 13:06:42.101080 +2021-07-13 13:06:42,103 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 13:06:42,103 INFO WriterThread:359333 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/run-2gq51wyk.wandb +2021-07-13 13:06:42,104 DEBUG SenderThread:359333 [sender.py:send():179] send: header +2021-07-13 13:06:42,104 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: check_version +2021-07-13 13:06:42,140 DEBUG SenderThread:359333 [sender.py:send():179] send: run +2021-07-13 13:06:42,313 INFO SenderThread:359333 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files +2021-07-13 13:06:42,313 INFO SenderThread:359333 [sender.py:_start_run_threads():716] run started: 2gq51wyk with start time 1626181601 +2021-07-13 13:06:42,313 DEBUG SenderThread:359333 [sender.py:send():179] send: summary +2021-07-13 13:06:42,314 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 13:06:42,315 INFO SenderThread:359333 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 13:06:43,318 INFO Thread-8 :359333 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/wandb-summary.json +2021-07-13 13:06:43,366 DEBUG HandlerThread:359333 [meta.py:__init__():39] meta init +2021-07-13 13:06:43,366 DEBUG HandlerThread:359333 [meta.py:__init__():53] meta init done +2021-07-13 13:06:43,367 DEBUG HandlerThread:359333 [meta.py:probe():210] probe +2021-07-13 13:06:43,368 DEBUG HandlerThread:359333 [meta.py:_setup_git():200] setup git +2021-07-13 13:06:43,393 DEBUG HandlerThread:359333 [meta.py:_setup_git():207] setup git done +2021-07-13 13:06:43,394 DEBUG HandlerThread:359333 [meta.py:_save_pip():57] save pip +2021-07-13 13:06:43,394 DEBUG HandlerThread:359333 [meta.py:_save_pip():71] save pip done +2021-07-13 13:06:43,394 DEBUG HandlerThread:359333 [meta.py:probe():252] probe done +2021-07-13 13:06:43,397 DEBUG SenderThread:359333 [sender.py:send():179] send: files +2021-07-13 13:06:43,397 INFO SenderThread:359333 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 13:06:43,405 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:06:43,406 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:06:43,532 DEBUG SenderThread:359333 [sender.py:send():179] send: config +2021-07-13 13:06:43,533 DEBUG SenderThread:359333 [sender.py:send():179] send: config +2021-07-13 13:06:43,533 DEBUG SenderThread:359333 [sender.py:send():179] send: config +2021-07-13 13:06:43,840 INFO Thread-11 :359333 [upload_job.py:push():137] Uploaded file /tmp/tmpvp99h86hwandb/2nevcblm-wandb-metadata.json +2021-07-13 13:06:44,317 INFO Thread-8 :359333 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/output.log +2021-07-13 13:06:44,318 INFO Thread-8 :359333 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/requirements.txt +2021-07-13 13:06:44,318 INFO Thread-8 :359333 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/wandb-metadata.json +2021-07-13 13:06:58,323 INFO Thread-8 :359333 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/output.log +2021-07-13 13:06:58,535 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:06:58,535 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:07:00,324 INFO Thread-8 :359333 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/output.log +2021-07-13 13:07:11,447 DEBUG SenderThread:359333 [sender.py:send():179] send: stats +2021-07-13 13:07:13,330 INFO Thread-8 :359333 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/config.yaml +2021-07-13 13:07:13,669 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:07:13,670 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:07:16,332 INFO Thread-8 :359333 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/output.log +2021-07-13 13:07:28,816 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:07:28,816 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:07:41,527 DEBUG SenderThread:359333 [sender.py:send():179] send: stats +2021-07-13 13:07:43,967 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:07:43,968 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:07:59,098 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:07:59,099 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:08:11,610 DEBUG SenderThread:359333 [sender.py:send():179] send: stats +2021-07-13 13:08:14,230 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:08:14,231 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:08:29,362 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:08:29,362 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:08:41,690 DEBUG SenderThread:359333 [sender.py:send():179] send: stats +2021-07-13 13:08:44,829 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:08:44,830 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:08:58,373 INFO Thread-8 :359333 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/output.log +2021-07-13 13:08:58,754 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:08:58,754 DEBUG SenderThread:359333 [sender.py:send():179] send: telemetry +2021-07-13 13:08:58,754 DEBUG SenderThread:359333 [sender.py:send():179] send: exit +2021-07-13 13:08:58,754 INFO SenderThread:359333 [sender.py:send_exit():287] handling exit code: 1 +2021-07-13 13:08:58,755 INFO SenderThread:359333 [sender.py:send_exit():295] send defer +2021-07-13 13:08:58,755 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:08:58,755 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:08:58,755 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-13 13:08:58,755 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:08:58,756 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-13 13:08:58,756 INFO SenderThread:359333 [sender.py:transition_state():308] send defer: 1 +2021-07-13 13:08:58,756 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:08:58,756 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-13 13:08:58,833 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:08:58,833 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-13 13:08:58,833 INFO SenderThread:359333 [sender.py:transition_state():308] send defer: 2 +2021-07-13 13:08:58,833 DEBUG SenderThread:359333 [sender.py:send():179] send: stats +2021-07-13 13:08:58,834 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:08:58,834 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-13 13:08:58,834 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:08:58,834 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-13 13:08:58,834 INFO SenderThread:359333 [sender.py:transition_state():308] send defer: 3 +2021-07-13 13:08:58,834 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:08:58,834 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-13 13:08:58,834 DEBUG SenderThread:359333 [sender.py:send():179] send: summary +2021-07-13 13:08:58,835 INFO SenderThread:359333 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 13:08:58,835 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:08:58,835 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-13 13:08:58,835 INFO SenderThread:359333 [sender.py:transition_state():308] send defer: 4 +2021-07-13 13:08:58,835 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:08:58,835 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-13 13:08:58,835 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:08:58,835 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-13 13:08:58,857 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:08:59,373 INFO Thread-8 :359333 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/wandb-summary.json +2021-07-13 13:08:59,373 INFO Thread-8 :359333 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/output.log +2021-07-13 13:08:59,806 INFO SenderThread:359333 [sender.py:transition_state():308] send defer: 5 +2021-07-13 13:08:59,806 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:08:59,807 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:08:59,807 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-13 13:08:59,807 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:08:59,807 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-13 13:08:59,807 INFO SenderThread:359333 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 13:08:59,908 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:00,374 INFO Thread-8 :359333 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/config.yaml +2021-07-13 13:09:00,374 INFO SenderThread:359333 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files +2021-07-13 13:09:00,374 INFO SenderThread:359333 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/requirements.txt requirements.txt +2021-07-13 13:09:00,375 INFO SenderThread:359333 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/output.log output.log +2021-07-13 13:09:00,375 INFO SenderThread:359333 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/wandb-metadata.json wandb-metadata.json +2021-07-13 13:09:00,375 INFO SenderThread:359333 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/config.yaml config.yaml +2021-07-13 13:09:00,375 INFO SenderThread:359333 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/wandb-summary.json wandb-summary.json +2021-07-13 13:09:00,375 INFO SenderThread:359333 [sender.py:transition_state():308] send defer: 6 +2021-07-13 13:09:00,375 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:00,378 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:09:00,379 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-13 13:09:00,379 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:09:00,379 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-13 13:09:00,379 INFO SenderThread:359333 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 13:09:00,479 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:00,480 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:00,581 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:00,582 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:00,683 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:00,683 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:00,785 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:00,785 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:00,837 INFO Thread-14 :359333 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/config.yaml +2021-07-13 13:09:00,842 INFO Thread-12 :359333 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/requirements.txt +2021-07-13 13:09:00,849 INFO Thread-13 :359333 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/output.log +2021-07-13 13:09:00,872 INFO Thread-15 :359333 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/files/wandb-summary.json +2021-07-13 13:09:00,887 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:00,887 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:00,989 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:00,989 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:01,072 INFO Thread-7 :359333 [sender.py:transition_state():308] send defer: 7 +2021-07-13 13:09:01,072 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:09:01,072 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-13 13:09:01,073 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:09:01,073 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-13 13:09:01,091 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:01,360 INFO SenderThread:359333 [sender.py:transition_state():308] send defer: 8 +2021-07-13 13:09:01,360 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:01,361 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:09:01,361 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-13 13:09:01,361 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:09:01,361 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-13 13:09:01,361 INFO SenderThread:359333 [sender.py:transition_state():308] send defer: 9 +2021-07-13 13:09:01,362 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:09:01,362 INFO HandlerThread:359333 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-13 13:09:01,362 DEBUG SenderThread:359333 [sender.py:send():179] send: final +2021-07-13 13:09:01,363 DEBUG SenderThread:359333 [sender.py:send():179] send: footer +2021-07-13 13:09:01,363 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: defer +2021-07-13 13:09:01,363 INFO SenderThread:359333 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-13 13:09:01,462 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:09:01,462 DEBUG SenderThread:359333 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:09:01,462 INFO SenderThread:359333 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 13:09:01,463 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: get_summary +2021-07-13 13:09:01,464 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-13 13:09:01,465 DEBUG HandlerThread:359333 [handler.py:handle_request():124] handle_request: shutdown +2021-07-13 13:09:01,465 INFO HandlerThread:359333 [handler.py:finish():638] shutting down handler +2021-07-13 13:09:02,363 INFO WriterThread:359333 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/run-2gq51wyk.wandb +2021-07-13 13:09:02,463 INFO SenderThread:359333 [sender.py:finish():945] shutting down sender +2021-07-13 13:09:02,463 INFO SenderThread:359333 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 13:09:02,463 INFO SenderThread:359333 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 13:09:02,465 INFO MainThread:359333 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_130641-2gq51wyk/logs/debug.log b/wandb/run-20210713_130641-2gq51wyk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d7cabb282097d60c21c2a1ddbed4c1ad96ff1831 --- /dev/null +++ b/wandb/run-20210713_130641-2gq51wyk/logs/debug.log @@ -0,0 +1,119 @@ +2021-07-13 13:06:41,425 INFO MainThread:357816 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 13:06:41,425 INFO MainThread:357816 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 13:06:41,425 INFO MainThread:357816 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/logs/debug.log +2021-07-13 13:06:41,425 INFO MainThread:357816 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_130641-2gq51wyk/logs/debug-internal.log +2021-07-13 13:06:41,425 INFO MainThread:357816 [wandb_init.py:init():370] calling init triggers +2021-07-13 13:06:41,425 INFO MainThread:357816 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 13:06:41,425 INFO MainThread:357816 [wandb_init.py:init():419] starting backend +2021-07-13 13:06:41,426 INFO MainThread:357816 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 13:06:41,473 INFO MainThread:357816 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 13:06:41,521 INFO MainThread:357816 [backend.py:ensure_launched():139] started backend process with pid: 359333 +2021-07-13 13:06:41,523 INFO MainThread:357816 [wandb_init.py:init():424] backend started and connected +2021-07-13 13:06:41,526 INFO MainThread:357816 [wandb_init.py:init():472] updated telemetry +2021-07-13 13:06:41,526 INFO MainThread:357816 [wandb_init.py:init():491] communicating current version +2021-07-13 13:06:42,139 INFO MainThread:357816 [wandb_init.py:init():496] got version response +2021-07-13 13:06:42,139 INFO MainThread:357816 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 13:06:42,313 INFO MainThread:357816 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 13:06:43,405 INFO MainThread:357816 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 13:06:43,405 INFO MainThread:357816 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 13:06:43,406 INFO MainThread:357816 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 13:06:43,408 INFO MainThread:357816 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 13:06:43,408 INFO MainThread:357816 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 13:06:43,414 INFO MainThread:357816 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_13-05-54_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 13:06:43,416 INFO MainThread:357816 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 13:06:43,417 INFO MainThread:357816 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 13:08:56,473 INFO MainThread:357816 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-13 13:08:56,475 INFO MainThread:357816 [wandb_run.py:_restore():1565] restore +2021-07-13 13:08:58,755 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 1388 +} + +2021-07-13 13:08:59,807 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 1388 +} + +2021-07-13 13:09:00,378 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 3 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 30901 +} + +2021-07-13 13:09:00,480 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 35507 +} + +2021-07-13 13:09:00,582 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 13:09:00,684 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 13:09:00,786 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 13:09:00,888 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 13:09:00,990 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 13:09:01,361 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 13:09:01,463 INFO MainThread:357816 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 35507 + total_bytes: 35507 +} + +2021-07-13 13:09:02,705 INFO MainThread:357816 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210713_130641-2gq51wyk/run-2gq51wyk.wandb b/wandb/run-20210713_130641-2gq51wyk/run-2gq51wyk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a2e517032f821bf0e7a6b26fdb6e19e1e8851432 Binary files /dev/null and b/wandb/run-20210713_130641-2gq51wyk/run-2gq51wyk.wandb differ diff --git a/wandb/run-20210713_131130-3eewirqy/files/config.yaml b/wandb/run-20210713_131130-3eewirqy/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d382c7ab62e8b9b622a782639f131cc74818f64 --- /dev/null +++ b/wandb/run-20210713_131130-3eewirqy/files/config.yaml @@ -0,0 +1,307 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: false +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 8 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_13-10-42_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_131130-3eewirqy/files/output.log b/wandb/run-20210713_131130-3eewirqy/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..bd757893e173479e9e1f631a2db2172fc3d9a11f --- /dev/null +++ b/wandb/run-20210713_131130-3eewirqy/files/output.log @@ -0,0 +1,37 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1647, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 637, in xla_pmap_impl + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Attempting to reserve 12.60G at the bottom of memory. That was not possible. There are 12.15G free, 0B reserved, and 12.13G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 725, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +RuntimeError: Resource exhausted: Attempting to reserve 12.60G at the bottom of memory. That was not possible. There are 12.15G free, 0B reserved, and 12.13G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). \ No newline at end of file diff --git a/wandb/run-20210713_131130-3eewirqy/files/requirements.txt b/wandb/run-20210713_131130-3eewirqy/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_131130-3eewirqy/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_131130-3eewirqy/files/wandb-metadata.json b/wandb/run-20210713_131130-3eewirqy/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2a2fa718cd708357b48416f3323e1bc4f8c6ca78 --- /dev/null +++ b/wandb/run-20210713_131130-3eewirqy/files/wandb-metadata.json @@ -0,0 +1,44 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T13:11:32.857018", + "startedAt": "2021-07-13T13:11:30.654921", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=8" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_131130-3eewirqy/files/wandb-summary.json b/wandb/run-20210713_131130-3eewirqy/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210713_131130-3eewirqy/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210713_131130-3eewirqy/logs/debug-internal.log b/wandb/run-20210713_131130-3eewirqy/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5598c5060eaef8e8837984f7d3502c8b02ef4306 --- /dev/null +++ b/wandb/run-20210713_131130-3eewirqy/logs/debug-internal.log @@ -0,0 +1,170 @@ +2021-07-13 13:11:31,441 INFO MainThread:361603 [internal.py:wandb_internal():88] W&B internal server running at pid: 361603, started at: 2021-07-13 13:11:31.440779 +2021-07-13 13:11:31,443 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 13:11:31,443 INFO WriterThread:361603 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/run-3eewirqy.wandb +2021-07-13 13:11:31,444 DEBUG SenderThread:361603 [sender.py:send():179] send: header +2021-07-13 13:11:31,444 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: check_version +2021-07-13 13:11:31,480 DEBUG SenderThread:361603 [sender.py:send():179] send: run +2021-07-13 13:11:31,724 INFO SenderThread:361603 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files +2021-07-13 13:11:31,724 INFO SenderThread:361603 [sender.py:_start_run_threads():716] run started: 3eewirqy with start time 1626181890 +2021-07-13 13:11:31,724 DEBUG SenderThread:361603 [sender.py:send():179] send: summary +2021-07-13 13:11:31,724 INFO SenderThread:361603 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 13:11:31,725 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 13:11:32,727 INFO Thread-8 :361603 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/wandb-summary.json +2021-07-13 13:11:32,856 DEBUG HandlerThread:361603 [meta.py:__init__():39] meta init +2021-07-13 13:11:32,856 DEBUG HandlerThread:361603 [meta.py:__init__():53] meta init done +2021-07-13 13:11:32,856 DEBUG HandlerThread:361603 [meta.py:probe():210] probe +2021-07-13 13:11:32,858 DEBUG HandlerThread:361603 [meta.py:_setup_git():200] setup git +2021-07-13 13:11:32,888 DEBUG HandlerThread:361603 [meta.py:_setup_git():207] setup git done +2021-07-13 13:11:32,888 DEBUG HandlerThread:361603 [meta.py:_save_pip():57] save pip +2021-07-13 13:11:32,888 DEBUG HandlerThread:361603 [meta.py:_save_pip():71] save pip done +2021-07-13 13:11:32,888 DEBUG HandlerThread:361603 [meta.py:probe():252] probe done +2021-07-13 13:11:32,891 DEBUG SenderThread:361603 [sender.py:send():179] send: files +2021-07-13 13:11:32,891 INFO SenderThread:361603 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 13:11:32,897 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:11:32,897 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:11:33,027 DEBUG SenderThread:361603 [sender.py:send():179] send: config +2021-07-13 13:11:33,027 DEBUG SenderThread:361603 [sender.py:send():179] send: config +2021-07-13 13:11:33,027 DEBUG SenderThread:361603 [sender.py:send():179] send: config +2021-07-13 13:11:33,332 INFO Thread-11 :361603 [upload_job.py:push():137] Uploaded file /tmp/tmpvsijxkjvwandb/1o2iztmz-wandb-metadata.json +2021-07-13 13:11:33,726 INFO Thread-8 :361603 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/requirements.txt +2021-07-13 13:11:33,726 INFO Thread-8 :361603 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/wandb-metadata.json +2021-07-13 13:11:33,726 INFO Thread-8 :361603 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/output.log +2021-07-13 13:11:48,029 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:11:48,029 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:11:49,731 INFO Thread-8 :361603 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/output.log +2021-07-13 13:12:00,934 DEBUG SenderThread:361603 [sender.py:send():179] send: stats +2021-07-13 13:12:02,735 INFO Thread-8 :361603 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/config.yaml +2021-07-13 13:12:03,162 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:12:03,163 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:12:18,294 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:12:18,294 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:12:31,014 DEBUG SenderThread:361603 [sender.py:send():179] send: stats +2021-07-13 13:12:33,426 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:12:33,426 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:12:48,558 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:12:48,559 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:13:01,089 DEBUG SenderThread:361603 [sender.py:send():179] send: stats +2021-07-13 13:13:03,691 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:13:03,691 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:13:18,832 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:13:18,832 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:13:21,761 INFO Thread-8 :361603 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/output.log +2021-07-13 13:13:31,163 DEBUG SenderThread:361603 [sender.py:send():179] send: stats +2021-07-13 13:13:33,987 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:13:33,987 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:13:49,124 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:13:49,124 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:14:01,237 DEBUG SenderThread:361603 [sender.py:send():179] send: stats +2021-07-13 13:14:04,256 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:14:04,256 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:14:19,385 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:14:19,386 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:14:31,314 DEBUG SenderThread:361603 [sender.py:send():179] send: stats +2021-07-13 13:14:34,523 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:14:34,523 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:14:49,653 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:14:49,654 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:15:01,389 DEBUG SenderThread:361603 [sender.py:send():179] send: stats +2021-07-13 13:15:05,548 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:05,549 DEBUG SenderThread:361603 [sender.py:send():179] send: telemetry +2021-07-13 13:15:05,549 DEBUG SenderThread:361603 [sender.py:send():179] send: exit +2021-07-13 13:15:05,549 INFO SenderThread:361603 [sender.py:send_exit():287] handling exit code: 1 +2021-07-13 13:15:05,550 INFO SenderThread:361603 [sender.py:send_exit():295] send defer +2021-07-13 13:15:05,550 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:05,550 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:05,551 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-13 13:15:05,551 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:05,551 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-13 13:15:05,551 INFO SenderThread:361603 [sender.py:transition_state():308] send defer: 1 +2021-07-13 13:15:05,551 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:05,551 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-13 13:15:05,600 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:05,600 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-13 13:15:05,600 INFO SenderThread:361603 [sender.py:transition_state():308] send defer: 2 +2021-07-13 13:15:05,600 DEBUG SenderThread:361603 [sender.py:send():179] send: stats +2021-07-13 13:15:05,601 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:05,601 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-13 13:15:05,601 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:05,601 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-13 13:15:05,601 INFO SenderThread:361603 [sender.py:transition_state():308] send defer: 3 +2021-07-13 13:15:05,601 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:05,601 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-13 13:15:05,602 DEBUG SenderThread:361603 [sender.py:send():179] send: summary +2021-07-13 13:15:05,602 INFO SenderThread:361603 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 13:15:05,602 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:05,602 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-13 13:15:05,602 INFO SenderThread:361603 [sender.py:transition_state():308] send defer: 4 +2021-07-13 13:15:05,602 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:05,602 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-13 13:15:05,603 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:05,603 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-13 13:15:05,652 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:05,770 INFO SenderThread:361603 [sender.py:transition_state():308] send defer: 5 +2021-07-13 13:15:05,770 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:05,771 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:05,771 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-13 13:15:05,771 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:05,771 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-13 13:15:05,771 INFO SenderThread:361603 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 13:15:05,785 INFO SenderThread:361603 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/config.yaml +2021-07-13 13:15:05,785 INFO SenderThread:361603 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/wandb-summary.json +2021-07-13 13:15:05,786 INFO SenderThread:361603 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/output.log +2021-07-13 13:15:05,786 INFO SenderThread:361603 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files +2021-07-13 13:15:05,786 INFO SenderThread:361603 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/requirements.txt requirements.txt +2021-07-13 13:15:05,786 INFO SenderThread:361603 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/output.log output.log +2021-07-13 13:15:05,786 INFO SenderThread:361603 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/wandb-metadata.json wandb-metadata.json +2021-07-13 13:15:05,787 INFO SenderThread:361603 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/config.yaml config.yaml +2021-07-13 13:15:05,789 INFO SenderThread:361603 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/wandb-summary.json wandb-summary.json +2021-07-13 13:15:05,789 INFO SenderThread:361603 [sender.py:transition_state():308] send defer: 6 +2021-07-13 13:15:05,792 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:05,792 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-13 13:15:05,792 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:05,792 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-13 13:15:05,792 INFO SenderThread:361603 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 13:15:05,872 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:05,873 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:05,975 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:05,975 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:06,077 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:06,077 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:06,179 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:06,179 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:06,226 INFO Thread-12 :361603 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/requirements.txt +2021-07-13 13:15:06,228 INFO Thread-13 :361603 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/output.log +2021-07-13 13:15:06,244 INFO Thread-15 :361603 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/wandb-summary.json +2021-07-13 13:15:06,280 INFO Thread-14 :361603 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/files/config.yaml +2021-07-13 13:15:06,282 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:06,283 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:06,385 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:06,385 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:06,480 INFO Thread-7 :361603 [sender.py:transition_state():308] send defer: 7 +2021-07-13 13:15:06,481 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:06,481 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-13 13:15:06,481 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:06,481 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-13 13:15:06,487 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:06,748 INFO SenderThread:361603 [sender.py:transition_state():308] send defer: 8 +2021-07-13 13:15:06,748 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:06,749 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:06,749 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-13 13:15:06,749 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:06,749 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-13 13:15:06,749 INFO SenderThread:361603 [sender.py:transition_state():308] send defer: 9 +2021-07-13 13:15:06,749 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:15:06,749 INFO HandlerThread:361603 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-13 13:15:06,750 DEBUG SenderThread:361603 [sender.py:send():179] send: final +2021-07-13 13:15:06,750 DEBUG SenderThread:361603 [sender.py:send():179] send: footer +2021-07-13 13:15:06,750 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: defer +2021-07-13 13:15:06,750 INFO SenderThread:361603 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-13 13:15:06,850 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:15:06,850 DEBUG SenderThread:361603 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:15:06,851 INFO SenderThread:361603 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 13:15:06,852 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: get_summary +2021-07-13 13:15:06,853 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-13 13:15:06,854 DEBUG HandlerThread:361603 [handler.py:handle_request():124] handle_request: shutdown +2021-07-13 13:15:06,854 INFO HandlerThread:361603 [handler.py:finish():638] shutting down handler +2021-07-13 13:15:07,750 INFO WriterThread:361603 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/run-3eewirqy.wandb +2021-07-13 13:15:07,851 INFO SenderThread:361603 [sender.py:finish():945] shutting down sender +2021-07-13 13:15:07,851 INFO SenderThread:361603 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 13:15:07,851 INFO SenderThread:361603 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 13:15:07,853 INFO MainThread:361603 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_131130-3eewirqy/logs/debug.log b/wandb/run-20210713_131130-3eewirqy/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e44dabd24cdaf7c16ab9a46149f8096a08cc63d6 --- /dev/null +++ b/wandb/run-20210713_131130-3eewirqy/logs/debug.log @@ -0,0 +1,111 @@ +2021-07-13 13:11:30,656 INFO MainThread:360037 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 13:11:30,656 INFO MainThread:360037 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 13:11:30,657 INFO MainThread:360037 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/logs/debug.log +2021-07-13 13:11:30,657 INFO MainThread:360037 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_131130-3eewirqy/logs/debug-internal.log +2021-07-13 13:11:30,657 INFO MainThread:360037 [wandb_init.py:init():370] calling init triggers +2021-07-13 13:11:30,657 INFO MainThread:360037 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 13:11:30,657 INFO MainThread:360037 [wandb_init.py:init():419] starting backend +2021-07-13 13:11:30,657 INFO MainThread:360037 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 13:11:30,729 INFO MainThread:360037 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 13:11:30,797 INFO MainThread:360037 [backend.py:ensure_launched():139] started backend process with pid: 361603 +2021-07-13 13:11:30,799 INFO MainThread:360037 [wandb_init.py:init():424] backend started and connected +2021-07-13 13:11:30,802 INFO MainThread:360037 [wandb_init.py:init():472] updated telemetry +2021-07-13 13:11:30,803 INFO MainThread:360037 [wandb_init.py:init():491] communicating current version +2021-07-13 13:11:31,478 INFO MainThread:360037 [wandb_init.py:init():496] got version response +2021-07-13 13:11:31,478 INFO MainThread:360037 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 13:11:31,724 INFO MainThread:360037 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 13:11:32,895 INFO MainThread:360037 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 13:11:32,896 INFO MainThread:360037 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 13:11:32,896 INFO MainThread:360037 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 13:11:32,898 INFO MainThread:360037 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 13:11:32,898 INFO MainThread:360037 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 13:11:32,904 INFO MainThread:360037 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_13-10-42_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 13:11:32,906 INFO MainThread:360037 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 13:11:32,908 INFO MainThread:360037 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 13:15:03,323 INFO MainThread:360037 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-13 13:15:03,325 INFO MainThread:360037 [wandb_run.py:_restore():1565] restore +2021-07-13 13:15:05,551 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 13:15:05,771 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 1365 +} + +2021-07-13 13:15:05,874 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1365 + total_bytes: 10810 +} + +2021-07-13 13:15:05,976 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10810 + total_bytes: 10810 +} + +2021-07-13 13:15:06,078 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10810 + total_bytes: 10810 +} + +2021-07-13 13:15:06,180 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10810 + total_bytes: 10810 +} + +2021-07-13 13:15:06,284 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10810 + total_bytes: 10810 +} + +2021-07-13 13:15:06,386 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10810 + total_bytes: 10810 +} + +2021-07-13 13:15:06,749 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10810 + total_bytes: 10810 +} + +2021-07-13 13:15:06,851 INFO MainThread:360037 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10810 + total_bytes: 10810 +} + +2021-07-13 13:15:08,100 INFO MainThread:360037 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210713_131130-3eewirqy/run-3eewirqy.wandb b/wandb/run-20210713_131130-3eewirqy/run-3eewirqy.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d26dda2e938007864b9714d0f870522738cf2f8a Binary files /dev/null and b/wandb/run-20210713_131130-3eewirqy/run-3eewirqy.wandb differ diff --git a/wandb/run-20210713_131615-w761mg4c/files/config.yaml b/wandb/run-20210713_131615-w761mg4c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3da7763f9f458e0e9789a1d83d5425322bc92464 --- /dev/null +++ b/wandb/run-20210713_131615-w761mg4c/files/config.yaml @@ -0,0 +1,307 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 8 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_13-15-26_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_131615-w761mg4c/files/output.log b/wandb/run-20210713_131615-w761mg4c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e0ab04b8dc779e70c1c10be7c93cc551c241e2f4 --- /dev/null +++ b/wandb/run-20210713_131615-w761mg4c/files/output.log @@ -0,0 +1,37 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1647, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 637, in xla_pmap_impl + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Attempting to allocate 276.41M. That was not possible. There are 337.62M free. Due to fragmentation, the largest contiguous region of free memory is 161.94M.; (0x0x0_HBM0): while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 725, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1152, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +RuntimeError: Resource exhausted: Attempting to allocate 276.41M. That was not possible. There are 337.62M free. Due to fragmentation, the largest contiguous region of free memory is 161.94M.; (0x0x0_HBM0): while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). \ No newline at end of file diff --git a/wandb/run-20210713_131615-w761mg4c/files/requirements.txt b/wandb/run-20210713_131615-w761mg4c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6a8b8359204c42ca0659d3680230134bc185fc2 --- /dev/null +++ b/wandb/run-20210713_131615-w761mg4c/files/requirements.txt @@ -0,0 +1,92 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.6.1 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==2.10 +jax==0.2.16 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210713_131615-w761mg4c/files/wandb-metadata.json b/wandb/run-20210713_131615-w761mg4c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..467601f4feed59f794599acace1940dae6d29381 --- /dev/null +++ b/wandb/run-20210713_131615-w761mg4c/files/wandb-metadata.json @@ -0,0 +1,45 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-13T13:16:18.041567", + "startedAt": "2021-07-13T13:16:15.966000", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=10", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=100001", + "--num_train_epochs=5", + "--preprocessing_num_workers=64", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--gradient_accumulation_steps=8", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "bc11ccfe77236f87575711b26034b9751449de4b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210713_131615-w761mg4c/files/wandb-summary.json b/wandb/run-20210713_131615-w761mg4c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210713_131615-w761mg4c/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210713_131615-w761mg4c/logs/debug-internal.log b/wandb/run-20210713_131615-w761mg4c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1db5c6fac3ccbf13c267fb5b8353085d84cc6e97 --- /dev/null +++ b/wandb/run-20210713_131615-w761mg4c/logs/debug-internal.log @@ -0,0 +1,172 @@ +2021-07-13 13:16:16,698 INFO MainThread:363863 [internal.py:wandb_internal():88] W&B internal server running at pid: 363863, started at: 2021-07-13 13:16:16.698045 +2021-07-13 13:16:16,700 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: check_version +2021-07-13 13:16:16,700 INFO WriterThread:363863 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/run-w761mg4c.wandb +2021-07-13 13:16:16,701 DEBUG SenderThread:363863 [sender.py:send():179] send: header +2021-07-13 13:16:16,701 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: check_version +2021-07-13 13:16:16,739 DEBUG SenderThread:363863 [sender.py:send():179] send: run +2021-07-13 13:16:16,916 INFO SenderThread:363863 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files +2021-07-13 13:16:16,916 INFO SenderThread:363863 [sender.py:_start_run_threads():716] run started: w761mg4c with start time 1626182176 +2021-07-13 13:16:16,916 DEBUG SenderThread:363863 [sender.py:send():179] send: summary +2021-07-13 13:16:16,917 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: run_start +2021-07-13 13:16:16,917 INFO SenderThread:363863 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 13:16:17,918 INFO Thread-8 :363863 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/wandb-summary.json +2021-07-13 13:16:18,041 DEBUG HandlerThread:363863 [meta.py:__init__():39] meta init +2021-07-13 13:16:18,041 DEBUG HandlerThread:363863 [meta.py:__init__():53] meta init done +2021-07-13 13:16:18,041 DEBUG HandlerThread:363863 [meta.py:probe():210] probe +2021-07-13 13:16:18,042 DEBUG HandlerThread:363863 [meta.py:_setup_git():200] setup git +2021-07-13 13:16:18,071 DEBUG HandlerThread:363863 [meta.py:_setup_git():207] setup git done +2021-07-13 13:16:18,071 DEBUG HandlerThread:363863 [meta.py:_save_pip():57] save pip +2021-07-13 13:16:18,072 DEBUG HandlerThread:363863 [meta.py:_save_pip():71] save pip done +2021-07-13 13:16:18,072 DEBUG HandlerThread:363863 [meta.py:probe():252] probe done +2021-07-13 13:16:18,075 DEBUG SenderThread:363863 [sender.py:send():179] send: files +2021-07-13 13:16:18,075 INFO SenderThread:363863 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-13 13:16:18,083 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:16:18,083 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:16:18,213 DEBUG SenderThread:363863 [sender.py:send():179] send: config +2021-07-13 13:16:18,214 DEBUG SenderThread:363863 [sender.py:send():179] send: config +2021-07-13 13:16:18,214 DEBUG SenderThread:363863 [sender.py:send():179] send: config +2021-07-13 13:16:18,530 INFO Thread-11 :363863 [upload_job.py:push():137] Uploaded file /tmp/tmpe18zfifmwandb/okixm8uo-wandb-metadata.json +2021-07-13 13:16:18,918 INFO Thread-8 :363863 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/output.log +2021-07-13 13:16:18,918 INFO Thread-8 :363863 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/wandb-metadata.json +2021-07-13 13:16:18,918 INFO Thread-8 :363863 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/requirements.txt +2021-07-13 13:16:33,215 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:16:33,215 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:16:34,924 INFO Thread-8 :363863 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/output.log +2021-07-13 13:16:46,124 DEBUG SenderThread:363863 [sender.py:send():179] send: stats +2021-07-13 13:16:47,928 INFO Thread-8 :363863 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/config.yaml +2021-07-13 13:16:48,349 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:16:48,350 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:17:03,483 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:17:03,484 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:17:16,202 DEBUG SenderThread:363863 [sender.py:send():179] send: stats +2021-07-13 13:17:18,616 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:17:18,617 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:17:33,748 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:17:33,749 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:17:46,277 DEBUG SenderThread:363863 [sender.py:send():179] send: stats +2021-07-13 13:17:48,883 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:17:48,883 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:18:04,023 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:18:04,023 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:18:04,956 INFO Thread-8 :363863 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/output.log +2021-07-13 13:18:16,347 DEBUG SenderThread:363863 [sender.py:send():179] send: stats +2021-07-13 13:18:19,179 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:18:19,179 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:18:34,324 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:18:34,325 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:18:46,418 DEBUG SenderThread:363863 [sender.py:send():179] send: stats +2021-07-13 13:18:49,456 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:18:49,456 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:19:04,586 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:19:04,586 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:19:16,489 DEBUG SenderThread:363863 [sender.py:send():179] send: stats +2021-07-13 13:19:19,718 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:19:19,718 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:19:34,851 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:19:34,852 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:19:46,568 DEBUG SenderThread:363863 [sender.py:send():179] send: stats +2021-07-13 13:19:49,985 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: stop_status +2021-07-13 13:19:49,985 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: stop_status +2021-07-13 13:20:00,715 DEBUG SenderThread:363863 [sender.py:send():179] send: telemetry +2021-07-13 13:20:00,715 DEBUG SenderThread:363863 [sender.py:send():179] send: exit +2021-07-13 13:20:00,715 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:00,715 INFO SenderThread:363863 [sender.py:send_exit():287] handling exit code: 1 +2021-07-13 13:20:00,716 INFO SenderThread:363863 [sender.py:send_exit():295] send defer +2021-07-13 13:20:00,716 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:00,716 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:00,716 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-13 13:20:00,717 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:00,717 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-13 13:20:00,717 INFO SenderThread:363863 [sender.py:transition_state():308] send defer: 1 +2021-07-13 13:20:00,717 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:00,717 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-13 13:20:00,805 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:00,805 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-13 13:20:00,805 INFO SenderThread:363863 [sender.py:transition_state():308] send defer: 2 +2021-07-13 13:20:00,806 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:00,806 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-13 13:20:00,806 DEBUG SenderThread:363863 [sender.py:send():179] send: stats +2021-07-13 13:20:00,806 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:00,806 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-13 13:20:00,806 INFO SenderThread:363863 [sender.py:transition_state():308] send defer: 3 +2021-07-13 13:20:00,806 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:00,807 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-13 13:20:00,807 DEBUG SenderThread:363863 [sender.py:send():179] send: summary +2021-07-13 13:20:00,807 INFO SenderThread:363863 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-13 13:20:00,807 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:00,807 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-13 13:20:00,807 INFO SenderThread:363863 [sender.py:transition_state():308] send defer: 4 +2021-07-13 13:20:00,808 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:00,808 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-13 13:20:00,808 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:00,808 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-13 13:20:00,818 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:00,985 INFO SenderThread:363863 [sender.py:transition_state():308] send defer: 5 +2021-07-13 13:20:00,986 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:00,986 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:00,986 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-13 13:20:00,987 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:00,987 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-13 13:20:00,987 INFO SenderThread:363863 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-13 13:20:00,998 INFO Thread-8 :363863 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/output.log +2021-07-13 13:20:00,999 INFO SenderThread:363863 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/config.yaml +2021-07-13 13:20:00,999 INFO SenderThread:363863 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/wandb-summary.json +2021-07-13 13:20:00,999 INFO SenderThread:363863 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files +2021-07-13 13:20:00,999 INFO SenderThread:363863 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/requirements.txt requirements.txt +2021-07-13 13:20:00,999 INFO SenderThread:363863 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/output.log output.log +2021-07-13 13:20:00,999 INFO SenderThread:363863 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/wandb-metadata.json wandb-metadata.json +2021-07-13 13:20:01,000 INFO SenderThread:363863 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/config.yaml config.yaml +2021-07-13 13:20:01,000 INFO SenderThread:363863 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/wandb-summary.json wandb-summary.json +2021-07-13 13:20:01,000 INFO SenderThread:363863 [sender.py:transition_state():308] send defer: 6 +2021-07-13 13:20:01,012 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:01,013 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-13 13:20:01,013 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:01,013 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-13 13:20:01,013 INFO SenderThread:363863 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 13:20:01,088 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:01,088 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:01,190 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:01,191 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:01,292 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:01,293 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:01,395 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:01,395 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:01,443 INFO Thread-13 :363863 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/output.log +2021-07-13 13:20:01,467 INFO Thread-15 :363863 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/wandb-summary.json +2021-07-13 13:20:01,468 INFO Thread-14 :363863 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/config.yaml +2021-07-13 13:20:01,474 INFO Thread-12 :363863 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/files/requirements.txt +2021-07-13 13:20:01,497 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:01,497 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:01,599 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:01,599 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:01,674 INFO Thread-7 :363863 [sender.py:transition_state():308] send defer: 7 +2021-07-13 13:20:01,675 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:01,675 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-13 13:20:01,675 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:01,675 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-13 13:20:01,701 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:01,941 INFO SenderThread:363863 [sender.py:transition_state():308] send defer: 8 +2021-07-13 13:20:01,942 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:01,942 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:01,942 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-13 13:20:01,942 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:01,943 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-13 13:20:01,943 INFO SenderThread:363863 [sender.py:transition_state():308] send defer: 9 +2021-07-13 13:20:01,943 DEBUG SenderThread:363863 [sender.py:send():179] send: final +2021-07-13 13:20:01,943 DEBUG SenderThread:363863 [sender.py:send():179] send: footer +2021-07-13 13:20:01,943 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: defer +2021-07-13 13:20:01,943 INFO HandlerThread:363863 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-13 13:20:01,944 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: defer +2021-07-13 13:20:01,944 INFO SenderThread:363863 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-13 13:20:02,044 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-13 13:20:02,044 DEBUG SenderThread:363863 [sender.py:send_request():193] send_request: poll_exit +2021-07-13 13:20:02,044 INFO SenderThread:363863 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 13:20:02,046 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: get_summary +2021-07-13 13:20:02,046 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-13 13:20:02,047 DEBUG HandlerThread:363863 [handler.py:handle_request():124] handle_request: shutdown +2021-07-13 13:20:02,047 INFO HandlerThread:363863 [handler.py:finish():638] shutting down handler +2021-07-13 13:20:02,944 INFO WriterThread:363863 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/run-w761mg4c.wandb +2021-07-13 13:20:03,044 INFO SenderThread:363863 [sender.py:finish():945] shutting down sender +2021-07-13 13:20:03,044 INFO SenderThread:363863 [file_pusher.py:finish():177] shutting down file pusher +2021-07-13 13:20:03,045 INFO SenderThread:363863 [file_pusher.py:join():182] waiting for file pusher +2021-07-13 13:20:03,047 INFO MainThread:363863 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210713_131615-w761mg4c/logs/debug.log b/wandb/run-20210713_131615-w761mg4c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..08d9a31a0fc00bcfc7a76fbee713959c002ce46e --- /dev/null +++ b/wandb/run-20210713_131615-w761mg4c/logs/debug.log @@ -0,0 +1,111 @@ +2021-07-13 13:16:15,967 INFO MainThread:362298 [wandb_setup.py:_flush():69] setting env: {} +2021-07-13 13:16:15,967 INFO MainThread:362298 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-13 13:16:15,967 INFO MainThread:362298 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/logs/debug.log +2021-07-13 13:16:15,967 INFO MainThread:362298 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_131615-w761mg4c/logs/debug-internal.log +2021-07-13 13:16:15,968 INFO MainThread:362298 [wandb_init.py:init():370] calling init triggers +2021-07-13 13:16:15,968 INFO MainThread:362298 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-13 13:16:15,968 INFO MainThread:362298 [wandb_init.py:init():419] starting backend +2021-07-13 13:16:15,968 INFO MainThread:362298 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-13 13:16:16,024 INFO MainThread:362298 [backend.py:ensure_launched():135] starting backend process... +2021-07-13 13:16:16,079 INFO MainThread:362298 [backend.py:ensure_launched():139] started backend process with pid: 363863 +2021-07-13 13:16:16,081 INFO MainThread:362298 [wandb_init.py:init():424] backend started and connected +2021-07-13 13:16:16,084 INFO MainThread:362298 [wandb_init.py:init():472] updated telemetry +2021-07-13 13:16:16,085 INFO MainThread:362298 [wandb_init.py:init():491] communicating current version +2021-07-13 13:16:16,737 INFO MainThread:362298 [wandb_init.py:init():496] got version response +2021-07-13 13:16:16,737 INFO MainThread:362298 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-13 13:16:16,916 INFO MainThread:362298 [wandb_init.py:init():529] starting run threads in backend +2021-07-13 13:16:18,079 INFO MainThread:362298 [wandb_run.py:_console_start():1623] atexit reg +2021-07-13 13:16:18,079 INFO MainThread:362298 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-13 13:16:18,080 INFO MainThread:362298 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-13 13:16:18,082 INFO MainThread:362298 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-13 13:16:18,082 INFO MainThread:362298 [wandb_init.py:init():554] run started, returning control to user process +2021-07-13 13:16:18,087 INFO MainThread:362298 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_13-15-26_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 100001, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-13 13:16:18,089 INFO MainThread:362298 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-13 13:16:18,091 INFO MainThread:362298 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-13 13:19:58,500 INFO MainThread:362298 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-13 13:19:58,502 INFO MainThread:362298 [wandb_run.py:_restore():1565] restore +2021-07-13 13:20:00,717 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 1388 +} + +2021-07-13 13:20:00,987 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 1388 +} + +2021-07-13 13:20:01,089 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1388 + total_bytes: 10904 +} + +2021-07-13 13:20:01,191 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10904 + total_bytes: 10904 +} + +2021-07-13 13:20:01,293 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10904 + total_bytes: 10904 +} + +2021-07-13 13:20:01,395 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10904 + total_bytes: 10904 +} + +2021-07-13 13:20:01,498 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10904 + total_bytes: 10904 +} + +2021-07-13 13:20:01,600 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10904 + total_bytes: 10904 +} + +2021-07-13 13:20:01,942 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10904 + total_bytes: 10904 +} + +2021-07-13 13:20:02,045 INFO MainThread:362298 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10904 + total_bytes: 10904 +} + +2021-07-13 13:20:03,309 INFO MainThread:362298 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210713_131615-w761mg4c/run-w761mg4c.wandb b/wandb/run-20210713_131615-w761mg4c/run-w761mg4c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2a543cda77a23365d7edf7dd51c805ae0865d4ab Binary files /dev/null and b/wandb/run-20210713_131615-w761mg4c/run-w761mg4c.wandb differ diff --git a/wandb/run-20210713_132755-1r3np4b7/files/config.yaml b/wandb/run-20210713_132755-1r3np4b7/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2830e5daf51a84bf3de6e75f6d13d210e27bfabf --- /dev/null +++ b/wandb/run-20210713_132755-1r3np4b7/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 100001 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 8 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul13_13-27-07_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 5.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 64 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 10 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210713_132755-1r3np4b7/files/output.log b/wandb/run-20210713_132755-1r3np4b7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e972f9ae8f781f0f32ef9ef835ff7917b4383384 --- /dev/null +++ b/wandb/run-20210713_132755-1r3np4b7/files/output.log @@ -0,0 +1,606 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/5): 0%| | 0/5 [00:00 + train_samples_idx = jax.random.permutation(input_rng, jnp.arange(num_train_samples)) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1669, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 637, in xla_pmap_impl + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1159, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Attempting to reserve 13.42G at the bottom of memory. That was not possible. There are 7.27G free, 0B reserved, and 7.22G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax_no_accum.py", line 673, in + train_samples_idx = jax.random.permutation(input_rng, jnp.arange(num_train_samples)) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1159, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +RuntimeError: Resource exhausted: Attempting to reserve 13.42G at the bottom of memory. That was not possible. There are 7.27G free, 0B reserved, and 7.22G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). \ No newline at end of file diff --git a/wandb/run-20210714_175518-t843i2it/files/requirements.txt b/wandb/run-20210714_175518-t843i2it/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7500433524bd88d37f4fbd1efbe772b80bf4ab23 --- /dev/null +++ b/wandb/run-20210714_175518-t843i2it/files/requirements.txt @@ -0,0 +1,93 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.1 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.7.0 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==3.2 +jax==0.2.17 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.26.0 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210714_175518-t843i2it/files/wandb-metadata.json b/wandb/run-20210714_175518-t843i2it/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..34745d833da8c72149dec8896e1cadac9363829c --- /dev/null +++ b/wandb/run-20210714_175518-t843i2it/files/wandb-metadata.json @@ -0,0 +1,47 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-14T17:55:32.372351", + "startedAt": "2021-07-14T17:55:18.068102", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=5000", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=40000", + "--num_train_epochs=3", + "--preprocessing_num_workers=96", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--max_eval_samples=20000", + "--adafactor", + "--overwrite_cache", + "False" + ], + "state": "running", + "program": "./run_mlm_flax_no_accum.py", + "codePath": "run_mlm_flax_no_accum.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "f6e0bf7955345b689d1a39a7ffb5ce2ba806da0b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210714_175518-t843i2it/files/wandb-summary.json b/wandb/run-20210714_175518-t843i2it/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210714_175518-t843i2it/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210714_175518-t843i2it/logs/debug-internal.log b/wandb/run-20210714_175518-t843i2it/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..185c44a7992d92b681e6b39b80e18fb5956ae5f2 --- /dev/null +++ b/wandb/run-20210714_175518-t843i2it/logs/debug-internal.log @@ -0,0 +1,236 @@ +2021-07-14 17:55:20,938 INFO MainThread:545287 [internal.py:wandb_internal():88] W&B internal server running at pid: 545287, started at: 2021-07-14 17:55:20.938091 +2021-07-14 17:55:20,940 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: check_version +2021-07-14 17:55:20,941 INFO WriterThread:545287 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/run-t843i2it.wandb +2021-07-14 17:55:20,942 DEBUG SenderThread:545287 [sender.py:send():179] send: header +2021-07-14 17:55:20,942 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: check_version +2021-07-14 17:55:20,985 DEBUG SenderThread:545287 [sender.py:send():179] send: run +2021-07-14 17:55:21,157 INFO SenderThread:545287 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files +2021-07-14 17:55:21,158 INFO SenderThread:545287 [sender.py:_start_run_threads():716] run started: t843i2it with start time 1626285318 +2021-07-14 17:55:21,158 DEBUG SenderThread:545287 [sender.py:send():179] send: summary +2021-07-14 17:55:21,158 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: run_start +2021-07-14 17:55:21,159 INFO SenderThread:545287 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 17:55:24,792 INFO Thread-8 :545287 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/wandb-summary.json +2021-07-14 17:55:26,883 INFO Thread-8 :545287 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/output.log +2021-07-14 17:55:32,372 DEBUG HandlerThread:545287 [meta.py:__init__():39] meta init +2021-07-14 17:55:32,372 DEBUG HandlerThread:545287 [meta.py:__init__():53] meta init done +2021-07-14 17:55:32,372 DEBUG HandlerThread:545287 [meta.py:probe():210] probe +2021-07-14 17:55:32,373 DEBUG HandlerThread:545287 [meta.py:_setup_git():200] setup git +2021-07-14 17:55:32,402 DEBUG HandlerThread:545287 [meta.py:_setup_git():207] setup git done +2021-07-14 17:55:32,402 DEBUG HandlerThread:545287 [meta.py:_save_pip():57] save pip +2021-07-14 17:55:32,403 DEBUG HandlerThread:545287 [meta.py:_save_pip():71] save pip done +2021-07-14 17:55:32,403 DEBUG HandlerThread:545287 [meta.py:probe():252] probe done +2021-07-14 17:55:32,406 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:55:32,406 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:55:32,544 DEBUG SenderThread:545287 [sender.py:send():179] send: config +2021-07-14 17:55:32,545 DEBUG SenderThread:545287 [sender.py:send():179] send: config +2021-07-14 17:55:32,545 DEBUG SenderThread:545287 [sender.py:send():179] send: config +2021-07-14 17:55:32,545 DEBUG SenderThread:545287 [sender.py:send():179] send: files +2021-07-14 17:55:32,545 INFO SenderThread:545287 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-14 17:55:32,952 INFO Thread-8 :545287 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/wandb-metadata.json +2021-07-14 17:55:32,953 INFO Thread-8 :545287 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/requirements.txt +2021-07-14 17:55:33,181 INFO Thread-11 :545287 [upload_job.py:push():137] Uploaded file /tmp/tmpldoi9p0fwandb/1bn7qo44-wandb-metadata.json +2021-07-14 17:55:42,956 INFO Thread-8 :545287 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/output.log +2021-07-14 17:55:44,957 INFO Thread-8 :545287 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/output.log +2021-07-14 17:55:47,548 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:55:47,549 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:55:51,960 INFO Thread-8 :545287 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/config.yaml +2021-07-14 17:56:00,458 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 17:56:02,681 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:56:02,681 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:56:17,816 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:56:17,816 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:56:30,528 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 17:56:32,958 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:56:32,959 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:56:48,094 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:56:48,094 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:57:00,620 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 17:57:03,228 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:57:03,229 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:57:18,364 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:57:18,364 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:57:30,693 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 17:57:33,502 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:57:33,502 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:57:48,640 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:57:48,640 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:58:00,768 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 17:58:03,776 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:58:03,776 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:58:18,928 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:58:18,928 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:58:30,841 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 17:58:34,063 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:58:34,063 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:58:49,196 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:58:49,196 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:59:00,919 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 17:59:04,329 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:59:04,330 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:59:19,466 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:59:19,467 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:59:30,993 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 17:59:34,601 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:59:34,601 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 17:59:49,736 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 17:59:49,736 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:00:01,070 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:00:04,866 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:00:04,867 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:00:20,000 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:00:20,001 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:00:31,138 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:00:35,144 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:00:35,144 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:00:50,279 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:00:50,280 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:01:01,215 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:01:05,414 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:01:05,414 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:01:20,564 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:01:20,565 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:01:31,294 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:01:35,699 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:01:35,699 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:01:50,829 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:01:50,830 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:02:01,371 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:02:05,966 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:02:05,967 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:02:21,099 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:02:21,099 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:02:31,448 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:02:36,234 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:02:36,235 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:02:51,370 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:02:51,370 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:03:01,521 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:03:06,821 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:03:06,821 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:03:22,036 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:03:22,036 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:03:31,600 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:03:33,147 INFO Thread-8 :545287 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/output.log +2021-07-14 18:03:37,189 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:03:37,190 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:03:52,336 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:03:52,337 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:04:01,678 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:04:07,470 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:04:07,470 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:04:22,603 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:04:22,604 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:04:31,760 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:04:37,735 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:04:37,735 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:04:52,873 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:04:52,873 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:05:01,843 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:05:08,009 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:05:08,010 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:05:21,189 INFO Thread-8 :545287 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/output.log +2021-07-14 18:05:22,751 DEBUG SenderThread:545287 [sender.py:send():179] send: telemetry +2021-07-14 18:05:22,752 DEBUG SenderThread:545287 [sender.py:send():179] send: exit +2021-07-14 18:05:22,752 INFO SenderThread:545287 [sender.py:send_exit():287] handling exit code: 1 +2021-07-14 18:05:22,753 INFO SenderThread:545287 [sender.py:send_exit():295] send defer +2021-07-14 18:05:22,753 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:22,754 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:22,754 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-14 18:05:22,754 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:22,754 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:22,754 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-14 18:05:22,754 INFO SenderThread:545287 [sender.py:transition_state():308] send defer: 1 +2021-07-14 18:05:22,755 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:22,755 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-14 18:05:22,795 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:22,795 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-14 18:05:22,795 INFO SenderThread:545287 [sender.py:transition_state():308] send defer: 2 +2021-07-14 18:05:22,796 DEBUG SenderThread:545287 [sender.py:send():179] send: stats +2021-07-14 18:05:22,796 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:22,796 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-14 18:05:22,796 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:22,797 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-14 18:05:22,797 INFO SenderThread:545287 [sender.py:transition_state():308] send defer: 3 +2021-07-14 18:05:22,797 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:22,797 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-14 18:05:22,797 DEBUG SenderThread:545287 [sender.py:send():179] send: summary +2021-07-14 18:05:22,797 INFO SenderThread:545287 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 18:05:22,798 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:22,798 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-14 18:05:22,798 INFO SenderThread:545287 [sender.py:transition_state():308] send defer: 4 +2021-07-14 18:05:22,798 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:22,798 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-14 18:05:22,798 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:22,798 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-14 18:05:22,856 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:22,985 INFO SenderThread:545287 [sender.py:transition_state():308] send defer: 5 +2021-07-14 18:05:22,985 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:22,986 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:22,986 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-14 18:05:22,986 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:22,986 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-14 18:05:22,986 INFO SenderThread:545287 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-14 18:05:23,087 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:23,190 INFO Thread-8 :545287 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/output.log +2021-07-14 18:05:23,190 INFO SenderThread:545287 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/wandb-summary.json +2021-07-14 18:05:23,190 INFO SenderThread:545287 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/config.yaml +2021-07-14 18:05:23,191 INFO SenderThread:545287 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files +2021-07-14 18:05:23,191 INFO SenderThread:545287 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/requirements.txt requirements.txt +2021-07-14 18:05:23,191 INFO SenderThread:545287 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/output.log output.log +2021-07-14 18:05:23,195 INFO SenderThread:545287 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/wandb-metadata.json wandb-metadata.json +2021-07-14 18:05:23,195 INFO SenderThread:545287 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/config.yaml config.yaml +2021-07-14 18:05:23,195 INFO SenderThread:545287 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/wandb-summary.json wandb-summary.json +2021-07-14 18:05:23,195 INFO SenderThread:545287 [sender.py:transition_state():308] send defer: 6 +2021-07-14 18:05:23,195 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:23,196 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:23,196 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-14 18:05:23,197 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:23,197 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-14 18:05:23,197 INFO SenderThread:545287 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 18:05:23,299 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:23,299 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:23,401 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:23,401 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:23,503 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:23,503 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:23,605 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:23,605 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:23,648 INFO Thread-12 :545287 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/requirements.txt +2021-07-14 18:05:23,655 INFO Thread-13 :545287 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/output.log +2021-07-14 18:05:23,658 INFO Thread-15 :545287 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/wandb-summary.json +2021-07-14 18:05:23,661 INFO Thread-14 :545287 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/files/config.yaml +2021-07-14 18:05:23,708 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:23,708 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:23,810 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:23,810 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:23,861 INFO Thread-7 :545287 [sender.py:transition_state():308] send defer: 7 +2021-07-14 18:05:23,862 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:23,862 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-14 18:05:23,862 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:23,862 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-14 18:05:23,912 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:24,152 INFO SenderThread:545287 [sender.py:transition_state():308] send defer: 8 +2021-07-14 18:05:24,152 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:24,153 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:24,153 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-14 18:05:24,154 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:24,154 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-14 18:05:24,154 INFO SenderThread:545287 [sender.py:transition_state():308] send defer: 9 +2021-07-14 18:05:24,154 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:05:24,155 INFO HandlerThread:545287 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-14 18:05:24,155 DEBUG SenderThread:545287 [sender.py:send():179] send: final +2021-07-14 18:05:24,155 DEBUG SenderThread:545287 [sender.py:send():179] send: footer +2021-07-14 18:05:24,155 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: defer +2021-07-14 18:05:24,155 INFO SenderThread:545287 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-14 18:05:24,255 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:05:24,255 DEBUG SenderThread:545287 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:05:24,255 INFO SenderThread:545287 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 18:05:24,257 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: get_summary +2021-07-14 18:05:24,257 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-14 18:05:24,258 DEBUG HandlerThread:545287 [handler.py:handle_request():124] handle_request: shutdown +2021-07-14 18:05:24,258 INFO HandlerThread:545287 [handler.py:finish():638] shutting down handler +2021-07-14 18:05:25,155 INFO WriterThread:545287 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/run-t843i2it.wandb +2021-07-14 18:05:25,256 INFO SenderThread:545287 [sender.py:finish():945] shutting down sender +2021-07-14 18:05:25,256 INFO SenderThread:545287 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 18:05:25,256 INFO SenderThread:545287 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 18:05:25,275 INFO MainThread:545287 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210714_175518-t843i2it/logs/debug.log b/wandb/run-20210714_175518-t843i2it/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8dedcc969c298fac4e3aa885ee88901fe53d1cb8 --- /dev/null +++ b/wandb/run-20210714_175518-t843i2it/logs/debug.log @@ -0,0 +1,119 @@ +2021-07-14 17:55:18,104 INFO MainThread:533662 [wandb_setup.py:_flush():69] setting env: {} +2021-07-14 17:55:18,104 INFO MainThread:533662 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-14 17:55:18,104 INFO MainThread:533662 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/logs/debug.log +2021-07-14 17:55:18,104 INFO MainThread:533662 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_175518-t843i2it/logs/debug-internal.log +2021-07-14 17:55:18,104 INFO MainThread:533662 [wandb_init.py:init():370] calling init triggers +2021-07-14 17:55:18,104 INFO MainThread:533662 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-14 17:55:18,104 INFO MainThread:533662 [wandb_init.py:init():419] starting backend +2021-07-14 17:55:18,104 INFO MainThread:533662 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-14 17:55:18,233 INFO MainThread:533662 [backend.py:ensure_launched():135] starting backend process... +2021-07-14 17:55:18,315 INFO MainThread:533662 [backend.py:ensure_launched():139] started backend process with pid: 545287 +2021-07-14 17:55:18,317 INFO MainThread:533662 [wandb_init.py:init():424] backend started and connected +2021-07-14 17:55:18,320 INFO MainThread:533662 [wandb_init.py:init():472] updated telemetry +2021-07-14 17:55:18,321 INFO MainThread:533662 [wandb_init.py:init():491] communicating current version +2021-07-14 17:55:20,984 INFO MainThread:533662 [wandb_init.py:init():496] got version response +2021-07-14 17:55:20,984 INFO MainThread:533662 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-14 17:55:21,157 INFO MainThread:533662 [wandb_init.py:init():529] starting run threads in backend +2021-07-14 17:55:26,162 INFO MainThread:533662 [wandb_run.py:_console_start():1623] atexit reg +2021-07-14 17:55:26,163 INFO MainThread:533662 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-14 17:55:26,163 INFO MainThread:533662 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-14 17:55:26,165 INFO MainThread:533662 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-14 17:55:26,165 INFO MainThread:533662 [wandb_init.py:init():554] run started, returning control to user process +2021-07-14 17:55:26,171 INFO MainThread:533662 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_09-57-27_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-14 17:55:26,173 INFO MainThread:533662 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-14 17:55:26,175 INFO MainThread:533662 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 20000} +2021-07-14 18:05:20,153 INFO MainThread:533662 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-14 18:05:20,155 INFO MainThread:533662 [wandb_run.py:_restore():1565] restore +2021-07-14 18:05:22,755 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1446 + total_bytes: 1446 +} + +2021-07-14 18:05:22,986 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1446 + total_bytes: 1446 +} + +2021-07-14 18:05:23,198 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 3 +} +pusher_stats { + uploaded_bytes: 1446 + total_bytes: 6385 +} + +2021-07-14 18:05:23,300 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1446 + total_bytes: 10955 +} + +2021-07-14 18:05:23,402 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10955 + total_bytes: 10955 +} + +2021-07-14 18:05:23,504 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10955 + total_bytes: 10955 +} + +2021-07-14 18:05:23,606 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10955 + total_bytes: 10955 +} + +2021-07-14 18:05:23,708 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10955 + total_bytes: 10955 +} + +2021-07-14 18:05:23,811 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10955 + total_bytes: 10955 +} + +2021-07-14 18:05:24,154 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10955 + total_bytes: 10955 +} + +2021-07-14 18:05:24,256 INFO MainThread:533662 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10955 + total_bytes: 10955 +} + +2021-07-14 18:05:27,055 INFO MainThread:533662 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210714_175518-t843i2it/run-t843i2it.wandb b/wandb/run-20210714_175518-t843i2it/run-t843i2it.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2cb90218aca412737f6fc26f7f4c8db4d77ff97a Binary files /dev/null and b/wandb/run-20210714_175518-t843i2it/run-t843i2it.wandb differ diff --git a/wandb/run-20210714_181954-351hrfn2/files/config.yaml b/wandb/run-20210714_181954-351hrfn2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e792ac06efc9b8161c382d078945c21b85b60c6 --- /dev/null +++ b/wandb/run-20210714_181954-351hrfn2/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: bfloat16 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 40000 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 1 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul14_18-10-04_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_eval_samples: + desc: null + value: 20000 +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 3.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 96 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 5000 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210714_181954-351hrfn2/files/output.log b/wandb/run-20210714_181954-351hrfn2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3c14a30ebefcce203a88ff04c3bb74b479c0631f --- /dev/null +++ b/wandb/run-20210714_181954-351hrfn2/files/output.log @@ -0,0 +1,37 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 + model_inputs = shard(model_inputs.data) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1669, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 637, in xla_pmap_impl + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1159, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Attempting to reserve 11.03G at the bottom of memory. That was not possible. There are 7.51G free, 0B reserved, and 7.45G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax_no_accum.py", line 683, in + model_inputs = shard(model_inputs.data) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1159, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +RuntimeError: Resource exhausted: Attempting to reserve 11.03G at the bottom of memory. That was not possible. There are 7.51G free, 0B reserved, and 7.45G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). \ No newline at end of file diff --git a/wandb/run-20210714_181954-351hrfn2/files/requirements.txt b/wandb/run-20210714_181954-351hrfn2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7500433524bd88d37f4fbd1efbe772b80bf4ab23 --- /dev/null +++ b/wandb/run-20210714_181954-351hrfn2/files/requirements.txt @@ -0,0 +1,93 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.1 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.7.0 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==3.2 +jax==0.2.17 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.26.0 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210714_181954-351hrfn2/files/wandb-metadata.json b/wandb/run-20210714_181954-351hrfn2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..12b0049e8abd05249b5d4e6d928957ac869ad6ec --- /dev/null +++ b/wandb/run-20210714_181954-351hrfn2/files/wandb-metadata.json @@ -0,0 +1,48 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-14T18:19:56.606543", + "startedAt": "2021-07-14T18:19:54.637569", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=5000", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=40000", + "--num_train_epochs=3", + "--preprocessing_num_workers=96", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--max_eval_samples=20000", + "--overwrite_cache", + "False", + "--dtype=bfloat16", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax_no_accum.py", + "codePath": "run_mlm_flax_no_accum.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "f6e0bf7955345b689d1a39a7ffb5ce2ba806da0b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210714_181954-351hrfn2/files/wandb-summary.json b/wandb/run-20210714_181954-351hrfn2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210714_181954-351hrfn2/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210714_181954-351hrfn2/logs/debug-internal.log b/wandb/run-20210714_181954-351hrfn2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..26634d8fad734d530eea519990b8a97e23d993dd --- /dev/null +++ b/wandb/run-20210714_181954-351hrfn2/logs/debug-internal.log @@ -0,0 +1,237 @@ +2021-07-14 18:19:55,251 INFO MainThread:549018 [internal.py:wandb_internal():88] W&B internal server running at pid: 549018, started at: 2021-07-14 18:19:55.251471 +2021-07-14 18:19:55,253 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: check_version +2021-07-14 18:19:55,254 INFO WriterThread:549018 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/run-351hrfn2.wandb +2021-07-14 18:19:55,255 DEBUG SenderThread:549018 [sender.py:send():179] send: header +2021-07-14 18:19:55,255 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: check_version +2021-07-14 18:19:55,291 DEBUG SenderThread:549018 [sender.py:send():179] send: run +2021-07-14 18:19:55,471 INFO SenderThread:549018 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files +2021-07-14 18:19:55,471 INFO SenderThread:549018 [sender.py:_start_run_threads():716] run started: 351hrfn2 with start time 1626286794 +2021-07-14 18:19:55,471 DEBUG SenderThread:549018 [sender.py:send():179] send: summary +2021-07-14 18:19:55,471 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: run_start +2021-07-14 18:19:55,472 INFO SenderThread:549018 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 18:19:56,473 INFO Thread-8 :549018 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/wandb-summary.json +2021-07-14 18:19:56,606 DEBUG HandlerThread:549018 [meta.py:__init__():39] meta init +2021-07-14 18:19:56,606 DEBUG HandlerThread:549018 [meta.py:__init__():53] meta init done +2021-07-14 18:19:56,606 DEBUG HandlerThread:549018 [meta.py:probe():210] probe +2021-07-14 18:19:56,607 DEBUG HandlerThread:549018 [meta.py:_setup_git():200] setup git +2021-07-14 18:19:56,636 DEBUG HandlerThread:549018 [meta.py:_setup_git():207] setup git done +2021-07-14 18:19:56,636 DEBUG HandlerThread:549018 [meta.py:_save_pip():57] save pip +2021-07-14 18:19:56,636 DEBUG HandlerThread:549018 [meta.py:_save_pip():71] save pip done +2021-07-14 18:19:56,636 DEBUG HandlerThread:549018 [meta.py:probe():252] probe done +2021-07-14 18:19:56,640 DEBUG SenderThread:549018 [sender.py:send():179] send: files +2021-07-14 18:19:56,640 INFO SenderThread:549018 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-14 18:19:56,648 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:19:56,648 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:19:56,776 DEBUG SenderThread:549018 [sender.py:send():179] send: config +2021-07-14 18:19:56,777 DEBUG SenderThread:549018 [sender.py:send():179] send: config +2021-07-14 18:19:56,777 DEBUG SenderThread:549018 [sender.py:send():179] send: config +2021-07-14 18:19:57,091 INFO Thread-11 :549018 [upload_job.py:push():137] Uploaded file /tmp/tmpup88esztwandb/nma268qr-wandb-metadata.json +2021-07-14 18:19:57,473 INFO Thread-8 :549018 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/wandb-metadata.json +2021-07-14 18:19:57,473 INFO Thread-8 :549018 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/output.log +2021-07-14 18:19:57,473 INFO Thread-8 :549018 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/requirements.txt +2021-07-14 18:20:11,779 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:20:11,779 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:20:15,481 INFO Thread-8 :549018 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/output.log +2021-07-14 18:20:24,687 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:20:26,485 INFO Thread-8 :549018 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/config.yaml +2021-07-14 18:20:26,913 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:20:26,913 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:20:42,047 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:20:42,048 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:20:54,758 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:20:57,181 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:20:57,182 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:21:12,314 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:21:12,315 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:21:24,825 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:21:27,445 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:21:27,446 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:21:42,580 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:21:42,580 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:21:54,891 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:21:57,716 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:21:57,716 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:22:12,849 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:22:12,849 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:22:24,963 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:22:27,983 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:22:27,984 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:22:43,121 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:22:43,121 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:22:55,039 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:22:58,254 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:22:58,254 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:23:13,385 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:23:13,385 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:23:25,114 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:23:28,517 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:23:28,517 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:23:43,648 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:23:43,649 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:23:55,185 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:23:58,785 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:23:58,785 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:24:13,918 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:24:13,918 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:24:25,261 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:24:29,053 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:24:29,053 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:24:44,185 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:24:44,185 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:24:55,335 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:24:59,318 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:24:59,318 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:25:14,450 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:25:14,450 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:25:25,409 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:25:29,584 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:25:29,584 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:25:44,713 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:25:44,713 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:25:55,479 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:25:59,845 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:25:59,845 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:26:14,976 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:26:14,977 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:26:25,544 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:26:30,114 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:26:30,114 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:26:45,247 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:26:45,247 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:26:55,606 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:27:00,379 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:27:00,380 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:27:15,542 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:27:15,542 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:27:25,670 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:27:30,683 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:27:30,684 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:27:45,816 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:27:45,817 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:27:55,730 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:28:00,949 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:28:00,949 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:28:05,678 INFO Thread-8 :549018 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/output.log +2021-07-14 18:28:16,097 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:28:16,097 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:28:25,799 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:28:31,288 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:28:31,288 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:28:46,425 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:28:46,425 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:28:55,882 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:29:01,559 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:29:01,559 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:29:16,690 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:29:16,690 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:29:25,961 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:29:31,823 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:29:31,823 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:29:43,722 INFO Thread-8 :549018 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/output.log +2021-07-14 18:29:44,729 DEBUG SenderThread:549018 [sender.py:send():179] send: telemetry +2021-07-14 18:29:44,730 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:44,730 DEBUG SenderThread:549018 [sender.py:send():179] send: exit +2021-07-14 18:29:44,730 INFO SenderThread:549018 [sender.py:send_exit():287] handling exit code: 1 +2021-07-14 18:29:44,731 INFO SenderThread:549018 [sender.py:send_exit():295] send defer +2021-07-14 18:29:44,731 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:44,731 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:44,731 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-14 18:29:44,732 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:44,732 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-14 18:29:44,732 INFO SenderThread:549018 [sender.py:transition_state():308] send defer: 1 +2021-07-14 18:29:44,732 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:44,732 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-14 18:29:44,807 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:44,807 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-14 18:29:44,807 INFO SenderThread:549018 [sender.py:transition_state():308] send defer: 2 +2021-07-14 18:29:44,807 DEBUG SenderThread:549018 [sender.py:send():179] send: stats +2021-07-14 18:29:44,807 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:44,807 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-14 18:29:44,808 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:44,808 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-14 18:29:44,808 INFO SenderThread:549018 [sender.py:transition_state():308] send defer: 3 +2021-07-14 18:29:44,808 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:44,808 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-14 18:29:44,808 DEBUG SenderThread:549018 [sender.py:send():179] send: summary +2021-07-14 18:29:44,808 INFO SenderThread:549018 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 18:29:44,809 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:44,809 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-14 18:29:44,809 INFO SenderThread:549018 [sender.py:transition_state():308] send defer: 4 +2021-07-14 18:29:44,809 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:44,809 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-14 18:29:44,809 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:44,809 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-14 18:29:44,833 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:44,983 INFO SenderThread:549018 [sender.py:transition_state():308] send defer: 5 +2021-07-14 18:29:44,983 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:44,984 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:44,984 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-14 18:29:44,984 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:44,984 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-14 18:29:44,984 INFO SenderThread:549018 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-14 18:29:45,085 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:45,722 INFO Thread-8 :549018 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/output.log +2021-07-14 18:29:45,723 INFO SenderThread:549018 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/wandb-summary.json +2021-07-14 18:29:45,723 INFO SenderThread:549018 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/config.yaml +2021-07-14 18:29:45,724 INFO SenderThread:549018 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files +2021-07-14 18:29:45,724 INFO SenderThread:549018 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/requirements.txt requirements.txt +2021-07-14 18:29:45,724 INFO SenderThread:549018 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/output.log output.log +2021-07-14 18:29:45,725 INFO SenderThread:549018 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/wandb-metadata.json wandb-metadata.json +2021-07-14 18:29:45,725 INFO SenderThread:549018 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/config.yaml config.yaml +2021-07-14 18:29:45,725 INFO SenderThread:549018 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/wandb-summary.json wandb-summary.json +2021-07-14 18:29:45,731 INFO SenderThread:549018 [sender.py:transition_state():308] send defer: 6 +2021-07-14 18:29:45,731 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:45,735 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:45,735 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-14 18:29:45,738 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:45,738 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-14 18:29:45,738 INFO SenderThread:549018 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 18:29:45,833 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:45,834 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:45,935 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:45,935 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:46,037 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:46,037 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:46,139 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:46,140 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:46,204 INFO Thread-15 :549018 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/wandb-summary.json +2021-07-14 18:29:46,205 INFO Thread-12 :549018 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/requirements.txt +2021-07-14 18:29:46,218 INFO Thread-14 :549018 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/config.yaml +2021-07-14 18:29:46,241 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:46,242 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:46,287 INFO Thread-13 :549018 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/files/output.log +2021-07-14 18:29:46,343 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:46,344 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:46,446 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:46,446 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:46,488 INFO Thread-7 :549018 [sender.py:transition_state():308] send defer: 7 +2021-07-14 18:29:46,489 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:46,489 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-14 18:29:46,489 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:46,489 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-14 18:29:46,548 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:46,780 INFO SenderThread:549018 [sender.py:transition_state():308] send defer: 8 +2021-07-14 18:29:46,780 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:46,780 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:46,780 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-14 18:29:46,780 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:46,781 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-14 18:29:46,781 INFO SenderThread:549018 [sender.py:transition_state():308] send defer: 9 +2021-07-14 18:29:46,781 DEBUG SenderThread:549018 [sender.py:send():179] send: final +2021-07-14 18:29:46,781 DEBUG SenderThread:549018 [sender.py:send():179] send: footer +2021-07-14 18:29:46,781 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:29:46,782 INFO HandlerThread:549018 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-14 18:29:46,782 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: defer +2021-07-14 18:29:46,782 INFO SenderThread:549018 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-14 18:29:46,882 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:29:46,882 DEBUG SenderThread:549018 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:29:46,882 INFO SenderThread:549018 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 18:29:46,883 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: get_summary +2021-07-14 18:29:46,884 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-14 18:29:46,884 DEBUG HandlerThread:549018 [handler.py:handle_request():124] handle_request: shutdown +2021-07-14 18:29:46,885 INFO HandlerThread:549018 [handler.py:finish():638] shutting down handler +2021-07-14 18:29:47,782 INFO WriterThread:549018 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/run-351hrfn2.wandb +2021-07-14 18:29:47,882 INFO SenderThread:549018 [sender.py:finish():945] shutting down sender +2021-07-14 18:29:47,882 INFO SenderThread:549018 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 18:29:47,882 INFO SenderThread:549018 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 18:29:47,885 INFO MainThread:549018 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210714_181954-351hrfn2/logs/debug.log b/wandb/run-20210714_181954-351hrfn2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..48f908635efe9a76976c0901174ecd2c31d331a2 --- /dev/null +++ b/wandb/run-20210714_181954-351hrfn2/logs/debug.log @@ -0,0 +1,127 @@ +2021-07-14 18:19:54,639 INFO MainThread:547087 [wandb_setup.py:_flush():69] setting env: {} +2021-07-14 18:19:54,639 INFO MainThread:547087 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-14 18:19:54,639 INFO MainThread:547087 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/logs/debug.log +2021-07-14 18:19:54,639 INFO MainThread:547087 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_181954-351hrfn2/logs/debug-internal.log +2021-07-14 18:19:54,639 INFO MainThread:547087 [wandb_init.py:init():370] calling init triggers +2021-07-14 18:19:54,639 INFO MainThread:547087 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-14 18:19:54,639 INFO MainThread:547087 [wandb_init.py:init():419] starting backend +2021-07-14 18:19:54,639 INFO MainThread:547087 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-14 18:19:54,677 INFO MainThread:547087 [backend.py:ensure_launched():135] starting backend process... +2021-07-14 18:19:54,714 INFO MainThread:547087 [backend.py:ensure_launched():139] started backend process with pid: 549018 +2021-07-14 18:19:54,716 INFO MainThread:547087 [wandb_init.py:init():424] backend started and connected +2021-07-14 18:19:54,718 INFO MainThread:547087 [wandb_init.py:init():472] updated telemetry +2021-07-14 18:19:54,719 INFO MainThread:547087 [wandb_init.py:init():491] communicating current version +2021-07-14 18:19:55,290 INFO MainThread:547087 [wandb_init.py:init():496] got version response +2021-07-14 18:19:55,290 INFO MainThread:547087 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-14 18:19:55,471 INFO MainThread:547087 [wandb_init.py:init():529] starting run threads in backend +2021-07-14 18:19:56,644 INFO MainThread:547087 [wandb_run.py:_console_start():1623] atexit reg +2021-07-14 18:19:56,644 INFO MainThread:547087 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-14 18:19:56,645 INFO MainThread:547087 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-14 18:19:56,646 INFO MainThread:547087 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-14 18:19:56,646 INFO MainThread:547087 [wandb_init.py:init():554] run started, returning control to user process +2021-07-14 18:19:56,653 INFO MainThread:547087 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_18-10-04_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-14 18:19:56,654 INFO MainThread:547087 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'} +2021-07-14 18:19:56,656 INFO MainThread:547087 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 20000} +2021-07-14 18:29:42,529 INFO MainThread:547087 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-14 18:29:42,531 INFO MainThread:547087 [wandb_run.py:_restore():1565] restore +2021-07-14 18:29:44,732 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1474 + total_bytes: 1474 +} + +2021-07-14 18:29:44,984 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1474 + total_bytes: 1474 +} + +2021-07-14 18:29:45,732 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 4 +} +pusher_stats { + uploaded_bytes: 1474 + total_bytes: 10923 +} + +2021-07-14 18:29:45,834 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1474 + total_bytes: 10925 +} + +2021-07-14 18:29:45,936 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 7680 + total_bytes: 10925 +} + +2021-07-14 18:29:46,038 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10925 + total_bytes: 10925 +} + +2021-07-14 18:29:46,140 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10925 + total_bytes: 10925 +} + +2021-07-14 18:29:46,242 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10925 + total_bytes: 10925 +} + +2021-07-14 18:29:46,344 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10925 + total_bytes: 10925 +} + +2021-07-14 18:29:46,446 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10925 + total_bytes: 10925 +} + +2021-07-14 18:29:46,780 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10925 + total_bytes: 10925 +} + +2021-07-14 18:29:46,882 INFO MainThread:547087 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10925 + total_bytes: 10925 +} + +2021-07-14 18:29:48,173 INFO MainThread:547087 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210714_181954-351hrfn2/run-351hrfn2.wandb b/wandb/run-20210714_181954-351hrfn2/run-351hrfn2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b22bf02258b38b1782fef779bf409c7641ee8d98 Binary files /dev/null and b/wandb/run-20210714_181954-351hrfn2/run-351hrfn2.wandb differ diff --git a/wandb/run-20210714_183304-3d4rorca/files/config.yaml b/wandb/run-20210714_183304-3d4rorca/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eefe04659aa49d790c84f0f061f8d9521a7e46c1 --- /dev/null +++ b/wandb/run-20210714_183304-3d4rorca/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: bfloat16 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 40000 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 1 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul14_18-32-57_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_eval_samples: + desc: null + value: 100 +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 3.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 96 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 5000 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210714_183304-3d4rorca/files/output.log b/wandb/run-20210714_183304-3d4rorca/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a106517de3cd0f73cc4b51567e306633ed01da0a --- /dev/null +++ b/wandb/run-20210714_183304-3d4rorca/files/output.log @@ -0,0 +1,37 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1669, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 637, in xla_pmap_impl + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1159, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Attempting to reserve 11.03G at the bottom of memory. That was not possible. There are 7.51G free, 0B reserved, and 7.45G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax_no_accum.py", line 684, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 1159, in execute_replicated + out_bufs = compiled.execute_sharded_on_local_devices(input_bufs) +RuntimeError: Resource exhausted: Attempting to reserve 11.03G at the bottom of memory. That was not possible. There are 7.51G free, 0B reserved, and 7.45G reservable.: while running replica 0 and partition 0 of a replicated computation (other replicas may have failed as well). \ No newline at end of file diff --git a/wandb/run-20210714_183304-3d4rorca/files/requirements.txt b/wandb/run-20210714_183304-3d4rorca/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7500433524bd88d37f4fbd1efbe772b80bf4ab23 --- /dev/null +++ b/wandb/run-20210714_183304-3d4rorca/files/requirements.txt @@ -0,0 +1,93 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.1 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.7.0 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==3.2 +jax==0.2.17 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.26.0 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210714_183304-3d4rorca/files/wandb-metadata.json b/wandb/run-20210714_183304-3d4rorca/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b823f384c0fc2658d4496811f825ef2d2a5076fd --- /dev/null +++ b/wandb/run-20210714_183304-3d4rorca/files/wandb-metadata.json @@ -0,0 +1,48 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-14T18:33:06.772696", + "startedAt": "2021-07-14T18:33:04.781003", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=5000", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=40000", + "--num_train_epochs=3", + "--preprocessing_num_workers=96", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--max_eval_samples=100", + "--overwrite_cache", + "False", + "--dtype=bfloat16", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax_no_accum.py", + "codePath": "run_mlm_flax_no_accum.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "f6e0bf7955345b689d1a39a7ffb5ce2ba806da0b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210714_183304-3d4rorca/files/wandb-summary.json b/wandb/run-20210714_183304-3d4rorca/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210714_183304-3d4rorca/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210714_183304-3d4rorca/logs/debug-internal.log b/wandb/run-20210714_183304-3d4rorca/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7906cd7733d4db9d347aecd99cbe13dc08faa15f --- /dev/null +++ b/wandb/run-20210714_183304-3d4rorca/logs/debug-internal.log @@ -0,0 +1,234 @@ +2021-07-14 18:33:05,441 INFO MainThread:551470 [internal.py:wandb_internal():88] W&B internal server running at pid: 551470, started at: 2021-07-14 18:33:05.441281 +2021-07-14 18:33:05,443 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: check_version +2021-07-14 18:33:05,443 INFO WriterThread:551470 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/run-3d4rorca.wandb +2021-07-14 18:33:05,444 DEBUG SenderThread:551470 [sender.py:send():179] send: header +2021-07-14 18:33:05,444 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: check_version +2021-07-14 18:33:05,481 DEBUG SenderThread:551470 [sender.py:send():179] send: run +2021-07-14 18:33:05,658 INFO SenderThread:551470 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files +2021-07-14 18:33:05,658 INFO SenderThread:551470 [sender.py:_start_run_threads():716] run started: 3d4rorca with start time 1626287584 +2021-07-14 18:33:05,658 DEBUG SenderThread:551470 [sender.py:send():179] send: summary +2021-07-14 18:33:05,659 INFO SenderThread:551470 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 18:33:05,659 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: run_start +2021-07-14 18:33:06,662 INFO Thread-8 :551470 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/wandb-summary.json +2021-07-14 18:33:06,772 DEBUG HandlerThread:551470 [meta.py:__init__():39] meta init +2021-07-14 18:33:06,772 DEBUG HandlerThread:551470 [meta.py:__init__():53] meta init done +2021-07-14 18:33:06,772 DEBUG HandlerThread:551470 [meta.py:probe():210] probe +2021-07-14 18:33:06,773 DEBUG HandlerThread:551470 [meta.py:_setup_git():200] setup git +2021-07-14 18:33:06,799 DEBUG HandlerThread:551470 [meta.py:_setup_git():207] setup git done +2021-07-14 18:33:06,799 DEBUG HandlerThread:551470 [meta.py:_save_pip():57] save pip +2021-07-14 18:33:06,799 DEBUG HandlerThread:551470 [meta.py:_save_pip():71] save pip done +2021-07-14 18:33:06,799 DEBUG HandlerThread:551470 [meta.py:probe():252] probe done +2021-07-14 18:33:06,802 DEBUG SenderThread:551470 [sender.py:send():179] send: files +2021-07-14 18:33:06,802 INFO SenderThread:551470 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-14 18:33:06,808 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:33:06,808 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:33:06,939 DEBUG SenderThread:551470 [sender.py:send():179] send: config +2021-07-14 18:33:06,940 DEBUG SenderThread:551470 [sender.py:send():179] send: config +2021-07-14 18:33:06,940 DEBUG SenderThread:551470 [sender.py:send():179] send: config +2021-07-14 18:33:07,267 INFO Thread-11 :551470 [upload_job.py:push():137] Uploaded file /tmp/tmpwkna8gakwandb/1910yhst-wandb-metadata.json +2021-07-14 18:33:07,661 INFO Thread-8 :551470 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/output.log +2021-07-14 18:33:07,661 INFO Thread-8 :551470 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/requirements.txt +2021-07-14 18:33:07,661 INFO Thread-8 :551470 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/wandb-metadata.json +2021-07-14 18:33:21,941 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:33:21,941 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:33:25,667 INFO Thread-8 :551470 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/output.log +2021-07-14 18:33:34,849 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:33:36,671 INFO Thread-8 :551470 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/config.yaml +2021-07-14 18:33:37,073 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:33:37,073 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:33:52,207 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:33:52,207 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:34:04,916 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:34:07,343 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:34:07,343 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:34:22,477 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:34:22,477 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:34:34,976 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:34:37,613 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:34:37,613 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:34:52,747 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:34:52,747 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:35:05,044 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:35:07,878 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:35:07,879 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:35:23,013 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:35:23,013 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:35:35,110 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:35:38,147 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:35:38,148 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:35:53,282 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:35:53,282 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:36:05,181 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:36:08,416 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:36:08,417 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:36:23,549 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:36:23,549 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:36:35,251 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:36:38,681 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:36:38,681 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:36:53,817 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:36:53,817 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:37:05,320 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:37:08,952 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:37:08,953 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:37:24,087 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:37:24,087 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:37:35,392 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:37:39,221 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:37:39,222 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:37:54,352 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:37:54,353 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:38:05,464 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:38:09,485 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:38:09,485 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:38:24,626 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:38:24,626 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:38:35,535 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:38:39,761 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:38:39,761 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:38:54,893 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:38:54,893 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:39:05,597 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:39:10,025 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:39:10,026 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:39:25,164 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:39:25,164 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:39:35,672 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:39:40,296 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:39:40,297 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:39:55,430 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:39:55,430 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:40:05,746 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:40:10,563 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:40:10,564 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:40:25,697 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:40:25,698 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:40:35,821 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:40:40,834 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:40:40,834 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:40:55,966 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:40:55,967 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:41:05,888 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:41:11,114 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:41:11,114 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:41:11,814 INFO Thread-8 :551470 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/output.log +2021-07-14 18:41:26,276 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:41:26,277 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:41:35,966 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:41:41,417 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:41:41,418 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:41:56,548 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:41:56,548 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:42:06,047 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:42:11,683 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:42:11,684 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:42:26,818 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:42:26,818 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:42:36,121 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:42:41,950 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:42:41,951 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:42:51,640 DEBUG SenderThread:551470 [sender.py:send():179] send: telemetry +2021-07-14 18:42:51,641 DEBUG SenderThread:551470 [sender.py:send():179] send: exit +2021-07-14 18:42:51,641 INFO SenderThread:551470 [sender.py:send_exit():287] handling exit code: 1 +2021-07-14 18:42:51,641 INFO SenderThread:551470 [sender.py:send_exit():295] send defer +2021-07-14 18:42:51,641 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:51,642 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:51,642 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-14 18:42:51,642 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:51,642 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:51,642 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-14 18:42:51,642 INFO SenderThread:551470 [sender.py:transition_state():308] send defer: 1 +2021-07-14 18:42:51,643 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:51,643 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-14 18:42:51,659 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:51,660 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-14 18:42:51,660 INFO SenderThread:551470 [sender.py:transition_state():308] send defer: 2 +2021-07-14 18:42:51,660 DEBUG SenderThread:551470 [sender.py:send():179] send: stats +2021-07-14 18:42:51,660 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:51,660 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-14 18:42:51,660 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:51,660 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-14 18:42:51,660 INFO SenderThread:551470 [sender.py:transition_state():308] send defer: 3 +2021-07-14 18:42:51,661 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:51,661 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-14 18:42:51,661 DEBUG SenderThread:551470 [sender.py:send():179] send: summary +2021-07-14 18:42:51,661 INFO SenderThread:551470 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 18:42:51,661 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:51,661 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-14 18:42:51,661 INFO SenderThread:551470 [sender.py:transition_state():308] send defer: 4 +2021-07-14 18:42:51,662 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:51,662 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-14 18:42:51,662 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:51,662 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-14 18:42:51,744 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:51,838 INFO SenderThread:551470 [sender.py:transition_state():308] send defer: 5 +2021-07-14 18:42:51,838 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:51,838 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:51,838 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-14 18:42:51,839 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:51,839 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-14 18:42:51,839 INFO SenderThread:551470 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-14 18:42:51,854 INFO Thread-8 :551470 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/config.yaml +2021-07-14 18:42:51,855 INFO SenderThread:551470 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/output.log +2021-07-14 18:42:51,855 INFO SenderThread:551470 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/wandb-summary.json +2021-07-14 18:42:51,855 INFO SenderThread:551470 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files +2021-07-14 18:42:51,855 INFO SenderThread:551470 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/requirements.txt requirements.txt +2021-07-14 18:42:51,855 INFO SenderThread:551470 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/output.log output.log +2021-07-14 18:42:51,856 INFO SenderThread:551470 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/wandb-metadata.json wandb-metadata.json +2021-07-14 18:42:51,856 INFO SenderThread:551470 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/config.yaml config.yaml +2021-07-14 18:42:51,856 INFO SenderThread:551470 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/wandb-summary.json wandb-summary.json +2021-07-14 18:42:51,856 INFO SenderThread:551470 [sender.py:transition_state():308] send defer: 6 +2021-07-14 18:42:51,862 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:51,862 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-14 18:42:51,865 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:51,865 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-14 18:42:51,865 INFO SenderThread:551470 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 18:42:51,939 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:51,940 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,041 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:52,041 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,143 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:52,143 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,245 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:52,245 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,308 INFO Thread-12 :551470 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/requirements.txt +2021-07-14 18:42:52,325 INFO Thread-13 :551470 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/output.log +2021-07-14 18:42:52,341 INFO Thread-15 :551470 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/wandb-summary.json +2021-07-14 18:42:52,346 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:52,346 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,361 INFO Thread-14 :551470 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/files/config.yaml +2021-07-14 18:42:52,448 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:52,448 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,549 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:52,550 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,562 INFO Thread-7 :551470 [sender.py:transition_state():308] send defer: 7 +2021-07-14 18:42:52,562 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:52,562 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-14 18:42:52,562 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:52,562 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-14 18:42:52,651 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:52,837 INFO SenderThread:551470 [sender.py:transition_state():308] send defer: 8 +2021-07-14 18:42:52,837 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,837 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:52,837 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-14 18:42:52,838 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:52,838 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-14 18:42:52,838 INFO SenderThread:551470 [sender.py:transition_state():308] send defer: 9 +2021-07-14 18:42:52,838 DEBUG SenderThread:551470 [sender.py:send():179] send: final +2021-07-14 18:42:52,838 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: defer +2021-07-14 18:42:52,838 INFO HandlerThread:551470 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-14 18:42:52,839 DEBUG SenderThread:551470 [sender.py:send():179] send: footer +2021-07-14 18:42:52,839 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: defer +2021-07-14 18:42:52,839 INFO SenderThread:551470 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-14 18:42:52,938 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 18:42:52,939 DEBUG SenderThread:551470 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 18:42:52,939 INFO SenderThread:551470 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 18:42:52,940 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: get_summary +2021-07-14 18:42:52,941 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-14 18:42:52,941 DEBUG HandlerThread:551470 [handler.py:handle_request():124] handle_request: shutdown +2021-07-14 18:42:52,941 INFO HandlerThread:551470 [handler.py:finish():638] shutting down handler +2021-07-14 18:42:53,839 INFO WriterThread:551470 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/run-3d4rorca.wandb +2021-07-14 18:42:53,939 INFO SenderThread:551470 [sender.py:finish():945] shutting down sender +2021-07-14 18:42:53,939 INFO SenderThread:551470 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 18:42:53,939 INFO SenderThread:551470 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 18:42:53,941 INFO MainThread:551470 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210714_183304-3d4rorca/logs/debug.log b/wandb/run-20210714_183304-3d4rorca/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..34ff11b601a6daa69120926e5dd5824f37736400 --- /dev/null +++ b/wandb/run-20210714_183304-3d4rorca/logs/debug.log @@ -0,0 +1,119 @@ +2021-07-14 18:33:04,782 INFO MainThread:550207 [wandb_setup.py:_flush():69] setting env: {} +2021-07-14 18:33:04,782 INFO MainThread:550207 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-14 18:33:04,782 INFO MainThread:550207 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/logs/debug.log +2021-07-14 18:33:04,782 INFO MainThread:550207 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_183304-3d4rorca/logs/debug-internal.log +2021-07-14 18:33:04,783 INFO MainThread:550207 [wandb_init.py:init():370] calling init triggers +2021-07-14 18:33:04,783 INFO MainThread:550207 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-14 18:33:04,783 INFO MainThread:550207 [wandb_init.py:init():419] starting backend +2021-07-14 18:33:04,783 INFO MainThread:550207 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-14 18:33:04,823 INFO MainThread:550207 [backend.py:ensure_launched():135] starting backend process... +2021-07-14 18:33:04,862 INFO MainThread:550207 [backend.py:ensure_launched():139] started backend process with pid: 551470 +2021-07-14 18:33:04,864 INFO MainThread:550207 [wandb_init.py:init():424] backend started and connected +2021-07-14 18:33:04,867 INFO MainThread:550207 [wandb_init.py:init():472] updated telemetry +2021-07-14 18:33:04,868 INFO MainThread:550207 [wandb_init.py:init():491] communicating current version +2021-07-14 18:33:05,480 INFO MainThread:550207 [wandb_init.py:init():496] got version response +2021-07-14 18:33:05,480 INFO MainThread:550207 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-14 18:33:05,659 INFO MainThread:550207 [wandb_init.py:init():529] starting run threads in backend +2021-07-14 18:33:06,807 INFO MainThread:550207 [wandb_run.py:_console_start():1623] atexit reg +2021-07-14 18:33:06,807 INFO MainThread:550207 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-14 18:33:06,807 INFO MainThread:550207 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-14 18:33:06,809 INFO MainThread:550207 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-14 18:33:06,809 INFO MainThread:550207 [wandb_init.py:init():554] run started, returning control to user process +2021-07-14 18:33:06,816 INFO MainThread:550207 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_18-32-57_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-14 18:33:06,817 INFO MainThread:550207 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'} +2021-07-14 18:33:06,819 INFO MainThread:550207 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 100} +2021-07-14 18:42:49,309 INFO MainThread:550207 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-14 18:42:49,311 INFO MainThread:550207 [wandb_run.py:_restore():1565] restore +2021-07-14 18:42:51,643 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 1472 +} + +2021-07-14 18:42:51,838 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 1472 +} + +2021-07-14 18:42:51,940 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 11071 +} + +2021-07-14 18:42:52,042 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11071 + total_bytes: 11071 +} + +2021-07-14 18:42:52,144 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11071 + total_bytes: 11071 +} + +2021-07-14 18:42:52,245 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11071 + total_bytes: 11071 +} + +2021-07-14 18:42:52,347 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11071 + total_bytes: 11071 +} + +2021-07-14 18:42:52,448 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11071 + total_bytes: 11071 +} + +2021-07-14 18:42:52,550 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11071 + total_bytes: 11071 +} + +2021-07-14 18:42:52,837 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11071 + total_bytes: 11071 +} + +2021-07-14 18:42:52,939 INFO MainThread:550207 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 11071 + total_bytes: 11071 +} + +2021-07-14 18:42:54,228 INFO MainThread:550207 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210714_183304-3d4rorca/run-3d4rorca.wandb b/wandb/run-20210714_183304-3d4rorca/run-3d4rorca.wandb new file mode 100644 index 0000000000000000000000000000000000000000..670078c094559184a3d344813dc295dfa87bd16d Binary files /dev/null and b/wandb/run-20210714_183304-3d4rorca/run-3d4rorca.wandb differ diff --git a/wandb/run-20210714_185537-379h21h2/files/config.yaml b/wandb/run-20210714_185537-379h21h2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8cfbafe6d5a26cb5992c45469b5f0234111abc1 --- /dev/null +++ b/wandb/run-20210714_185537-379h21h2/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: bfloat16 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 40000 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 1 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul14_18-55-30_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_eval_samples: + desc: null + value: 100 +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 3.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 1 +per_device_train_batch_size: + desc: null + value: 1 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 96 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 5000 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210714_185537-379h21h2/files/output.log b/wandb/run-20210714_185537-379h21h2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f27cce6413083da20dd3427b25a87544419fc743 --- /dev/null +++ b/wandb/run-20210714_185537-379h21h2/files/output.log @@ -0,0 +1,32 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 + train_batch_idx = generate_batch_splits(train_samples_idx, train_batch_size) + File "./run_mlm_flax_no_accum.py", line 255, in generate_batch_splits + batch_idx = np.split(samples_idx, sections_split) + File "<__array_function__ internals>", line 5, in split + File "/home/dat/pino/lib/python3.8/site-packages/numpy/lib/shape_base.py", line 874, in split + return array_split(ary, indices_or_sections, axis) + File "<__array_function__ internals>", line 5, in array_split + File "/home/dat/pino/lib/python3.8/site-packages/numpy/lib/shape_base.py", line 790, in array_split + sub_arys.append(_nx.swapaxes(sary[st:end], axis, 0)) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py", line 5009, in _rewriting_take + return _gather(arr, treedef, static_idx, dynamic_idx, indices_are_sorted, + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py", line 5028, in _gather + y = lax.gather( + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/lax/lax.py", line 984, in gather + return gather_p.bind( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 264, in bind + out = top_trace.process_primitive(self, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 603, in process_primitive + return primitive.impl(*tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 249, in apply_primitive + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 365, in _execute_compiled_primitive + out_bufs = compiled.execute(input_bufs) +RuntimeError: Resource exhausted: Attempting to allocate 17.0K. That was not possible. There are 48.0K free. Due to fragmentation, the largest contiguous region of free memory is 16.0K.; (0x0x0_HBM0) \ No newline at end of file diff --git a/wandb/run-20210714_185537-379h21h2/files/requirements.txt b/wandb/run-20210714_185537-379h21h2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7500433524bd88d37f4fbd1efbe772b80bf4ab23 --- /dev/null +++ b/wandb/run-20210714_185537-379h21h2/files/requirements.txt @@ -0,0 +1,93 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.1 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.7.0 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==3.2 +jax==0.2.17 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.26.0 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210714_185537-379h21h2/files/wandb-metadata.json b/wandb/run-20210714_185537-379h21h2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..625e3746d93158422e42b00194cad421da427535 --- /dev/null +++ b/wandb/run-20210714_185537-379h21h2/files/wandb-metadata.json @@ -0,0 +1,48 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-14T18:55:39.338735", + "startedAt": "2021-07-14T18:55:37.305626", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=5000", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=40000", + "--num_train_epochs=3", + "--preprocessing_num_workers=96", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=1", + "--per_device_eval_batch_size=1", + "--save_total_limit=5", + "--max_eval_samples=100", + "--overwrite_cache", + "False", + "--dtype=bfloat16", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax_no_accum.py", + "codePath": "run_mlm_flax_no_accum.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "f6e0bf7955345b689d1a39a7ffb5ce2ba806da0b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210714_185537-379h21h2/files/wandb-summary.json b/wandb/run-20210714_185537-379h21h2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210714_185537-379h21h2/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210714_185537-379h21h2/logs/debug-internal.log b/wandb/run-20210714_185537-379h21h2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bf5bef6b9a4ee97542e20eed3091e068d6e4150c --- /dev/null +++ b/wandb/run-20210714_185537-379h21h2/logs/debug-internal.log @@ -0,0 +1,290 @@ +2021-07-14 18:55:37,982 INFO MainThread:555087 [internal.py:wandb_internal():88] W&B internal server running at pid: 555087, started at: 2021-07-14 18:55:37.982262 +2021-07-14 18:55:37,984 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: check_version +2021-07-14 18:55:37,985 INFO WriterThread:555087 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/run-379h21h2.wandb +2021-07-14 18:55:37,986 DEBUG SenderThread:555087 [sender.py:send():179] send: header +2021-07-14 18:55:37,986 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: check_version +2021-07-14 18:55:38,025 DEBUG SenderThread:555087 [sender.py:send():179] send: run +2021-07-14 18:55:38,206 INFO SenderThread:555087 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files +2021-07-14 18:55:38,206 INFO SenderThread:555087 [sender.py:_start_run_threads():716] run started: 379h21h2 with start time 1626288937 +2021-07-14 18:55:38,206 DEBUG SenderThread:555087 [sender.py:send():179] send: summary +2021-07-14 18:55:38,206 INFO SenderThread:555087 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 18:55:38,207 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: run_start +2021-07-14 18:55:39,207 INFO Thread-8 :555087 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/wandb-summary.json +2021-07-14 18:55:39,338 DEBUG HandlerThread:555087 [meta.py:__init__():39] meta init +2021-07-14 18:55:39,338 DEBUG HandlerThread:555087 [meta.py:__init__():53] meta init done +2021-07-14 18:55:39,338 DEBUG HandlerThread:555087 [meta.py:probe():210] probe +2021-07-14 18:55:39,340 DEBUG HandlerThread:555087 [meta.py:_setup_git():200] setup git +2021-07-14 18:55:39,371 DEBUG HandlerThread:555087 [meta.py:_setup_git():207] setup git done +2021-07-14 18:55:39,371 DEBUG HandlerThread:555087 [meta.py:_save_pip():57] save pip +2021-07-14 18:55:39,372 DEBUG HandlerThread:555087 [meta.py:_save_pip():71] save pip done +2021-07-14 18:55:39,372 DEBUG HandlerThread:555087 [meta.py:probe():252] probe done +2021-07-14 18:55:39,397 DEBUG SenderThread:555087 [sender.py:send():179] send: files +2021-07-14 18:55:39,397 INFO SenderThread:555087 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-14 18:55:39,404 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:55:39,405 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:55:39,533 DEBUG SenderThread:555087 [sender.py:send():179] send: config +2021-07-14 18:55:39,534 DEBUG SenderThread:555087 [sender.py:send():179] send: config +2021-07-14 18:55:39,534 DEBUG SenderThread:555087 [sender.py:send():179] send: config +2021-07-14 18:55:39,858 INFO Thread-11 :555087 [upload_job.py:push():137] Uploaded file /tmp/tmpywmfy6u4wandb/302tickz-wandb-metadata.json +2021-07-14 18:55:40,207 INFO Thread-8 :555087 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/output.log +2021-07-14 18:55:40,207 INFO Thread-8 :555087 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/wandb-metadata.json +2021-07-14 18:55:40,207 INFO Thread-8 :555087 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/requirements.txt +2021-07-14 18:55:54,543 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:55:54,543 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:55:58,216 INFO Thread-8 :555087 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/output.log +2021-07-14 18:56:07,426 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 18:56:09,220 INFO Thread-8 :555087 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/config.yaml +2021-07-14 18:56:09,675 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:56:09,675 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:56:24,806 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:56:24,806 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:56:37,502 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 18:56:39,942 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:56:39,942 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:56:55,073 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:56:55,073 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:57:07,579 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 18:57:10,210 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:57:10,210 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:57:25,342 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:57:25,343 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:57:37,658 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 18:57:40,474 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:57:40,475 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:57:55,605 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:57:55,606 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:58:07,732 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 18:58:10,737 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:58:10,737 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:58:25,870 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:58:25,870 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:58:37,810 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 18:58:41,005 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:58:41,005 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:58:56,137 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:58:56,138 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:59:07,886 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 18:59:11,274 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:59:11,274 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:59:26,406 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:59:26,406 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:59:37,963 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 18:59:41,543 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:59:41,543 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 18:59:56,673 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 18:59:56,674 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:00:08,040 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:00:11,805 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:00:11,806 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:00:26,938 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:00:26,938 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:00:38,115 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:00:42,071 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:00:42,073 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:00:57,206 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:00:57,206 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:01:08,189 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:01:12,338 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:01:12,338 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:01:27,470 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:01:27,470 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:01:38,253 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:01:42,602 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:01:42,602 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:01:57,736 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:01:57,736 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:02:08,323 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:02:12,869 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:02:12,869 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:02:28,002 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:02:28,002 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:02:38,399 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:02:43,167 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:02:43,167 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:02:58,298 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:02:58,299 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:03:08,473 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:03:13,435 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:03:13,435 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:03:28,570 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:03:28,570 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:03:38,550 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:03:43,702 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:03:43,702 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:03:58,838 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:03:58,838 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:04:08,624 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:04:13,976 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:04:13,976 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:04:29,120 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:04:29,121 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:04:38,701 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:04:44,257 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:04:44,258 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:04:59,397 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:04:59,397 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:05:08,777 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:05:14,531 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:05:14,532 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:05:29,687 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:05:29,688 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:05:38,851 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:05:44,819 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:05:44,820 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:05:59,950 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:05:59,951 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:06:08,925 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:06:15,090 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:06:15,091 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:06:30,226 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:06:30,226 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:06:39,001 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:06:45,360 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:06:45,360 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:07:00,496 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:07:00,496 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:07:09,079 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:07:15,632 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:07:15,633 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:07:30,767 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:07:30,767 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:07:39,157 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:07:45,903 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:07:45,903 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:08:01,041 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:08:01,041 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:08:09,235 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:08:16,178 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:08:16,179 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:08:31,336 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:08:31,336 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:08:39,312 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:08:46,469 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:08:46,470 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:09:01,600 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:09:01,601 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:09:09,391 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:09:16,736 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:09:16,736 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:09:31,869 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:09:31,870 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:09:39,454 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:09:47,008 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:09:47,008 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:10:02,139 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:10:02,140 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:10:09,532 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:10:17,277 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:10:17,277 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:10:32,410 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:10:32,410 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:10:39,609 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:10:47,545 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:10:47,545 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:11:01,425 DEBUG SenderThread:555087 [sender.py:send():179] send: telemetry +2021-07-14 19:11:01,425 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:01,425 DEBUG SenderThread:555087 [sender.py:send():179] send: exit +2021-07-14 19:11:01,426 INFO SenderThread:555087 [sender.py:send_exit():287] handling exit code: 1 +2021-07-14 19:11:01,426 INFO SenderThread:555087 [sender.py:send_exit():295] send defer +2021-07-14 19:11:01,427 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:01,427 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:01,427 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-14 19:11:01,428 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:01,428 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-14 19:11:01,428 INFO SenderThread:555087 [sender.py:transition_state():308] send defer: 1 +2021-07-14 19:11:01,428 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:01,428 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-14 19:11:01,460 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:01,460 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-14 19:11:01,460 INFO SenderThread:555087 [sender.py:transition_state():308] send defer: 2 +2021-07-14 19:11:01,460 DEBUG SenderThread:555087 [sender.py:send():179] send: stats +2021-07-14 19:11:01,461 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:01,461 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-14 19:11:01,461 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:01,461 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-14 19:11:01,461 INFO SenderThread:555087 [sender.py:transition_state():308] send defer: 3 +2021-07-14 19:11:01,461 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:01,462 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-14 19:11:01,462 DEBUG SenderThread:555087 [sender.py:send():179] send: summary +2021-07-14 19:11:01,462 INFO SenderThread:555087 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 19:11:01,462 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:01,462 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-14 19:11:01,462 INFO SenderThread:555087 [sender.py:transition_state():308] send defer: 4 +2021-07-14 19:11:01,463 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:01,463 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-14 19:11:01,463 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:01,463 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-14 19:11:01,529 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:01,566 INFO Thread-8 :555087 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/output.log +2021-07-14 19:11:01,566 INFO Thread-8 :555087 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/wandb-summary.json +2021-07-14 19:11:01,667 INFO SenderThread:555087 [sender.py:transition_state():308] send defer: 5 +2021-07-14 19:11:01,667 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:01,668 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:01,668 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-14 19:11:01,668 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:01,668 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-14 19:11:01,668 INFO SenderThread:555087 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-14 19:11:01,769 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:02,566 INFO Thread-8 :555087 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/config.yaml +2021-07-14 19:11:02,567 INFO SenderThread:555087 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files +2021-07-14 19:11:02,567 INFO SenderThread:555087 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/requirements.txt requirements.txt +2021-07-14 19:11:02,567 INFO SenderThread:555087 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/output.log output.log +2021-07-14 19:11:02,567 INFO SenderThread:555087 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/wandb-metadata.json wandb-metadata.json +2021-07-14 19:11:02,567 INFO SenderThread:555087 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/config.yaml config.yaml +2021-07-14 19:11:02,568 INFO SenderThread:555087 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/wandb-summary.json wandb-summary.json +2021-07-14 19:11:02,568 INFO SenderThread:555087 [sender.py:transition_state():308] send defer: 6 +2021-07-14 19:11:02,569 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:02,572 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:02,572 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-14 19:11:02,575 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:02,575 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-14 19:11:02,575 INFO SenderThread:555087 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 19:11:02,671 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:02,671 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:02,773 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:02,774 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:02,876 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:02,876 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:02,977 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:02,978 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:03,031 INFO Thread-14 :555087 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/config.yaml +2021-07-14 19:11:03,066 INFO Thread-13 :555087 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/output.log +2021-07-14 19:11:03,074 INFO Thread-12 :555087 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/requirements.txt +2021-07-14 19:11:03,079 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:03,079 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:03,088 INFO Thread-15 :555087 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/files/wandb-summary.json +2021-07-14 19:11:03,181 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:03,181 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:03,283 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:03,283 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:03,288 INFO Thread-7 :555087 [sender.py:transition_state():308] send defer: 7 +2021-07-14 19:11:03,288 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:03,289 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-14 19:11:03,289 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:03,289 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-14 19:11:03,385 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:03,599 INFO SenderThread:555087 [sender.py:transition_state():308] send defer: 8 +2021-07-14 19:11:03,600 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:03,600 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:03,600 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-14 19:11:03,600 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:03,600 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-14 19:11:03,601 INFO SenderThread:555087 [sender.py:transition_state():308] send defer: 9 +2021-07-14 19:11:03,601 DEBUG SenderThread:555087 [sender.py:send():179] send: final +2021-07-14 19:11:03,601 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:11:03,601 INFO HandlerThread:555087 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-14 19:11:03,601 DEBUG SenderThread:555087 [sender.py:send():179] send: footer +2021-07-14 19:11:03,602 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: defer +2021-07-14 19:11:03,602 INFO SenderThread:555087 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-14 19:11:03,702 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:11:03,702 DEBUG SenderThread:555087 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:11:03,702 INFO SenderThread:555087 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 19:11:03,704 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: get_summary +2021-07-14 19:11:03,704 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-14 19:11:03,705 DEBUG HandlerThread:555087 [handler.py:handle_request():124] handle_request: shutdown +2021-07-14 19:11:03,705 INFO HandlerThread:555087 [handler.py:finish():638] shutting down handler +2021-07-14 19:11:04,602 INFO WriterThread:555087 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/run-379h21h2.wandb +2021-07-14 19:11:04,703 INFO SenderThread:555087 [sender.py:finish():945] shutting down sender +2021-07-14 19:11:04,703 INFO SenderThread:555087 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 19:11:04,703 INFO SenderThread:555087 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 19:11:04,706 INFO MainThread:555087 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210714_185537-379h21h2/logs/debug.log b/wandb/run-20210714_185537-379h21h2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..769bc395450cb00759c4d72dc127cee022b3d4cf --- /dev/null +++ b/wandb/run-20210714_185537-379h21h2/logs/debug.log @@ -0,0 +1,127 @@ +2021-07-14 18:55:37,307 INFO MainThread:553832 [wandb_setup.py:_flush():69] setting env: {} +2021-07-14 18:55:37,307 INFO MainThread:553832 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-14 18:55:37,307 INFO MainThread:553832 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/logs/debug.log +2021-07-14 18:55:37,307 INFO MainThread:553832 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_185537-379h21h2/logs/debug-internal.log +2021-07-14 18:55:37,307 INFO MainThread:553832 [wandb_init.py:init():370] calling init triggers +2021-07-14 18:55:37,307 INFO MainThread:553832 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-14 18:55:37,307 INFO MainThread:553832 [wandb_init.py:init():419] starting backend +2021-07-14 18:55:37,307 INFO MainThread:553832 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-14 18:55:37,346 INFO MainThread:553832 [backend.py:ensure_launched():135] starting backend process... +2021-07-14 18:55:37,383 INFO MainThread:553832 [backend.py:ensure_launched():139] started backend process with pid: 555087 +2021-07-14 18:55:37,385 INFO MainThread:553832 [wandb_init.py:init():424] backend started and connected +2021-07-14 18:55:37,388 INFO MainThread:553832 [wandb_init.py:init():472] updated telemetry +2021-07-14 18:55:37,389 INFO MainThread:553832 [wandb_init.py:init():491] communicating current version +2021-07-14 18:55:38,023 INFO MainThread:553832 [wandb_init.py:init():496] got version response +2021-07-14 18:55:38,023 INFO MainThread:553832 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-14 18:55:38,206 INFO MainThread:553832 [wandb_init.py:init():529] starting run threads in backend +2021-07-14 18:55:39,401 INFO MainThread:553832 [wandb_run.py:_console_start():1623] atexit reg +2021-07-14 18:55:39,401 INFO MainThread:553832 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-14 18:55:39,402 INFO MainThread:553832 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-14 18:55:39,404 INFO MainThread:553832 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-14 18:55:39,404 INFO MainThread:553832 [wandb_init.py:init():554] run started, returning control to user process +2021-07-14 18:55:39,410 INFO MainThread:553832 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_18-55-30_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-14 18:55:39,411 INFO MainThread:553832 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'} +2021-07-14 18:55:39,413 INFO MainThread:553832 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 100} +2021-07-14 19:10:59,131 INFO MainThread:553832 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-14 19:10:59,133 INFO MainThread:553832 [wandb_run.py:_restore():1565] restore +2021-07-14 19:11:01,428 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 1472 +} + +2021-07-14 19:11:01,668 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 1472 +} + +2021-07-14 19:11:02,569 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 4 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 10689 +} + +2021-07-14 19:11:02,672 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 10691 +} + +2021-07-14 19:11:02,774 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10691 + total_bytes: 10691 +} + +2021-07-14 19:11:02,876 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10691 + total_bytes: 10691 +} + +2021-07-14 19:11:02,978 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10691 + total_bytes: 10691 +} + +2021-07-14 19:11:03,080 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10691 + total_bytes: 10691 +} + +2021-07-14 19:11:03,182 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10691 + total_bytes: 10691 +} + +2021-07-14 19:11:03,284 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10691 + total_bytes: 10691 +} + +2021-07-14 19:11:03,600 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10691 + total_bytes: 10691 +} + +2021-07-14 19:11:03,703 INFO MainThread:553832 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10691 + total_bytes: 10691 +} + +2021-07-14 19:11:05,088 INFO MainThread:553832 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210714_185537-379h21h2/run-379h21h2.wandb b/wandb/run-20210714_185537-379h21h2/run-379h21h2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ece725b78dfaedab1895220af855fc8d475c8880 Binary files /dev/null and b/wandb/run-20210714_185537-379h21h2/run-379h21h2.wandb differ diff --git a/wandb/run-20210714_192514-fdn39gsq/files/config.yaml b/wandb/run-20210714_192514-fdn39gsq/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da059a05a9d9d460d8433571d1d2c0e7656457ba --- /dev/null +++ b/wandb/run-20210714_192514-fdn39gsq/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 2: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: bfloat16 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 40000 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 1 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul14_19-25-07_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_eval_samples: + desc: null + value: 100 +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 3.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 1 +per_device_train_batch_size: + desc: null + value: 1 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 96 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 5000 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210714_192514-fdn39gsq/files/output.log b/wandb/run-20210714_192514-fdn39gsq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..97a5d2ea04f526e679895861e925fdfa811d6406 --- /dev/null +++ b/wandb/run-20210714_192514-fdn39gsq/files/output.log @@ -0,0 +1,32 @@ +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 + train_batch_idx = generate_batch_splits(train_samples_idx, train_batch_size) + File "./run_mlm_flax_no_accum.py", line 255, in generate_batch_splits + batch_idx = np.split(samples_idx, sections_split) + File "<__array_function__ internals>", line 5, in split + File "/home/dat/pino/lib/python3.8/site-packages/numpy/lib/shape_base.py", line 874, in split + return array_split(ary, indices_or_sections, axis) + File "<__array_function__ internals>", line 5, in array_split + File "/home/dat/pino/lib/python3.8/site-packages/numpy/lib/shape_base.py", line 790, in array_split + sub_arys.append(_nx.swapaxes(sary[st:end], axis, 0)) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py", line 5009, in _rewriting_take + return _gather(arr, treedef, static_idx, dynamic_idx, indices_are_sorted, + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py", line 5028, in _gather + y = lax.gather( + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/lax/lax.py", line 984, in gather + return gather_p.bind( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 264, in bind + out = top_trace.process_primitive(self, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 603, in process_primitive + return primitive.impl(*tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 249, in apply_primitive + return compiled_fun(*args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 365, in _execute_compiled_primitive + out_bufs = compiled.execute(input_bufs) +RuntimeError: Resource exhausted: Attempting to allocate 17.0K. That was not possible. There are 48.0K free. Due to fragmentation, the largest contiguous region of free memory is 16.0K.; (0x0x0_HBM0) \ No newline at end of file diff --git a/wandb/run-20210714_192514-fdn39gsq/files/requirements.txt b/wandb/run-20210714_192514-fdn39gsq/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7500433524bd88d37f4fbd1efbe772b80bf4ab23 --- /dev/null +++ b/wandb/run-20210714_192514-fdn39gsq/files/requirements.txt @@ -0,0 +1,93 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.1 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.7.0 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==3.2 +jax==0.2.17 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.26.0 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210714_192514-fdn39gsq/files/wandb-metadata.json b/wandb/run-20210714_192514-fdn39gsq/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5f82c7711bbb9aa0b1fea3ff10f150dcbf5a9da7 --- /dev/null +++ b/wandb/run-20210714_192514-fdn39gsq/files/wandb-metadata.json @@ -0,0 +1,48 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-14T19:25:16.843750", + "startedAt": "2021-07-14T19:25:14.795463", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=5000", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=500", + "--eval_steps=40000", + "--num_train_epochs=3", + "--preprocessing_num_workers=96", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=1", + "--per_device_eval_batch_size=1", + "--save_total_limit=5", + "--max_eval_samples=100", + "--overwrite_cache", + "False", + "--dtype=bfloat16", + "--adafactor" + ], + "state": "running", + "program": "./run_mlm_flax_no_accum.py", + "codePath": "run_mlm_flax_no_accum.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "f6e0bf7955345b689d1a39a7ffb5ce2ba806da0b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210714_192514-fdn39gsq/files/wandb-summary.json b/wandb/run-20210714_192514-fdn39gsq/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210714_192514-fdn39gsq/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210714_192514-fdn39gsq/logs/debug-internal.log b/wandb/run-20210714_192514-fdn39gsq/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7fded04817a04d894e6f7544439f80b4beb7417f --- /dev/null +++ b/wandb/run-20210714_192514-fdn39gsq/logs/debug-internal.log @@ -0,0 +1,292 @@ +2021-07-14 19:25:15,501 INFO MainThread:558812 [internal.py:wandb_internal():88] W&B internal server running at pid: 558812, started at: 2021-07-14 19:25:15.501475 +2021-07-14 19:25:15,503 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: check_version +2021-07-14 19:25:15,504 INFO WriterThread:558812 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/run-fdn39gsq.wandb +2021-07-14 19:25:15,505 DEBUG SenderThread:558812 [sender.py:send():179] send: header +2021-07-14 19:25:15,505 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: check_version +2021-07-14 19:25:15,550 DEBUG SenderThread:558812 [sender.py:send():179] send: run +2021-07-14 19:25:15,726 INFO SenderThread:558812 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files +2021-07-14 19:25:15,726 INFO SenderThread:558812 [sender.py:_start_run_threads():716] run started: fdn39gsq with start time 1626290714 +2021-07-14 19:25:15,726 DEBUG SenderThread:558812 [sender.py:send():179] send: summary +2021-07-14 19:25:15,726 INFO SenderThread:558812 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 19:25:15,727 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: run_start +2021-07-14 19:25:16,728 INFO Thread-8 :558812 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/wandb-summary.json +2021-07-14 19:25:16,843 DEBUG HandlerThread:558812 [meta.py:__init__():39] meta init +2021-07-14 19:25:16,843 DEBUG HandlerThread:558812 [meta.py:__init__():53] meta init done +2021-07-14 19:25:16,843 DEBUG HandlerThread:558812 [meta.py:probe():210] probe +2021-07-14 19:25:16,844 DEBUG HandlerThread:558812 [meta.py:_setup_git():200] setup git +2021-07-14 19:25:16,874 DEBUG HandlerThread:558812 [meta.py:_setup_git():207] setup git done +2021-07-14 19:25:16,874 DEBUG HandlerThread:558812 [meta.py:_save_pip():57] save pip +2021-07-14 19:25:16,875 DEBUG HandlerThread:558812 [meta.py:_save_pip():71] save pip done +2021-07-14 19:25:16,875 DEBUG HandlerThread:558812 [meta.py:probe():252] probe done +2021-07-14 19:25:16,878 DEBUG SenderThread:558812 [sender.py:send():179] send: files +2021-07-14 19:25:16,878 INFO SenderThread:558812 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-14 19:25:16,883 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:25:16,884 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:25:17,012 DEBUG SenderThread:558812 [sender.py:send():179] send: config +2021-07-14 19:25:17,013 DEBUG SenderThread:558812 [sender.py:send():179] send: config +2021-07-14 19:25:17,013 DEBUG SenderThread:558812 [sender.py:send():179] send: config +2021-07-14 19:25:17,401 INFO Thread-11 :558812 [upload_job.py:push():137] Uploaded file /tmp/tmp2t3dxezvwandb/1xrc7l9y-wandb-metadata.json +2021-07-14 19:25:17,727 INFO Thread-8 :558812 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/requirements.txt +2021-07-14 19:25:17,727 INFO Thread-8 :558812 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/wandb-metadata.json +2021-07-14 19:25:17,727 INFO Thread-8 :558812 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/output.log +2021-07-14 19:25:32,014 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:25:32,014 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:25:35,734 INFO Thread-8 :558812 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/output.log +2021-07-14 19:25:44,927 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:25:46,738 INFO Thread-8 :558812 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/config.yaml +2021-07-14 19:25:47,145 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:25:47,146 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:26:02,275 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:26:02,276 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:26:14,992 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:26:17,407 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:26:17,407 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:26:32,541 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:26:32,541 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:26:45,071 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:26:47,675 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:26:47,676 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:27:02,817 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:27:02,817 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:27:15,150 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:27:17,954 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:27:17,955 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:27:33,130 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:27:33,130 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:27:45,227 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:27:48,264 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:27:48,264 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:28:03,396 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:28:03,397 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:28:15,303 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:28:18,531 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:28:18,532 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:28:33,662 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:28:33,662 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:28:45,377 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:28:48,794 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:28:48,795 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:29:03,929 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:29:03,929 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:29:15,441 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:29:19,068 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:29:19,068 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:29:34,207 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:29:34,207 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:29:45,507 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:29:49,342 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:29:49,342 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:30:04,484 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:30:04,485 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:30:15,573 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:30:19,618 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:30:19,619 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:30:34,752 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:30:34,753 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:30:45,637 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:30:49,884 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:30:49,885 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:31:05,016 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:31:05,017 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:31:15,702 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:31:20,150 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:31:20,150 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:31:35,285 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:31:35,285 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:31:45,768 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:31:50,417 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:31:50,418 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:32:05,556 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:32:05,557 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:32:15,834 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:32:20,694 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:32:20,695 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:32:35,831 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:32:35,831 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:32:45,903 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:32:50,966 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:32:50,967 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:33:06,100 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:33:06,100 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:33:15,972 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:33:21,242 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:33:21,242 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:33:36,374 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:33:36,374 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:33:46,047 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:33:51,520 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:33:51,520 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:34:06,655 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:34:06,655 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:34:16,121 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:34:21,789 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:34:21,790 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:34:36,924 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:34:36,925 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:34:46,200 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:34:52,186 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:34:52,187 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:35:07,323 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:35:07,323 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:35:16,277 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:35:22,455 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:35:22,455 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:35:37,590 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:35:37,590 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:35:46,355 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:35:52,727 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:35:52,727 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:36:07,858 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:36:07,858 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:36:16,432 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:36:22,989 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:36:22,989 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:36:38,123 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:36:38,124 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:36:46,506 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:36:53,255 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:36:53,255 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:37:08,418 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:37:08,419 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:37:16,581 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:37:23,567 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:37:23,567 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:37:38,702 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:37:38,702 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:37:46,660 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:37:53,835 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:37:53,835 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:38:08,972 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:38:08,972 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:38:16,734 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:38:24,103 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:38:24,103 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:38:39,236 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:38:39,236 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:38:46,811 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:38:54,369 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:38:54,369 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:39:09,502 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:39:09,503 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:39:16,888 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:39:24,635 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:39:24,635 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:39:39,766 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:39:39,766 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:39:46,967 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:39:54,898 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:39:54,899 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:40:10,033 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:40:10,033 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:40:17,045 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:40:25,168 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:40:25,168 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:40:40,300 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 19:40:40,300 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: stop_status +2021-07-14 19:40:47,122 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:40:49,097 INFO Thread-8 :558812 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/output.log +2021-07-14 19:40:50,137 DEBUG SenderThread:558812 [sender.py:send():179] send: telemetry +2021-07-14 19:40:50,137 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:50,137 DEBUG SenderThread:558812 [sender.py:send():179] send: exit +2021-07-14 19:40:50,137 INFO SenderThread:558812 [sender.py:send_exit():287] handling exit code: 1 +2021-07-14 19:40:50,138 INFO SenderThread:558812 [sender.py:send_exit():295] send defer +2021-07-14 19:40:50,138 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:50,138 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:50,139 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-14 19:40:50,139 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:50,139 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-14 19:40:50,139 INFO SenderThread:558812 [sender.py:transition_state():308] send defer: 1 +2021-07-14 19:40:50,139 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:50,139 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-14 19:40:50,231 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:50,231 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-14 19:40:50,231 INFO SenderThread:558812 [sender.py:transition_state():308] send defer: 2 +2021-07-14 19:40:50,231 DEBUG SenderThread:558812 [sender.py:send():179] send: stats +2021-07-14 19:40:50,232 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:50,232 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-14 19:40:50,232 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:50,233 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-14 19:40:50,233 INFO SenderThread:558812 [sender.py:transition_state():308] send defer: 3 +2021-07-14 19:40:50,233 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:50,233 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-14 19:40:50,233 DEBUG SenderThread:558812 [sender.py:send():179] send: summary +2021-07-14 19:40:50,234 INFO SenderThread:558812 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 19:40:50,234 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:50,234 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-14 19:40:50,234 INFO SenderThread:558812 [sender.py:transition_state():308] send defer: 4 +2021-07-14 19:40:50,234 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:50,234 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-14 19:40:50,234 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:50,235 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-14 19:40:50,240 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:50,432 INFO SenderThread:558812 [sender.py:transition_state():308] send defer: 5 +2021-07-14 19:40:50,432 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:50,433 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:50,433 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-14 19:40:50,433 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:50,433 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-14 19:40:50,433 INFO SenderThread:558812 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-14 19:40:50,534 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:51,098 INFO Thread-8 :558812 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/wandb-summary.json +2021-07-14 19:40:51,099 INFO SenderThread:558812 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/output.log +2021-07-14 19:40:51,099 INFO SenderThread:558812 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/config.yaml +2021-07-14 19:40:51,099 INFO SenderThread:558812 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files +2021-07-14 19:40:51,099 INFO SenderThread:558812 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/requirements.txt requirements.txt +2021-07-14 19:40:51,100 INFO SenderThread:558812 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/output.log output.log +2021-07-14 19:40:51,100 INFO SenderThread:558812 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/wandb-metadata.json wandb-metadata.json +2021-07-14 19:40:51,100 INFO SenderThread:558812 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/config.yaml config.yaml +2021-07-14 19:40:51,101 INFO SenderThread:558812 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/wandb-summary.json wandb-summary.json +2021-07-14 19:40:51,101 INFO SenderThread:558812 [sender.py:transition_state():308] send defer: 6 +2021-07-14 19:40:51,104 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:51,110 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:51,110 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-14 19:40:51,113 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:51,114 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-14 19:40:51,114 INFO SenderThread:558812 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 19:40:51,206 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:51,206 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:51,309 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:51,309 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:51,411 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:51,411 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:51,512 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:51,513 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:51,533 INFO Thread-14 :558812 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/config.yaml +2021-07-14 19:40:51,534 INFO Thread-12 :558812 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/requirements.txt +2021-07-14 19:40:51,535 INFO Thread-15 :558812 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/wandb-summary.json +2021-07-14 19:40:51,537 INFO Thread-13 :558812 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/files/output.log +2021-07-14 19:40:51,614 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:51,615 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:51,716 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:51,717 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:51,737 INFO Thread-7 :558812 [sender.py:transition_state():308] send defer: 7 +2021-07-14 19:40:51,738 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:51,738 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-14 19:40:51,738 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:51,738 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-14 19:40:51,818 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:52,030 INFO SenderThread:558812 [sender.py:transition_state():308] send defer: 8 +2021-07-14 19:40:52,030 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:52,031 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:52,031 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-14 19:40:52,031 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:52,031 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-14 19:40:52,031 INFO SenderThread:558812 [sender.py:transition_state():308] send defer: 9 +2021-07-14 19:40:52,032 DEBUG SenderThread:558812 [sender.py:send():179] send: final +2021-07-14 19:40:52,032 DEBUG SenderThread:558812 [sender.py:send():179] send: footer +2021-07-14 19:40:52,032 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: defer +2021-07-14 19:40:52,032 INFO HandlerThread:558812 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-14 19:40:52,032 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: defer +2021-07-14 19:40:52,032 INFO SenderThread:558812 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-14 19:40:52,132 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 19:40:52,133 DEBUG SenderThread:558812 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 19:40:52,133 INFO SenderThread:558812 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 19:40:52,134 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: get_summary +2021-07-14 19:40:52,135 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-14 19:40:52,135 DEBUG HandlerThread:558812 [handler.py:handle_request():124] handle_request: shutdown +2021-07-14 19:40:52,135 INFO HandlerThread:558812 [handler.py:finish():638] shutting down handler +2021-07-14 19:40:53,033 INFO WriterThread:558812 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/run-fdn39gsq.wandb +2021-07-14 19:40:53,133 INFO SenderThread:558812 [sender.py:finish():945] shutting down sender +2021-07-14 19:40:53,133 INFO SenderThread:558812 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 19:40:53,133 INFO SenderThread:558812 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 19:40:53,136 INFO MainThread:558812 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210714_192514-fdn39gsq/logs/debug.log b/wandb/run-20210714_192514-fdn39gsq/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ec3d1144139d28973edd05d2ee8a586240ae297e --- /dev/null +++ b/wandb/run-20210714_192514-fdn39gsq/logs/debug.log @@ -0,0 +1,119 @@ +2021-07-14 19:25:14,797 INFO MainThread:557554 [wandb_setup.py:_flush():69] setting env: {} +2021-07-14 19:25:14,797 INFO MainThread:557554 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-14 19:25:14,797 INFO MainThread:557554 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/logs/debug.log +2021-07-14 19:25:14,797 INFO MainThread:557554 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_192514-fdn39gsq/logs/debug-internal.log +2021-07-14 19:25:14,797 INFO MainThread:557554 [wandb_init.py:init():370] calling init triggers +2021-07-14 19:25:14,797 INFO MainThread:557554 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-14 19:25:14,797 INFO MainThread:557554 [wandb_init.py:init():419] starting backend +2021-07-14 19:25:14,797 INFO MainThread:557554 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-14 19:25:14,843 INFO MainThread:557554 [backend.py:ensure_launched():135] starting backend process... +2021-07-14 19:25:14,887 INFO MainThread:557554 [backend.py:ensure_launched():139] started backend process with pid: 558812 +2021-07-14 19:25:14,889 INFO MainThread:557554 [wandb_init.py:init():424] backend started and connected +2021-07-14 19:25:14,892 INFO MainThread:557554 [wandb_init.py:init():472] updated telemetry +2021-07-14 19:25:14,893 INFO MainThread:557554 [wandb_init.py:init():491] communicating current version +2021-07-14 19:25:15,549 INFO MainThread:557554 [wandb_init.py:init():496] got version response +2021-07-14 19:25:15,549 INFO MainThread:557554 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-14 19:25:15,726 INFO MainThread:557554 [wandb_init.py:init():529] starting run threads in backend +2021-07-14 19:25:16,881 INFO MainThread:557554 [wandb_run.py:_console_start():1623] atexit reg +2021-07-14 19:25:16,881 INFO MainThread:557554 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-14 19:25:16,882 INFO MainThread:557554 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-14 19:25:16,884 INFO MainThread:557554 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-14 19:25:16,884 INFO MainThread:557554 [wandb_init.py:init():554] run started, returning control to user process +2021-07-14 19:25:16,890 INFO MainThread:557554 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_19-25-07_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-14 19:25:16,892 INFO MainThread:557554 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'} +2021-07-14 19:25:16,893 INFO MainThread:557554 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 100} +2021-07-14 19:40:47,601 INFO MainThread:557554 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-14 19:40:47,603 INFO MainThread:557554 [wandb_run.py:_restore():1565] restore +2021-07-14 19:40:50,139 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 1472 +} + +2021-07-14 19:40:50,433 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 1472 +} + +2021-07-14 19:40:51,105 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 3 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 5986 +} + +2021-07-14 19:40:51,207 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1472 + total_bytes: 10555 +} + +2021-07-14 19:40:51,309 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10555 + total_bytes: 10555 +} + +2021-07-14 19:40:51,411 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10555 + total_bytes: 10555 +} + +2021-07-14 19:40:51,513 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10555 + total_bytes: 10555 +} + +2021-07-14 19:40:51,615 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10555 + total_bytes: 10555 +} + +2021-07-14 19:40:51,717 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10555 + total_bytes: 10555 +} + +2021-07-14 19:40:52,031 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10555 + total_bytes: 10555 +} + +2021-07-14 19:40:52,133 INFO MainThread:557554 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 10555 + total_bytes: 10555 +} + +2021-07-14 19:40:53,438 INFO MainThread:557554 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210714_192514-fdn39gsq/run-fdn39gsq.wandb b/wandb/run-20210714_192514-fdn39gsq/run-fdn39gsq.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5f6e1f7b8f4406db1a845707a03b5b4cc4ca8be5 Binary files /dev/null and b/wandb/run-20210714_192514-fdn39gsq/run-fdn39gsq.wandb differ diff --git a/wandb/run-20210714_194801-35ol5oes/files/config.yaml b/wandb/run-20210714_194801-35ol5oes/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b296027eda7da92b66c3ff9e63fcef7c20e851a1 --- /dev/null +++ b/wandb/run-20210714_194801-35ol5oes/files/config.yaml @@ -0,0 +1,301 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: bfloat16 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 40000 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 1 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul14_19-47-54_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_eval_samples: + desc: null + value: 5000 +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 3.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 96 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 5000 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210714_194801-35ol5oes/files/output.log b/wandb/run-20210714_194801-35ol5oes/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9927b01b53cb76728ae4bbccd9aa35e745c837ed --- /dev/null +++ b/wandb/run-20210714_194801-35ol5oes/files/output.log @@ -0,0 +1,283 @@ +[19:48:19] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`. +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 + optax.clip_grad_by_global_norm(1.), +AttributeError: module 'optax' has no attribute 'clip_grad_by_global_norm' \ No newline at end of file diff --git a/wandb/run-20210714_200107-3qnuldhi/files/requirements.txt b/wandb/run-20210714_200107-3qnuldhi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7500433524bd88d37f4fbd1efbe772b80bf4ab23 --- /dev/null +++ b/wandb/run-20210714_200107-3qnuldhi/files/requirements.txt @@ -0,0 +1,93 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.1 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.7.0 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==3.2 +jax==0.2.17 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.26.0 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210714_200107-3qnuldhi/files/wandb-metadata.json b/wandb/run-20210714_200107-3qnuldhi/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3be4253317f67e51535fe6abc8d1b9fbc8c6d2b1 --- /dev/null +++ b/wandb/run-20210714_200107-3qnuldhi/files/wandb-metadata.json @@ -0,0 +1,47 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-14T20:01:09.340419", + "startedAt": "2021-07-14T20:01:07.295786", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=5000", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=250", + "--eval_steps=40000", + "--num_train_epochs=3", + "--preprocessing_num_workers=96", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--max_eval_samples=5000", + "--overwrite_cache", + "False", + "--gradient_accumulation_steps=2" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "f6e0bf7955345b689d1a39a7ffb5ce2ba806da0b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210714_200107-3qnuldhi/files/wandb-summary.json b/wandb/run-20210714_200107-3qnuldhi/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210714_200107-3qnuldhi/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210714_200107-3qnuldhi/logs/debug-internal.log b/wandb/run-20210714_200107-3qnuldhi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8baef046d8c8e4ef0bede2110c2f173338c00183 --- /dev/null +++ b/wandb/run-20210714_200107-3qnuldhi/logs/debug-internal.log @@ -0,0 +1,139 @@ +2021-07-14 20:01:08,001 INFO MainThread:564646 [internal.py:wandb_internal():88] W&B internal server running at pid: 564646, started at: 2021-07-14 20:01:08.000744 +2021-07-14 20:01:08,003 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: check_version +2021-07-14 20:01:08,003 INFO WriterThread:564646 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/run-3qnuldhi.wandb +2021-07-14 20:01:08,004 DEBUG SenderThread:564646 [sender.py:send():179] send: header +2021-07-14 20:01:08,004 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: check_version +2021-07-14 20:01:08,042 DEBUG SenderThread:564646 [sender.py:send():179] send: run +2021-07-14 20:01:08,213 INFO SenderThread:564646 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files +2021-07-14 20:01:08,213 INFO SenderThread:564646 [sender.py:_start_run_threads():716] run started: 3qnuldhi with start time 1626292867 +2021-07-14 20:01:08,213 DEBUG SenderThread:564646 [sender.py:send():179] send: summary +2021-07-14 20:01:08,214 INFO SenderThread:564646 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:01:08,214 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: run_start +2021-07-14 20:01:09,216 INFO Thread-8 :564646 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/wandb-summary.json +2021-07-14 20:01:09,340 DEBUG HandlerThread:564646 [meta.py:__init__():39] meta init +2021-07-14 20:01:09,340 DEBUG HandlerThread:564646 [meta.py:__init__():53] meta init done +2021-07-14 20:01:09,340 DEBUG HandlerThread:564646 [meta.py:probe():210] probe +2021-07-14 20:01:09,342 DEBUG HandlerThread:564646 [meta.py:_setup_git():200] setup git +2021-07-14 20:01:09,376 DEBUG HandlerThread:564646 [meta.py:_setup_git():207] setup git done +2021-07-14 20:01:09,376 DEBUG HandlerThread:564646 [meta.py:_save_pip():57] save pip +2021-07-14 20:01:09,377 DEBUG HandlerThread:564646 [meta.py:_save_pip():71] save pip done +2021-07-14 20:01:09,377 DEBUG HandlerThread:564646 [meta.py:probe():252] probe done +2021-07-14 20:01:09,380 DEBUG SenderThread:564646 [sender.py:send():179] send: files +2021-07-14 20:01:09,381 INFO SenderThread:564646 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-14 20:01:09,388 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:01:09,388 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:01:09,525 DEBUG SenderThread:564646 [sender.py:send():179] send: config +2021-07-14 20:01:09,526 DEBUG SenderThread:564646 [sender.py:send():179] send: config +2021-07-14 20:01:09,526 DEBUG SenderThread:564646 [sender.py:send():179] send: config +2021-07-14 20:01:09,828 INFO Thread-11 :564646 [upload_job.py:push():137] Uploaded file /tmp/tmp6zvg4jijwandb/288ietle-wandb-metadata.json +2021-07-14 20:01:10,214 INFO Thread-8 :564646 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/requirements.txt +2021-07-14 20:01:10,215 INFO Thread-8 :564646 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/output.log +2021-07-14 20:01:10,215 INFO Thread-8 :564646 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/wandb-metadata.json +2021-07-14 20:01:24,220 INFO Thread-8 :564646 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/output.log +2021-07-14 20:01:24,538 DEBUG SenderThread:564646 [sender.py:send():179] send: telemetry +2021-07-14 20:01:24,539 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:24,539 DEBUG SenderThread:564646 [sender.py:send():179] send: exit +2021-07-14 20:01:24,539 INFO SenderThread:564646 [sender.py:send_exit():287] handling exit code: 1 +2021-07-14 20:01:24,539 INFO SenderThread:564646 [sender.py:send_exit():295] send defer +2021-07-14 20:01:24,540 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:24,540 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:24,540 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-14 20:01:24,541 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:24,541 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-14 20:01:24,541 INFO SenderThread:564646 [sender.py:transition_state():308] send defer: 1 +2021-07-14 20:01:24,541 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:24,541 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-14 20:01:24,590 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:24,590 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-14 20:01:24,590 INFO SenderThread:564646 [sender.py:transition_state():308] send defer: 2 +2021-07-14 20:01:24,590 DEBUG SenderThread:564646 [sender.py:send():179] send: stats +2021-07-14 20:01:24,591 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:24,591 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-14 20:01:24,591 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:24,591 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-14 20:01:24,591 INFO SenderThread:564646 [sender.py:transition_state():308] send defer: 3 +2021-07-14 20:01:24,591 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:24,591 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-14 20:01:24,592 DEBUG SenderThread:564646 [sender.py:send():179] send: summary +2021-07-14 20:01:24,592 INFO SenderThread:564646 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:01:24,592 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:24,592 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-14 20:01:24,592 INFO SenderThread:564646 [sender.py:transition_state():308] send defer: 4 +2021-07-14 20:01:24,593 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:24,593 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-14 20:01:24,593 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:24,593 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-14 20:01:24,642 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:24,785 INFO SenderThread:564646 [sender.py:transition_state():308] send defer: 5 +2021-07-14 20:01:24,786 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:24,786 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:24,786 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-14 20:01:24,787 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:24,787 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-14 20:01:24,787 INFO SenderThread:564646 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-14 20:01:24,888 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:25,221 INFO Thread-8 :564646 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/wandb-summary.json +2021-07-14 20:01:25,221 INFO SenderThread:564646 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/config.yaml +2021-07-14 20:01:25,222 INFO SenderThread:564646 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/output.log +2021-07-14 20:01:25,222 INFO SenderThread:564646 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files +2021-07-14 20:01:25,222 INFO SenderThread:564646 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/requirements.txt requirements.txt +2021-07-14 20:01:25,222 INFO SenderThread:564646 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/output.log output.log +2021-07-14 20:01:25,223 INFO SenderThread:564646 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/wandb-metadata.json wandb-metadata.json +2021-07-14 20:01:25,223 INFO SenderThread:564646 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/config.yaml config.yaml +2021-07-14 20:01:25,223 INFO SenderThread:564646 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/wandb-summary.json wandb-summary.json +2021-07-14 20:01:25,223 INFO SenderThread:564646 [sender.py:transition_state():308] send defer: 6 +2021-07-14 20:01:25,223 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:25,227 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:25,227 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-14 20:01:25,230 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:25,230 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-14 20:01:25,230 INFO SenderThread:564646 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 20:01:25,328 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:25,328 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:25,430 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:25,431 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:25,533 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:25,533 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:25,635 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:25,635 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:25,672 INFO Thread-14 :564646 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/config.yaml +2021-07-14 20:01:25,678 INFO Thread-15 :564646 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/wandb-summary.json +2021-07-14 20:01:25,692 INFO Thread-13 :564646 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/output.log +2021-07-14 20:01:25,737 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:25,737 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:25,743 INFO Thread-12 :564646 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/files/requirements.txt +2021-07-14 20:01:25,839 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:25,839 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:25,941 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:25,941 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:25,943 INFO Thread-7 :564646 [sender.py:transition_state():308] send defer: 7 +2021-07-14 20:01:25,944 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:25,944 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-14 20:01:25,944 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:25,944 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-14 20:01:26,043 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:26,074 INFO SenderThread:564646 [sender.py:transition_state():308] send defer: 8 +2021-07-14 20:01:26,074 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:26,075 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:26,075 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-14 20:01:26,075 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:26,075 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-14 20:01:26,075 INFO SenderThread:564646 [sender.py:transition_state():308] send defer: 9 +2021-07-14 20:01:26,076 DEBUG SenderThread:564646 [sender.py:send():179] send: final +2021-07-14 20:01:26,076 DEBUG SenderThread:564646 [sender.py:send():179] send: footer +2021-07-14 20:01:26,076 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: defer +2021-07-14 20:01:26,076 INFO HandlerThread:564646 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-14 20:01:26,076 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: defer +2021-07-14 20:01:26,076 INFO SenderThread:564646 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-14 20:01:26,176 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 20:01:26,177 DEBUG SenderThread:564646 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 20:01:26,177 INFO SenderThread:564646 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 20:01:26,178 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: get_summary +2021-07-14 20:01:26,179 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-14 20:01:26,180 DEBUG HandlerThread:564646 [handler.py:handle_request():124] handle_request: shutdown +2021-07-14 20:01:26,180 INFO HandlerThread:564646 [handler.py:finish():638] shutting down handler +2021-07-14 20:01:27,076 INFO WriterThread:564646 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/run-3qnuldhi.wandb +2021-07-14 20:01:27,177 INFO SenderThread:564646 [sender.py:finish():945] shutting down sender +2021-07-14 20:01:27,177 INFO SenderThread:564646 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 20:01:27,177 INFO SenderThread:564646 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 20:01:27,180 INFO MainThread:564646 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210714_200107-3qnuldhi/logs/debug.log b/wandb/run-20210714_200107-3qnuldhi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e244bb17699667567e7b6748688f4c405fbf3cbb --- /dev/null +++ b/wandb/run-20210714_200107-3qnuldhi/logs/debug.log @@ -0,0 +1,127 @@ +2021-07-14 20:01:07,297 INFO MainThread:563390 [wandb_setup.py:_flush():69] setting env: {} +2021-07-14 20:01:07,297 INFO MainThread:563390 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-14 20:01:07,297 INFO MainThread:563390 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/logs/debug.log +2021-07-14 20:01:07,297 INFO MainThread:563390 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_200107-3qnuldhi/logs/debug-internal.log +2021-07-14 20:01:07,297 INFO MainThread:563390 [wandb_init.py:init():370] calling init triggers +2021-07-14 20:01:07,297 INFO MainThread:563390 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-14 20:01:07,297 INFO MainThread:563390 [wandb_init.py:init():419] starting backend +2021-07-14 20:01:07,298 INFO MainThread:563390 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-14 20:01:07,344 INFO MainThread:563390 [backend.py:ensure_launched():135] starting backend process... +2021-07-14 20:01:07,390 INFO MainThread:563390 [backend.py:ensure_launched():139] started backend process with pid: 564646 +2021-07-14 20:01:07,392 INFO MainThread:563390 [wandb_init.py:init():424] backend started and connected +2021-07-14 20:01:07,395 INFO MainThread:563390 [wandb_init.py:init():472] updated telemetry +2021-07-14 20:01:07,396 INFO MainThread:563390 [wandb_init.py:init():491] communicating current version +2021-07-14 20:01:08,041 INFO MainThread:563390 [wandb_init.py:init():496] got version response +2021-07-14 20:01:08,041 INFO MainThread:563390 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-14 20:01:08,214 INFO MainThread:563390 [wandb_init.py:init():529] starting run threads in backend +2021-07-14 20:01:09,385 INFO MainThread:563390 [wandb_run.py:_console_start():1623] atexit reg +2021-07-14 20:01:09,386 INFO MainThread:563390 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-14 20:01:09,386 INFO MainThread:563390 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-14 20:01:09,388 INFO MainThread:563390 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-14 20:01:09,388 INFO MainThread:563390 [wandb_init.py:init():554] run started, returning control to user process +2021-07-14 20:01:09,395 INFO MainThread:563390 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_20-00-58_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-14 20:01:09,396 INFO MainThread:563390 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True} +2021-07-14 20:01:09,398 INFO MainThread:563390 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000} +2021-07-14 20:01:21,935 INFO MainThread:563390 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-14 20:01:21,936 INFO MainThread:563390 [wandb_run.py:_restore():1565] restore +2021-07-14 20:01:24,540 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1447 + total_bytes: 1447 +} + +2021-07-14 20:01:24,787 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1447 + total_bytes: 1447 +} + +2021-07-14 20:01:25,227 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 2 +} +pusher_stats { + uploaded_bytes: 1447 + total_bytes: 3082 +} + +2021-07-14 20:01:25,329 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1447 + total_bytes: 8058 +} + +2021-07-14 20:01:25,431 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 8058 + total_bytes: 8058 +} + +2021-07-14 20:01:25,534 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 8058 + total_bytes: 8058 +} + +2021-07-14 20:01:25,636 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 8058 + total_bytes: 8058 +} + +2021-07-14 20:01:25,738 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 8058 + total_bytes: 8058 +} + +2021-07-14 20:01:25,840 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 8058 + total_bytes: 8058 +} + +2021-07-14 20:01:25,942 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 8058 + total_bytes: 8058 +} + +2021-07-14 20:01:26,075 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 8058 + total_bytes: 8058 +} + +2021-07-14 20:01:26,177 INFO MainThread:563390 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 8058 + total_bytes: 8058 +} + +2021-07-14 20:01:27,458 INFO MainThread:563390 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210714_200107-3qnuldhi/run-3qnuldhi.wandb b/wandb/run-20210714_200107-3qnuldhi/run-3qnuldhi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d8ff66b7974ca94f27f77837f1ba5986aeb66f19 Binary files /dev/null and b/wandb/run-20210714_200107-3qnuldhi/run-3qnuldhi.wandb differ diff --git a/wandb/run-20210714_200731-1nr232tb/files/config.yaml b/wandb/run-20210714_200731-1nr232tb/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d2b65b69653e32d44eaea4effe9fd30d405e328 --- /dev/null +++ b/wandb/run-20210714_200731-1nr232tb/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: false +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 40000 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 2 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul14_20-07-23_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 250 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_eval_samples: + desc: null + value: 5000 +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 3.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 96 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_optimizer: + desc: null + value: true +save_steps: + desc: null + value: 20000 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 5000 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210714_200731-1nr232tb/files/output.log b/wandb/run-20210714_200731-1nr232tb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c8616b0eabf3c8086bf15171a22198ff972d8b79 --- /dev/null +++ b/wandb/run-20210714_200731-1nr232tb/files/output.log @@ -0,0 +1,588 @@ +[20:07:45] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`. +/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3132: UserWarning: Explicitly requested dtype requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 + samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx] + File "./run_mlm_flax.py", line 785, in + samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx] + File "/home/dat/datasets/src/datasets/arrow_dataset.py", line 1517, in __getitem__ + return self._getitem( + File "/home/dat/datasets/src/datasets/arrow_dataset.py", line 1510, in _getitem + formatted_output = format_table( + File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 414, in format_table + return formatter(pa_table, query_type=query_type) + File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 194, in __call__ + return self.format_row(pa_table) + File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 223, in format_row + return self.python_arrow_extractor().extract_row(pa_table) + File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 134, in extract_row + return _unnest(pa_table.to_pydict()) diff --git a/wandb/run-20210714_200731-1nr232tb/files/requirements.txt b/wandb/run-20210714_200731-1nr232tb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..554e3a34bc91714a8462f65295a12e9a04537637 --- /dev/null +++ b/wandb/run-20210714_200731-1nr232tb/files/requirements.txt @@ -0,0 +1,94 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.1 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.7.0 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==3.2 +install==1.3.4 +jax==0.2.17 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.26.0 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210714_200731-1nr232tb/files/wandb-metadata.json b/wandb/run-20210714_200731-1nr232tb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8b9d5490ea9f69012b47ec8cd2c7d9d193b02390 --- /dev/null +++ b/wandb/run-20210714_200731-1nr232tb/files/wandb-metadata.json @@ -0,0 +1,47 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-14T20:07:33.069869", + "startedAt": "2021-07-14T20:07:31.018475", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=5000", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=250", + "--eval_steps=40000", + "--num_train_epochs=3", + "--preprocessing_num_workers=96", + "--save_steps=20000", + "--learning_rate=5e-5", + "--per_device_train_batch_size=2", + "--per_device_eval_batch_size=2", + "--save_total_limit=5", + "--max_eval_samples=5000", + "--overwrite_cache", + "False", + "--gradient_accumulation_steps=2" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "f6e0bf7955345b689d1a39a7ffb5ce2ba806da0b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json b/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..88690b86d5179c2d909dafaa8bdd7847c4f9c4b1 --- /dev/null +++ b/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json @@ -0,0 +1 @@ +{"training_step": 1750, "learning_rate": 8.75000114319846e-06, "train_loss": 7.591007232666016, "_runtime": 1373, "_timestamp": 1626294624, "_step": 6} \ No newline at end of file diff --git a/wandb/run-20210714_200731-1nr232tb/logs/debug-internal.log b/wandb/run-20210714_200731-1nr232tb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..cb37ba22816e4321300a208d1b218281740235f2 --- /dev/null +++ b/wandb/run-20210714_200731-1nr232tb/logs/debug-internal.log @@ -0,0 +1,865 @@ +2021-07-14 20:07:31,715 INFO MainThread:566515 [internal.py:wandb_internal():88] W&B internal server running at pid: 566515, started at: 2021-07-14 20:07:31.714999 +2021-07-14 20:07:31,717 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: check_version +2021-07-14 20:07:31,717 INFO WriterThread:566515 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/run-1nr232tb.wandb +2021-07-14 20:07:31,718 DEBUG SenderThread:566515 [sender.py:send():179] send: header +2021-07-14 20:07:31,718 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: check_version +2021-07-14 20:07:31,754 DEBUG SenderThread:566515 [sender.py:send():179] send: run +2021-07-14 20:07:31,935 INFO SenderThread:566515 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files +2021-07-14 20:07:31,935 INFO SenderThread:566515 [sender.py:_start_run_threads():716] run started: 1nr232tb with start time 1626293251 +2021-07-14 20:07:31,935 DEBUG SenderThread:566515 [sender.py:send():179] send: summary +2021-07-14 20:07:31,936 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:07:31,936 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: run_start +2021-07-14 20:07:32,938 INFO Thread-8 :566515 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:07:33,069 DEBUG HandlerThread:566515 [meta.py:__init__():39] meta init +2021-07-14 20:07:33,069 DEBUG HandlerThread:566515 [meta.py:__init__():53] meta init done +2021-07-14 20:07:33,069 DEBUG HandlerThread:566515 [meta.py:probe():210] probe +2021-07-14 20:07:33,071 DEBUG HandlerThread:566515 [meta.py:_setup_git():200] setup git +2021-07-14 20:07:33,100 DEBUG HandlerThread:566515 [meta.py:_setup_git():207] setup git done +2021-07-14 20:07:33,100 DEBUG HandlerThread:566515 [meta.py:_save_pip():57] save pip +2021-07-14 20:07:33,101 DEBUG HandlerThread:566515 [meta.py:_save_pip():71] save pip done +2021-07-14 20:07:33,101 DEBUG HandlerThread:566515 [meta.py:probe():252] probe done +2021-07-14 20:07:33,104 DEBUG SenderThread:566515 [sender.py:send():179] send: files +2021-07-14 20:07:33,104 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-14 20:07:33,110 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:07:33,110 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:07:33,237 DEBUG SenderThread:566515 [sender.py:send():179] send: config +2021-07-14 20:07:33,237 DEBUG SenderThread:566515 [sender.py:send():179] send: config +2021-07-14 20:07:33,237 DEBUG SenderThread:566515 [sender.py:send():179] send: config +2021-07-14 20:07:33,539 INFO Thread-11 :566515 [upload_job.py:push():137] Uploaded file /tmp/tmpx7_gpcyxwandb/2fjtpy4h-wandb-metadata.json +2021-07-14 20:07:33,936 INFO Thread-8 :566515 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:07:33,936 INFO Thread-8 :566515 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/requirements.txt +2021-07-14 20:07:33,937 INFO Thread-8 :566515 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-metadata.json +2021-07-14 20:07:47,942 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:07:48,239 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:07:48,239 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:07:53,944 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:08:01,150 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:08:02,948 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/config.yaml +2021-07-14 20:08:03,386 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:08:03,386 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:08:18,538 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:08:18,538 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:08:31,226 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:08:33,669 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:08:33,670 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:08:48,802 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:08:48,802 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:09:01,300 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:09:03,932 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:09:03,932 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:09:17,979 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:19,060 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:09:19,061 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:09:19,980 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:21,980 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:23,981 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:25,982 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:27,983 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:29,984 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:31,376 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:09:31,984 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:33,985 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:34,206 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:09:34,206 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:09:35,986 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:37,987 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:39,988 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:41,988 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:43,989 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:45,990 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:47,991 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:49,337 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:09:49,338 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:09:49,992 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:51,993 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:53,993 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:55,994 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:57,995 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:09:59,996 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:01,448 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:10:01,997 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:03,998 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:04,466 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:10:04,466 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:10:05,999 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:07,999 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:10,000 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:12,001 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:14,002 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:16,003 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:18,004 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:19,597 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:10:19,597 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:10:20,004 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:22,005 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:24,006 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:26,007 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:28,007 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:30,008 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:31,520 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:10:32,009 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:34,010 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:34,732 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:10:34,732 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:10:36,011 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:38,012 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:40,012 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:42,013 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:44,014 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:46,015 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:48,016 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:49,870 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:10:49,871 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:10:50,017 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:52,018 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:54,018 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:56,019 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:10:58,020 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:00,021 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:01,597 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:11:02,022 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:04,023 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:05,128 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:11:05,129 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:11:06,024 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:08,025 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:10,025 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:12,026 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:14,027 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:16,028 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:18,029 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:20,029 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:20,264 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:11:20,264 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:11:22,030 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:24,031 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:26,032 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:28,033 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:30,034 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:31,674 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:11:32,035 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:34,036 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:35,397 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:11:35,397 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:11:36,036 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:38,037 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:40,038 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:42,039 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:44,040 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:46,041 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:48,041 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:50,042 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:50,529 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:11:50,530 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:11:52,043 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:54,044 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:11:56,045 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:01,751 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:12:05,662 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:12:05,662 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:12:18,086 DEBUG SenderThread:566515 [sender.py:send():179] send: history +2021-07-14 20:12:18,087 DEBUG SenderThread:566515 [sender.py:send():179] send: summary +2021-07-14 20:12:18,089 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:12:19,054 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:12:20,055 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:20,796 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:12:20,797 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:12:22,055 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:24,056 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:26,057 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:28,058 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:30,059 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:31,829 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:12:32,060 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:34,061 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:35,940 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:12:35,941 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:12:36,061 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:38,062 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:40,063 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:42,064 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:44,065 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:46,065 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:48,066 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:50,067 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:51,073 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:12:51,073 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:12:52,068 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:54,069 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:56,070 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:12:58,070 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:00,071 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:01,907 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:13:02,072 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:04,073 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:06,073 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:06,206 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:13:06,206 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:13:08,074 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:10,075 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:12,076 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:14,077 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:16,078 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:18,079 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:20,079 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:21,336 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:13:21,336 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:13:22,080 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:24,081 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:26,082 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:28,083 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:30,083 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:31,983 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:13:32,084 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:34,085 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:36,086 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:36,469 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:13:36,469 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:13:38,086 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:40,087 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:43,089 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:45,089 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:47,090 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:49,091 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:51,092 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:51,600 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:13:51,601 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:13:53,093 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:55,094 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:57,095 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:13:59,096 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:01,096 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:02,063 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:14:03,097 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:05,098 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:06,737 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:14:06,738 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:14:07,099 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:09,099 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:11,100 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:13,101 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:15,102 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:17,103 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:19,104 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:21,105 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:21,868 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:14:21,868 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:14:23,106 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:25,106 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:27,107 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:29,108 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:31,109 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:32,140 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:14:33,110 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:35,111 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:36,998 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:14:36,998 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:14:37,112 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:39,113 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:41,114 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:43,115 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:45,116 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:47,116 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:49,117 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:51,118 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:52,130 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:14:52,130 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:14:53,119 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:55,120 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:14:57,120 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:02,218 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:15:07,259 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:15:07,259 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:15:18,883 DEBUG SenderThread:566515 [sender.py:send():179] send: history +2021-07-14 20:15:18,884 DEBUG SenderThread:566515 [sender.py:send():179] send: summary +2021-07-14 20:15:18,884 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:15:19,129 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:15:20,129 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:21,130 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:22,387 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:15:22,388 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:15:23,130 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:25,131 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:27,132 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:29,133 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:31,134 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:32,295 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:15:33,135 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:35,136 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:37,137 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:37,548 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:15:37,549 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:15:39,137 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:41,138 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:43,139 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:45,140 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:47,141 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:49,141 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:51,142 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:52,677 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:15:52,677 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:15:53,144 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:55,144 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:57,145 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:15:59,146 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:01,147 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:02,377 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:16:03,148 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:05,149 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:07,150 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:07,812 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:16:07,812 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:16:09,151 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:11,152 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:13,152 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:15,153 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:17,154 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:19,155 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:21,156 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:22,944 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:16:22,944 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:16:23,157 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:25,158 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:27,158 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:29,159 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:31,160 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:32,451 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:16:33,161 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:35,162 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:37,163 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:38,076 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:16:38,076 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:16:39,164 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:41,165 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:43,165 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:45,166 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:47,167 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:49,168 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:51,169 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:53,169 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:53,207 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:16:53,207 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:16:55,170 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:57,171 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:16:59,172 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:01,173 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:02,529 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:17:03,173 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:05,174 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:07,175 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:08,337 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:17:08,337 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:17:09,176 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:11,177 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:13,178 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:15,179 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:17,180 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:19,180 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:21,181 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:23,182 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:23,470 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:17:23,470 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:17:25,183 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:27,184 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:29,184 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:31,185 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:32,603 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:17:33,186 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:35,187 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:37,189 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:38,604 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:17:38,604 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:17:39,190 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:41,190 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:43,191 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:45,192 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:47,193 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:49,194 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:51,195 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:53,196 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:53,735 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:17:53,735 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:17:55,198 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:17:57,198 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:02,679 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:18:08,867 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:18:08,867 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:18:19,684 DEBUG SenderThread:566515 [sender.py:send():179] send: history +2021-07-14 20:18:19,684 DEBUG SenderThread:566515 [sender.py:send():179] send: summary +2021-07-14 20:18:19,685 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:18:20,207 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:18:21,207 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:23,208 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:24,001 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:18:24,002 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:18:25,209 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:27,210 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:29,211 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:31,212 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:32,757 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:18:33,212 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:35,213 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:37,214 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:39,132 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:18:39,133 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:18:39,215 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:41,216 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:43,217 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:45,218 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:47,219 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:49,219 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:51,220 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:53,221 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:54,264 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:18:54,265 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:18:55,222 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:57,223 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:18:59,224 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:01,225 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:02,833 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:19:03,226 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:05,227 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:07,228 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:09,228 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:09,395 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:19:09,395 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:19:11,229 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:13,230 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:15,231 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:17,232 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:19,233 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:21,234 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:23,235 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:24,524 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:19:24,524 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:19:25,235 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:27,236 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:29,237 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:31,238 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:32,913 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:19:33,239 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:35,240 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:37,241 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:39,242 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:39,653 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:19:39,653 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:19:41,242 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:43,243 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:45,244 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:47,245 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:49,246 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:51,247 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:53,247 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:54,783 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:19:54,783 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:19:55,248 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:57,249 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:19:59,250 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:01,251 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:02,990 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:20:03,252 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:05,252 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:07,253 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:09,254 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:09,917 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:20:09,918 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:20:11,255 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:13,256 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:15,256 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:17,257 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:19,258 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:21,259 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:23,260 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:25,051 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:20:25,052 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:20:25,261 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:27,261 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:29,262 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:31,263 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:33,069 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:20:33,264 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:35,264 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:37,265 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:39,266 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:40,196 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:20:40,197 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:20:41,267 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:43,267 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:45,268 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:47,269 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:50,270 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:52,271 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:54,272 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:55,327 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:20:55,328 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:20:56,273 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:20:58,274 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:03,146 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:21:10,456 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:21:10,456 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:21:20,479 DEBUG SenderThread:566515 [sender.py:send():179] send: history +2021-07-14 20:21:20,479 DEBUG SenderThread:566515 [sender.py:send():179] send: summary +2021-07-14 20:21:20,479 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:21:21,283 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:21,283 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:21:22,284 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:24,284 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:25,588 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:21:25,589 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:21:26,285 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:28,286 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:30,287 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:32,288 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:33,224 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:21:34,289 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:36,290 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:38,290 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:40,291 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:40,733 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:21:40,733 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:21:42,292 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:44,293 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:46,294 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:48,294 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:50,295 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:52,296 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:54,297 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:55,896 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:21:55,896 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:21:56,297 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:21:58,298 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:00,299 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:02,300 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:03,301 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:22:04,301 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:06,302 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:08,303 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:10,304 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:11,047 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:22:11,047 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:22:12,305 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:14,305 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:16,306 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:18,307 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:20,308 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:22,309 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:24,310 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:26,203 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:22:26,204 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:22:26,310 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:28,311 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:30,312 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:32,313 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:33,378 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:22:34,314 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:36,315 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:38,316 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:40,316 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:41,363 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:22:41,363 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:22:42,317 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:44,318 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:46,319 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:48,319 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:50,320 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:52,321 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:54,322 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:56,323 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:22:56,674 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:22:56,674 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:22:58,324 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:00,324 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:02,325 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:03,454 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:23:04,326 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:06,327 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:08,328 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:10,329 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:11,826 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:23:11,826 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:23:12,330 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:14,330 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:16,331 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:18,332 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:20,333 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:22,333 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:24,334 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:26,335 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:26,984 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:23:26,985 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:23:28,336 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:30,337 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:32,338 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:33,531 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:23:34,339 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:36,339 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:38,340 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:40,341 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:42,135 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:23:42,135 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:23:42,342 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:44,343 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:46,344 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:48,344 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:50,345 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:52,346 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:54,347 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:56,348 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:23:57,279 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:23:57,280 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:23:58,348 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:03,607 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:24:12,410 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:24:12,411 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:24:22,445 DEBUG SenderThread:566515 [sender.py:send():179] send: history +2021-07-14 20:24:22,445 DEBUG SenderThread:566515 [sender.py:send():179] send: summary +2021-07-14 20:24:22,446 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:24:23,359 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:24:24,359 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:26,360 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:27,539 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:24:27,540 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:24:28,361 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:30,362 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:32,363 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:33,686 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:24:34,364 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:36,365 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:38,365 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:40,366 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:42,367 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:42,685 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:24:42,685 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:24:44,368 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:46,369 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:48,370 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:50,370 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:52,371 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:54,372 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:56,373 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:24:57,841 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:24:57,841 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:24:58,374 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:00,375 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:02,376 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:03,764 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:25:04,377 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:06,378 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:08,378 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:10,379 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:12,380 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:12,999 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:25:12,999 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:25:14,381 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:16,382 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:18,382 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:20,383 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:22,384 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:24,385 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:26,386 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:28,149 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:25:28,149 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:25:28,387 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:30,388 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:32,388 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:33,841 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:25:34,389 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:36,390 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:38,391 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:40,392 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:42,393 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:43,294 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:25:43,295 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:25:44,394 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:46,395 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:48,395 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:50,396 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:52,397 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:54,398 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:56,399 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:58,399 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:25:58,450 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:25:58,451 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:26:00,400 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:02,401 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:03,919 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:26:04,402 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:06,403 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:08,404 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:10,405 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:12,406 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:13,618 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:26:13,618 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:26:14,406 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:16,407 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:18,408 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:20,409 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:22,409 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:24,410 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:26,411 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:28,412 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:28,768 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:26:28,769 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:26:30,413 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:32,414 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:33,994 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:26:34,415 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:36,415 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:38,416 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:40,417 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:42,418 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:43,915 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:26:43,915 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:26:44,419 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:46,420 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:48,420 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:50,421 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:52,422 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:54,423 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:56,423 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:58,424 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:26:59,082 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:26:59,083 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:27:00,425 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:04,074 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:27:14,232 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:27:14,233 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:27:23,233 DEBUG SenderThread:566515 [sender.py:send():179] send: history +2021-07-14 20:27:23,233 DEBUG SenderThread:566515 [sender.py:send():179] send: summary +2021-07-14 20:27:23,234 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:27:23,434 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:27:24,434 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:27,436 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:29,366 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:27:29,366 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:27:29,436 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:31,437 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:33,438 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:34,152 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:27:35,439 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:37,440 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:39,441 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:41,442 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:43,443 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:44,501 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:27:44,501 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:27:45,444 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:47,445 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:49,445 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:51,446 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:53,447 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:55,448 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:57,448 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:59,449 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:27:59,635 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:27:59,635 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:28:01,450 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:03,451 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:04,228 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:28:05,451 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:07,452 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:09,453 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:11,454 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:13,455 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:14,769 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:28:14,769 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:28:15,456 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:17,457 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:19,458 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:21,458 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:23,459 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:25,460 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:27,461 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:29,462 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:29,907 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:28:29,907 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:28:31,462 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:33,463 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:34,306 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:28:35,464 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:37,465 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:39,466 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:41,466 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:43,467 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:45,053 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:28:45,053 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:28:45,468 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:47,469 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:49,470 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:51,471 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:53,472 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:55,472 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:57,473 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:28:59,474 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:00,186 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:29:00,186 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:29:01,475 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:03,476 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:04,385 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:29:05,477 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:07,477 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:09,478 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:11,479 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:13,480 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:15,423 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:29:15,424 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:29:15,481 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:17,481 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:19,482 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:21,483 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:23,484 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:25,485 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:27,485 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:29,486 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:30,554 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:29:30,554 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:29:31,487 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:33,488 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:34,462 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:29:35,489 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:37,490 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:39,490 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:41,491 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:43,492 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:45,493 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:45,688 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:29:45,688 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:29:47,494 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:49,495 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:51,495 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:53,496 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:55,497 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:57,498 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:29:59,499 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:30:00,818 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:30:00,818 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:30:01,500 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:30:04,537 DEBUG SenderThread:566515 [sender.py:send():179] send: stats +2021-07-14 20:30:15,952 DEBUG HandlerThread:566515 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 20:30:15,952 DEBUG SenderThread:566515 [sender.py:send_request():193] send_request: stop_status +2021-07-14 20:30:24,030 DEBUG SenderThread:566515 [sender.py:send():179] send: history +2021-07-14 20:30:24,031 DEBUG SenderThread:566515 [sender.py:send():179] send: summary +2021-07-14 20:30:24,031 INFO SenderThread:566515 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 20:30:24,509 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:30:25,509 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:30:26,440 WARNING MainThread:566515 [internal.py:wandb_internal():147] Internal process interrupt: 1 +2021-07-14 20:30:27,510 INFO Thread-8 :566515 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:30:27,654 WARNING MainThread:566515 [internal.py:wandb_internal():147] Internal process interrupt: 2 +2021-07-14 20:30:27,654 ERROR MainThread:566515 [internal.py:wandb_internal():150] Internal process interrupted. +2021-07-14 20:30:27,903 INFO SenderThread:566515 [sender.py:finish():945] shutting down sender +2021-07-14 20:30:27,903 INFO WriterThread:566515 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/run-1nr232tb.wandb +2021-07-14 20:30:27,903 INFO SenderThread:566515 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-14 20:30:27,904 INFO HandlerThread:566515 [handler.py:finish():638] shutting down handler +2021-07-14 20:30:28,511 INFO SenderThread:566515 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files +2021-07-14 20:30:28,511 INFO SenderThread:566515 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/requirements.txt requirements.txt +2021-07-14 20:30:28,511 INFO SenderThread:566515 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log output.log +2021-07-14 20:30:28,511 INFO SenderThread:566515 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-metadata.json wandb-metadata.json +2021-07-14 20:30:28,511 INFO SenderThread:566515 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/config.yaml config.yaml +2021-07-14 20:30:28,512 INFO SenderThread:566515 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json wandb-summary.json +2021-07-14 20:30:28,515 INFO SenderThread:566515 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 20:30:28,515 INFO SenderThread:566515 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 20:30:29,047 INFO Thread-15 :566515 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/wandb-summary.json +2021-07-14 20:30:29,220 INFO Thread-12 :566515 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/requirements.txt +2021-07-14 20:30:29,221 INFO Thread-14 :566515 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/config.yaml +2021-07-14 20:30:29,222 INFO Thread-13 :566515 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/files/output.log +2021-07-14 20:30:29,224 INFO MainThread:566515 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210714_200731-1nr232tb/logs/debug.log b/wandb/run-20210714_200731-1nr232tb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ecc9575be95c8b398ab4eec7a8c5bff12cd3884e --- /dev/null +++ b/wandb/run-20210714_200731-1nr232tb/logs/debug.log @@ -0,0 +1,27 @@ +2021-07-14 20:07:31,020 INFO MainThread:565261 [wandb_setup.py:_flush():69] setting env: {} +2021-07-14 20:07:31,020 INFO MainThread:565261 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-14 20:07:31,020 INFO MainThread:565261 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/logs/debug.log +2021-07-14 20:07:31,020 INFO MainThread:565261 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_200731-1nr232tb/logs/debug-internal.log +2021-07-14 20:07:31,020 INFO MainThread:565261 [wandb_init.py:init():370] calling init triggers +2021-07-14 20:07:31,020 INFO MainThread:565261 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-14 20:07:31,020 INFO MainThread:565261 [wandb_init.py:init():419] starting backend +2021-07-14 20:07:31,020 INFO MainThread:565261 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-14 20:07:31,066 INFO MainThread:565261 [backend.py:ensure_launched():135] starting backend process... +2021-07-14 20:07:31,112 INFO MainThread:565261 [backend.py:ensure_launched():139] started backend process with pid: 566515 +2021-07-14 20:07:31,114 INFO MainThread:565261 [wandb_init.py:init():424] backend started and connected +2021-07-14 20:07:31,117 INFO MainThread:565261 [wandb_init.py:init():472] updated telemetry +2021-07-14 20:07:31,118 INFO MainThread:565261 [wandb_init.py:init():491] communicating current version +2021-07-14 20:07:31,753 INFO MainThread:565261 [wandb_init.py:init():496] got version response +2021-07-14 20:07:31,753 INFO MainThread:565261 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-14 20:07:31,936 INFO MainThread:565261 [wandb_init.py:init():529] starting run threads in backend +2021-07-14 20:07:33,107 INFO MainThread:565261 [wandb_run.py:_console_start():1623] atexit reg +2021-07-14 20:07:33,108 INFO MainThread:565261 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-14 20:07:33,108 INFO MainThread:565261 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-14 20:07:33,110 INFO MainThread:565261 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-14 20:07:33,110 INFO MainThread:565261 [wandb_init.py:init():554] run started, returning control to user process +2021-07-14 20:07:33,116 INFO MainThread:565261 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_20-07-23_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-14 20:07:33,118 INFO MainThread:565261 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True} +2021-07-14 20:07:33,120 INFO MainThread:565261 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000} +2021-07-14 20:30:26,444 INFO MainThread:565261 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255 +2021-07-14 20:30:26,445 INFO MainThread:565261 [wandb_run.py:_restore():1565] restore diff --git a/wandb/run-20210714_200731-1nr232tb/run-1nr232tb.wandb b/wandb/run-20210714_200731-1nr232tb/run-1nr232tb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4319e20297c0243e86af54d338ef2514dd349a65 Binary files /dev/null and b/wandb/run-20210714_200731-1nr232tb/run-1nr232tb.wandb differ diff --git a/wandb/run-20210714_203117-3emij6kq/files/config.yaml b/wandb/run-20210714_203117-3emij6kq/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d0bd25e2730f234eb77b989beb577d688e47e91 --- /dev/null +++ b/wandb/run-20210714_203117-3emij6kq/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: true +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 40000 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 2 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 5.0e-05 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul14_20-31-09_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 250 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_eval_samples: + desc: null + value: 5000 +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 3.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 96 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_optimizer: + desc: null + value: true +save_steps: + desc: null + value: 2500 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 5000 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210714_203117-3emij6kq/files/output.log b/wandb/run-20210714_203117-3emij6kq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a24f61065ab8324f0b03091946547b5520ce7880 --- /dev/null +++ b/wandb/run-20210714_203117-3emij6kq/files/output.log @@ -0,0 +1,174 @@ +[20:31:32] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`. +/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3132: UserWarning: Explicitly requested dtype requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00 + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback + return fun(*args, **kwargs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1669, in f_pmapped + out = pxla.xla_pmap( + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind + return call_bind(self, fun, *args, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind + outs = primitive.process(top_trace, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process + return trace.process_map(self, fun, tracers, params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call + return primitive.impl(f, *tracers, **params) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 624, in xla_pmap_impl + compiled_fun, fingerprint = parallel_callable(fun, backend, axis_name, axis_size, + File "/home/dat/pino/lib/python3.8/site-packages/jax/linear_util.py", line 262, in memoized_fun + ans = call(fun, *args) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 906, in parallel_callable + compiled = xla.backend_compile(backend, built, compile_options) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile + return backend.compile(built_c, compile_options=options) +jax._src.traceback_util.UnfilteredStackTrace: RuntimeError: Resource exhausted: Ran out of memory in memory space hbm. Used 17.79G of 15.48G hbm. Exceeded hbm capacity by 2.31G. +Total hbm usage >= 18.31G: + reserved 530.00M + program 17.79G + arguments 0B +Output size 0B; shares 0B with arguments. +Program hbm requirement 17.79G: + global 884.0K + scoped 253.0K + HLO temp 17.79G (97.6% utilization: Unpadded (17.27G) Padded (17.68G), 0.6% fragmentation (106.34M)) + Largest program allocations in hbm: + 1. Size: 3.07G + Operator: op_type="dot_general" op_name="pmap(train_step)/dot_general[ dimension_numbers=(((2,), (0,)), ((), ()))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/pino/lib/python3.8/site-packages/flax/linen/linear.py" source_line=175 + Shape: f32[4,4096,50358]{1,2,0:T(8,128)} + Unpadded size: 3.07G + Extra memory due to padding: 128.0K (1.0x expansion) + XLA label: %fusion.1233.remat4 = f32[4,4096,50358]{1,2,0:T(8,128)} fusion(f32[50358]{0:T(1024)} %get-tuple-element.21733, f32[768,50358,1]{0,1,2:T(8,128)} %bitcast.4927, f32[768]{0:T(1024)} %get-tuple-element.21734, f32[768]{0:T(1024)} %get-tuple-element.21735, f32[4... + Allocation type: HLO temp + ========================== + 2. Size: 336.00M + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.12188 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1904, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8899, f32[4,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 3. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1304.remat6 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1906, f32[4,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 4. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1304.remat6 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1906, f32[4,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 5. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1306.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1908, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8903, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 6. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1307.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1909, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8904, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 7. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1308.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1910, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8905, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 8. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1309.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1911, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8906, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 9. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1310.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1912, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8907, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 10. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1311.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1913, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8908, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 11. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1312.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1914, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8909, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 12. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1305 = bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1907, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8902, f32[4,12,28,128,128]{3,4,2,1,0:T(8,128)} %get-tuple-element.19534, f32[4,12,28,128,384]{... + Allocation type: HLO temp + ========================== + 13. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1301.remat6 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1903, f32[4,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 14. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1301.remat6 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1903, f32[4,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 15. Size: 336.00M + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.12187 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1905, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8900, f32[4,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 16. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.10998 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.23248, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28551, f32[4,12,32,128,64]{3,2,4,1... + Allocation type: HLO temp + ========================== + 17. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.11022 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.23245, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28656.remat_uncompressed, f32[4,12... + Allocation type: HLO temp + ========================== + 18. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.11014 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.23246, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28621.remat_uncompressed, f32[4,12... + Allocation type: HLO temp + ========================== + 19. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.11006 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.23247, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28586.remat_uncompressed, f32[4,12... + Allocation type: HLO temp + ========================== + 20. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.10934 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.19864, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28270, f32[4,12,32,128,64]{3,2,4,1... + Allocation type: HLO temp + ========================== +The stack trace below excludes JAX-internal frames. +The preceding is the original exception that occurred, unmodified. +-------------------- +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "./run_mlm_flax.py", line 790, in + state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs) + File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile + return backend.compile(built_c, compile_options=options) +RuntimeError: Resource exhausted: Ran out of memory in memory space hbm. Used 17.79G of 15.48G hbm. Exceeded hbm capacity by 2.31G. +Total hbm usage >= 18.31G: + reserved 530.00M + program 17.79G + arguments 0B +Output size 0B; shares 0B with arguments. +Program hbm requirement 17.79G: + global 884.0K + scoped 253.0K + HLO temp 17.79G (97.6% utilization: Unpadded (17.27G) Padded (17.68G), 0.6% fragmentation (106.34M)) + Largest program allocations in hbm: + 1. Size: 3.07G + Operator: op_type="dot_general" op_name="pmap(train_step)/dot_general[ dimension_numbers=(((2,), (0,)), ((), ()))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/pino/lib/python3.8/site-packages/flax/linen/linear.py" source_line=175 + Shape: f32[4,4096,50358]{1,2,0:T(8,128)} + Unpadded size: 3.07G + Extra memory due to padding: 128.0K (1.0x expansion) + XLA label: %fusion.1233.remat4 = f32[4,4096,50358]{1,2,0:T(8,128)} fusion(f32[50358]{0:T(1024)} %get-tuple-element.21733, f32[768,50358,1]{0,1,2:T(8,128)} %bitcast.4927, f32[768]{0:T(1024)} %get-tuple-element.21734, f32[768]{0:T(1024)} %get-tuple-element.21735, f32[4... + Allocation type: HLO temp + ========================== + 2. Size: 336.00M + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.12188 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1904, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8899, f32[4,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 3. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1304.remat6 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1906, f32[4,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 4. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1304.remat6 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1906, f32[4,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 5. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1306.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1908, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8903, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 6. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1307.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1909, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8904, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 7. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1308.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1910, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8905, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 8. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1309.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1911, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8906, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 9. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1310.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1912, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8907, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 10. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1311.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1913, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8908, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 11. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1312.remat = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1914, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8909, f32[4,12,28,128,128]{3,4,2,1,0:... + Allocation type: HLO temp + ========================== + 12. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1305 = bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1907, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8902, f32[4,12,28,128,128]{3,4,2,1,0:T(8,128)} %get-tuple-element.19534, f32[4,12,28,128,384]{... + Allocation type: HLO temp + ========================== + 13. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1301.remat6 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1903, f32[4,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 14. Size: 336.00M + Operator: op_type="div" op_name="pmap(train_step)/div" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=619 + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.1301.remat6 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1903, f32[4,12,28,128]{3,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 15. Size: 336.00M + Shape: bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)} + Unpadded size: 336.00M + XLA label: %fusion.12187 = (bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}, bf16[4,12,28,128,1024]{3,4,2,1,0:T(8,128)(2,1)}) fusion(f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.1905, f32[4,12,28,128]{3,2,1,0:T(8,128)} %fusion.8900, f32[4,12,28,128,128]{3,4,2,1,0:T(8,1... + Allocation type: HLO temp + ========================== + 16. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.10998 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.23248, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28551, f32[4,12,32,128,64]{3,2,4,1... + Allocation type: HLO temp + ========================== + 17. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.11022 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.23245, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28656.remat_uncompressed, f32[4,12... + Allocation type: HLO temp + ========================== + 18. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.11014 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.23246, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28621.remat_uncompressed, f32[4,12... + Allocation type: HLO temp + ========================== + 19. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.11006 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.23247, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28586.remat_uncompressed, f32[4,12... + Allocation type: HLO temp + ========================== + 20. Size: 252.00M + Operator: op_type="dot_general" op_name="pmap(train_step)/jit(jvp(_einsum))/dot_general[ dimension_numbers=(((4,), (4,)), ((0, 1, 2), (0, 1, 2)))\n precision=None\n preferred_element_type=None ]" source_file="/home/dat/transformers/src/transformers/models/big_bird/modeling_flax_big_bird.py" source_line=591 + Shape: f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)} + Unpadded size: 252.00M + XLA label: %fusion.10934 = (f32[4,12,28,128]{3,2,1,0:T(8,128)}, f32[4,12,28,128,384]{3,4,2,1,0:T(8,128)}) fusion(s32[4,12,30,128,384]{3,4,2,1,0:T(8,128)} %get-tuple-element.19864, bf16[4,12,28,384,64]{3,2,1,0,4:T(8,128)(2,1)} %slice.28270, f32[4,12,32,128,64]{3,2,4,1... + Allocation type: HLO temp + ========================== \ No newline at end of file diff --git a/wandb/run-20210714_210057-dn4954s5/files/requirements.txt b/wandb/run-20210714_210057-dn4954s5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..554e3a34bc91714a8462f65295a12e9a04537637 --- /dev/null +++ b/wandb/run-20210714_210057-dn4954s5/files/requirements.txt @@ -0,0 +1,94 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +astunparse==1.6.3 +async-timeout==3.0.1 +attrs==21.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.1 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==1.12 +flax==0.3.4 +fsspec==2021.7.0 +gast==0.4.0 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +google-pasta==0.2.0 +grpcio==1.34.1 +h5py==3.1.0 +huggingface-hub==0.0.12 +idna==3.2 +install==1.3.4 +jax==0.2.17 +jaxlib==0.1.68 +joblib==1.0.1 +keras-nightly==2.5.0.dev2021032900 +keras-preprocessing==1.1.2 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.19.5 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +pathtools==0.1.2 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +protobuf==3.17.3 +psutil==5.8.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.26.0 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.15.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tensorflow-estimator==2.5.0 +tensorflow==2.5.0 +termcolor==1.1.0 +tokenizers==0.10.3 +toolz==0.11.1 +tqdm==4.61.2 +transformers==4.9.0.dev0 +typing-extensions==3.7.4.3 +urllib3==1.26.6 +wandb==0.10.33 +werkzeug==2.0.1 +wheel==0.36.2 +wrapt==1.12.1 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210714_210057-dn4954s5/files/wandb-metadata.json b/wandb/run-20210714_210057-dn4954s5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b4f967de3156a4d36b4d9244b6c6b99076747ab4 --- /dev/null +++ b/wandb/run-20210714_210057-dn4954s5/files/wandb-metadata.json @@ -0,0 +1,47 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-14T21:00:59.959435", + "startedAt": "2021-07-14T21:00:57.957288", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--push_to_hub", + "--output_dir=./", + "--model_type=big_bird", + "--config_name=./", + "--tokenizer_name=./", + "--max_seq_length=4096", + "--weight_decay=0.0095", + "--warmup_steps=5000", + "--overwrite_output_dir", + "--adam_beta1=0.9", + "--adam_beta2=0.98", + "--logging_steps=250", + "--eval_steps=40000", + "--num_train_epochs=3", + "--preprocessing_num_workers=96", + "--save_steps=2500", + "--learning_rate=5e-5", + "--per_device_train_batch_size=4", + "--per_device_eval_batch_size=4", + "--save_total_limit=5", + "--max_eval_samples=5000", + "--overwrite_cache", + "False", + "--gradient_accumulation_steps=4" + ], + "state": "running", + "program": "./run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/pino-roberta-base", + "commit": "f6e0bf7955345b689d1a39a7ffb5ce2ba806da0b" + }, + "email": null, + "root": "/home/dat/pino-roberta-base", + "host": "t1v-n-f5c06ea1-w-0", + "username": "dat", + "executable": "/home/dat/pino/bin/python" +} diff --git a/wandb/run-20210714_210057-dn4954s5/files/wandb-summary.json b/wandb/run-20210714_210057-dn4954s5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20210714_210057-dn4954s5/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20210714_210057-dn4954s5/logs/debug-internal.log b/wandb/run-20210714_210057-dn4954s5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e75b0cc1207657f6d743b181ee442c7b1723cb1e --- /dev/null +++ b/wandb/run-20210714_210057-dn4954s5/logs/debug-internal.log @@ -0,0 +1,133 @@ +2021-07-14 21:00:58,647 INFO MainThread:574948 [internal.py:wandb_internal():88] W&B internal server running at pid: 574948, started at: 2021-07-14 21:00:58.647052 +2021-07-14 21:00:58,649 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: check_version +2021-07-14 21:00:58,649 INFO WriterThread:574948 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/run-dn4954s5.wandb +2021-07-14 21:00:58,650 DEBUG SenderThread:574948 [sender.py:send():179] send: header +2021-07-14 21:00:58,650 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: check_version +2021-07-14 21:00:58,686 DEBUG SenderThread:574948 [sender.py:send():179] send: run +2021-07-14 21:00:58,856 INFO SenderThread:574948 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files +2021-07-14 21:00:58,856 INFO SenderThread:574948 [sender.py:_start_run_threads():716] run started: dn4954s5 with start time 1626296458 +2021-07-14 21:00:58,856 DEBUG SenderThread:574948 [sender.py:send():179] send: summary +2021-07-14 21:00:58,857 INFO SenderThread:574948 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 21:00:58,857 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: run_start +2021-07-14 21:00:59,858 INFO Thread-8 :574948 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/wandb-summary.json +2021-07-14 21:00:59,959 DEBUG HandlerThread:574948 [meta.py:__init__():39] meta init +2021-07-14 21:00:59,959 DEBUG HandlerThread:574948 [meta.py:__init__():53] meta init done +2021-07-14 21:00:59,959 DEBUG HandlerThread:574948 [meta.py:probe():210] probe +2021-07-14 21:00:59,960 DEBUG HandlerThread:574948 [meta.py:_setup_git():200] setup git +2021-07-14 21:00:59,989 DEBUG HandlerThread:574948 [meta.py:_setup_git():207] setup git done +2021-07-14 21:00:59,989 DEBUG HandlerThread:574948 [meta.py:_save_pip():57] save pip +2021-07-14 21:00:59,989 DEBUG HandlerThread:574948 [meta.py:_save_pip():71] save pip done +2021-07-14 21:00:59,990 DEBUG HandlerThread:574948 [meta.py:probe():252] probe done +2021-07-14 21:00:59,992 DEBUG SenderThread:574948 [sender.py:send():179] send: files +2021-07-14 21:00:59,993 INFO SenderThread:574948 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-14 21:00:59,998 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 21:00:59,999 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: stop_status +2021-07-14 21:01:00,132 DEBUG SenderThread:574948 [sender.py:send():179] send: config +2021-07-14 21:01:00,133 DEBUG SenderThread:574948 [sender.py:send():179] send: config +2021-07-14 21:01:00,133 DEBUG SenderThread:574948 [sender.py:send():179] send: config +2021-07-14 21:01:00,445 INFO Thread-11 :574948 [upload_job.py:push():137] Uploaded file /tmp/tmpqq6ketpgwandb/y90ikahm-wandb-metadata.json +2021-07-14 21:01:00,857 INFO Thread-8 :574948 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/output.log +2021-07-14 21:01:00,857 INFO Thread-8 :574948 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/requirements.txt +2021-07-14 21:01:00,857 INFO Thread-8 :574948 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/wandb-metadata.json +2021-07-14 21:01:14,863 INFO Thread-8 :574948 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/output.log +2021-07-14 21:01:15,134 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 21:01:15,135 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: stop_status +2021-07-14 21:01:20,865 INFO Thread-8 :574948 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/output.log +2021-07-14 21:01:28,042 DEBUG SenderThread:574948 [sender.py:send():179] send: stats +2021-07-14 21:01:29,869 INFO Thread-8 :574948 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/config.yaml +2021-07-14 21:01:30,286 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 21:01:30,287 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: stop_status +2021-07-14 21:01:45,439 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 21:01:45,440 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: stop_status +2021-07-14 21:01:58,125 DEBUG SenderThread:574948 [sender.py:send():179] send: stats +2021-07-14 21:02:00,570 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 21:02:00,571 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: stop_status +2021-07-14 21:02:15,704 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 21:02:15,704 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: stop_status +2021-07-14 21:02:28,204 DEBUG SenderThread:574948 [sender.py:send():179] send: stats +2021-07-14 21:02:30,841 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: stop_status +2021-07-14 21:02:30,841 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: stop_status +2021-07-14 21:02:32,894 INFO Thread-8 :574948 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/output.log +2021-07-14 21:02:33,493 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 21:02:33,493 DEBUG SenderThread:574948 [sender.py:send():179] send: telemetry +2021-07-14 21:02:33,493 DEBUG SenderThread:574948 [sender.py:send():179] send: exit +2021-07-14 21:02:33,493 INFO SenderThread:574948 [sender.py:send_exit():287] handling exit code: 1 +2021-07-14 21:02:33,494 INFO SenderThread:574948 [sender.py:send_exit():295] send defer +2021-07-14 21:02:33,494 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 21:02:33,494 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:33,494 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-14 21:02:33,495 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: defer +2021-07-14 21:02:33,495 INFO SenderThread:574948 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-14 21:02:33,495 INFO SenderThread:574948 [sender.py:transition_state():308] send defer: 1 +2021-07-14 21:02:33,495 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:33,495 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-14 21:02:33,518 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: defer +2021-07-14 21:02:33,518 INFO SenderThread:574948 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-14 21:02:33,518 INFO SenderThread:574948 [sender.py:transition_state():308] send defer: 2 +2021-07-14 21:02:33,518 DEBUG SenderThread:574948 [sender.py:send():179] send: stats +2021-07-14 21:02:33,519 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:33,519 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-14 21:02:33,519 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: defer +2021-07-14 21:02:33,519 INFO SenderThread:574948 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-14 21:02:33,519 INFO SenderThread:574948 [sender.py:transition_state():308] send defer: 3 +2021-07-14 21:02:33,520 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:33,520 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-14 21:02:33,520 DEBUG SenderThread:574948 [sender.py:send():179] send: summary +2021-07-14 21:02:33,520 INFO SenderThread:574948 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-14 21:02:33,520 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: defer +2021-07-14 21:02:33,520 INFO SenderThread:574948 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-14 21:02:33,520 INFO SenderThread:574948 [sender.py:transition_state():308] send defer: 4 +2021-07-14 21:02:33,521 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:33,521 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-14 21:02:33,521 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: defer +2021-07-14 21:02:33,521 INFO SenderThread:574948 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-14 21:02:33,596 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 21:02:33,707 INFO SenderThread:574948 [sender.py:transition_state():308] send defer: 5 +2021-07-14 21:02:33,708 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 21:02:33,708 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:33,708 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-14 21:02:33,709 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: defer +2021-07-14 21:02:33,709 INFO SenderThread:574948 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-14 21:02:33,709 INFO SenderThread:574948 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-14 21:02:33,810 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 21:02:33,895 INFO Thread-8 :574948 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/config.yaml +2021-07-14 21:02:33,895 INFO SenderThread:574948 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/output.log +2021-07-14 21:02:33,895 INFO SenderThread:574948 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/wandb-summary.json +2021-07-14 21:02:33,895 INFO SenderThread:574948 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files +2021-07-14 21:02:33,896 INFO SenderThread:574948 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/requirements.txt requirements.txt +2021-07-14 21:02:33,896 INFO SenderThread:574948 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/output.log output.log +2021-07-14 21:02:33,896 INFO SenderThread:574948 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/wandb-metadata.json wandb-metadata.json +2021-07-14 21:02:33,896 INFO SenderThread:574948 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/config.yaml config.yaml +2021-07-14 21:02:33,896 INFO SenderThread:574948 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/wandb-summary.json wandb-summary.json +2021-07-14 21:02:33,896 INFO SenderThread:574948 [sender.py:transition_state():308] send defer: 6 +2021-07-14 21:02:33,897 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 21:02:33,908 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:33,908 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-14 21:02:33,909 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: defer +2021-07-14 21:02:33,909 INFO SenderThread:574948 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-14 21:02:33,909 INFO SenderThread:574948 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 21:02:34,002 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 21:02:34,002 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 21:02:34,104 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-14 21:02:34,104 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: poll_exit +2021-07-14 21:02:34,203 WARNING MainThread:574948 [internal.py:wandb_internal():147] Internal process interrupt: 1 +2021-07-14 21:02:34,354 INFO Thread-14 :574948 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/config.yaml +2021-07-14 21:02:34,366 INFO Thread-13 :574948 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/output.log +2021-07-14 21:02:34,377 INFO Thread-12 :574948 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/requirements.txt +2021-07-14 21:02:34,379 INFO Thread-15 :574948 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/files/wandb-summary.json +2021-07-14 21:02:34,579 INFO Thread-7 :574948 [sender.py:transition_state():308] send defer: 7 +2021-07-14 21:02:34,580 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:34,580 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-14 21:02:34,580 DEBUG SenderThread:574948 [sender.py:send_request():193] send_request: defer +2021-07-14 21:02:34,580 INFO SenderThread:574948 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-14 21:02:34,801 WARNING MainThread:574948 [internal.py:wandb_internal():147] Internal process interrupt: 2 +2021-07-14 21:02:34,801 ERROR MainThread:574948 [internal.py:wandb_internal():150] Internal process interrupted. +2021-07-14 21:02:34,872 INFO SenderThread:574948 [sender.py:transition_state():308] send defer: 8 +2021-07-14 21:02:34,872 INFO SenderThread:574948 [sender.py:finish():945] shutting down sender +2021-07-14 21:02:34,872 INFO SenderThread:574948 [file_pusher.py:finish():177] shutting down file pusher +2021-07-14 21:02:34,872 INFO SenderThread:574948 [file_pusher.py:join():182] waiting for file pusher +2021-07-14 21:02:34,873 DEBUG HandlerThread:574948 [handler.py:handle_request():124] handle_request: defer +2021-07-14 21:02:34,873 INFO HandlerThread:574948 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-14 21:02:34,875 INFO HandlerThread:574948 [handler.py:finish():638] shutting down handler +2021-07-14 21:02:35,521 INFO WriterThread:574948 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/run-dn4954s5.wandb +2021-07-14 21:02:35,521 INFO MainThread:574948 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210714_210057-dn4954s5/logs/debug.log b/wandb/run-20210714_210057-dn4954s5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e8a447d7a099de2af712b179433b7d692d708f14 --- /dev/null +++ b/wandb/run-20210714_210057-dn4954s5/logs/debug.log @@ -0,0 +1,67 @@ +2021-07-14 21:00:57,958 INFO MainThread:573688 [wandb_setup.py:_flush():69] setting env: {} +2021-07-14 21:00:57,958 INFO MainThread:573688 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-14 21:00:57,959 INFO MainThread:573688 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/logs/debug.log +2021-07-14 21:00:57,959 INFO MainThread:573688 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_210057-dn4954s5/logs/debug-internal.log +2021-07-14 21:00:57,959 INFO MainThread:573688 [wandb_init.py:init():370] calling init triggers +2021-07-14 21:00:57,959 INFO MainThread:573688 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-14 21:00:57,959 INFO MainThread:573688 [wandb_init.py:init():419] starting backend +2021-07-14 21:00:57,959 INFO MainThread:573688 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-14 21:00:58,004 INFO MainThread:573688 [backend.py:ensure_launched():135] starting backend process... +2021-07-14 21:00:58,049 INFO MainThread:573688 [backend.py:ensure_launched():139] started backend process with pid: 574948 +2021-07-14 21:00:58,051 INFO MainThread:573688 [wandb_init.py:init():424] backend started and connected +2021-07-14 21:00:58,054 INFO MainThread:573688 [wandb_init.py:init():472] updated telemetry +2021-07-14 21:00:58,055 INFO MainThread:573688 [wandb_init.py:init():491] communicating current version +2021-07-14 21:00:58,685 INFO MainThread:573688 [wandb_init.py:init():496] got version response +2021-07-14 21:00:58,685 INFO MainThread:573688 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-14 21:00:58,857 INFO MainThread:573688 [wandb_init.py:init():529] starting run threads in backend +2021-07-14 21:00:59,996 INFO MainThread:573688 [wandb_run.py:_console_start():1623] atexit reg +2021-07-14 21:00:59,996 INFO MainThread:573688 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-14 21:00:59,997 INFO MainThread:573688 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-14 21:00:59,999 INFO MainThread:573688 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-14 21:00:59,999 INFO MainThread:573688 [wandb_init.py:init():554] run started, returning control to user process +2021-07-14 21:01:00,005 INFO MainThread:573688 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_21-00-50_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 2500, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-14 21:01:00,007 INFO MainThread:573688 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True} +2021-07-14 21:01:00,008 INFO MainThread:573688 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000} +2021-07-14 21:02:31,122 INFO MainThread:573688 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-14 21:02:31,123 INFO MainThread:573688 [wandb_run.py:_restore():1565] restore +2021-07-14 21:02:33,495 INFO MainThread:573688 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1446 + total_bytes: 1446 +} + +2021-07-14 21:02:33,708 INFO MainThread:573688 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1446 + total_bytes: 1446 +} + +2021-07-14 21:02:33,900 INFO MainThread:573688 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1446 + total_bytes: 39216 +} + +2021-07-14 21:02:34,003 INFO MainThread:573688 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 1446 + total_bytes: 39216 +} + +2021-07-14 21:02:34,105 INFO MainThread:573688 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 39216 + total_bytes: 39216 +} + diff --git a/wandb/run-20210714_210057-dn4954s5/run-dn4954s5.wandb b/wandb/run-20210714_210057-dn4954s5/run-dn4954s5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ffe83ac301cd9f5015ce5eb66fdf08c53ebf9100 Binary files /dev/null and b/wandb/run-20210714_210057-dn4954s5/run-dn4954s5.wandb differ diff --git a/wandb/run-20210714_210351-1msvb4w4/files/config.yaml b/wandb/run-20210714_210351-1msvb4w4/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48166f04ddd5a5770d9343fdc402e89310e44f55 --- /dev/null +++ b/wandb/run-20210714_210351-1msvb4w4/files/config.yaml @@ -0,0 +1,304 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 3 + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: false +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.98 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: null +dataset_name: + desc: null + value: null +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 500 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 4 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 0.0001 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul14_21-03-44_t1v-n-f5c06ea1-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 250 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_eval_samples: + desc: null + value: 500 +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 4096 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: big_bird +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 3.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 2 +per_device_train_batch_size: + desc: null + value: 2 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: 96 +push_to_hub: + desc: null + value: true +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_optimizer: + desc: null + value: true +save_steps: + desc: null + value: 1250 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: 5 +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 5000 +weight_decay: + desc: null + value: 0.0095 diff --git a/wandb/run-20210714_210351-1msvb4w4/files/output.log b/wandb/run-20210714_210351-1msvb4w4/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f9e4f69f995346f3cff4d9b4a84ace185fd0349a --- /dev/null +++ b/wandb/run-20210714_210351-1msvb4w4/files/output.log @@ -0,0 +1,418 @@ +[21:04:06] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`. +/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3132: UserWarning: Explicitly requested dtype requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more. + lax._check_user_dtype_supported(dtype, "zeros") +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/3): 0%| | 0/3 [00:00