End of training

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230711_182238-06r7b4m3/run-06r7b4m3.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230711_182238-k29fqnbt/run-k29fqnbt.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230711_182238-wt8le4qg/run-wt8le4qg.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230711_182238-zq31sjjd/run-zq31sjjd.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230712_000308-hbsh7yzb/run-hbsh7yzb.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230712_000308-lh5nw6i8/run-lh5nw6i8.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,21 @@
+
+---
+license: creativeml-openrail-m
+base_model: runwayml/stable-diffusion-v1-5
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- text-to-image
+- diffusers
+- lora
+inference: true
+---
+    
+# LoRA text2image fine-tuning - asrimanth/person-thumbs-up-plain-lora
+These are LoRA adaption weights for runwayml/stable-diffusion-v1-5. The weights were fine-tuned on the Custom dataset dataset. You can find some example images in the following. 
+
+![img_0](./image_0.png)
+![img_1](./image_1.png)
+![img_2](./image_2.png)
+![img_3](./image_3.png)
+

checkpoint-1000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56d6cc84384435472775b08eb1b36db0e391123d01838f0bf629e2a6903872d7
+size 6591685

checkpoint-1000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42e0eb5b8867df808ad9c97d3e100db59d7a3c5dd829b4390ef4e78ea0eb56cb
+size 3285965

checkpoint-1000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70f46034afce00e501c95a101d672135d81d8b659749abecb24ea30660bb64f2
+size 16719

checkpoint-1000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68cff80b680ddf6e7abbef98b5f336b97f9b5963e2209307f639383870e8cc71
+size 557

checkpoint-1000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4b7a8c9615fae491ac411adbefa23ee4ae0a27ebb4cf4f6f55a147af73f533c
+size 563

checkpoint-1500/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a0c4c30117df639f85bc2f99d27765eff32414af8f41a52e6ef9c63e3429216
+size 6591685

checkpoint-1500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:536da8c93d88a2239aff4391d558d214de69e19b44032d69c8ee4c9b74af5c61
+size 3285965

checkpoint-1500/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9310e30cf8e27e0d708811d592605f0ccdf6429e6bb14aa308d1444c87992b06
+size 16719

checkpoint-1500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:203a72d6c29f42a0e2964fdddc8d7a98df1eccee78fea9de0fa416613390f5c6
+size 557

checkpoint-1500/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:618f752fb0c91576dd5a9070cf872ce0234171aa96bd3ff7a24b782d2bc652d4
+size 563

checkpoint-2000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc8aa73d11f31e98f851dc5ac6195bcdbab6a439a8245c0435cc4d2344c480b6
+size 6591685

checkpoint-2000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52e8292631286ee159204a56501523d48fde5bcd6904a0c88eabee6c59ad6072
+size 3285965

checkpoint-2000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ed377c06c2b574bdf491162327cf3b3094dfc0ba0fcff6006b15ce6bd4ab280
+size 16719

checkpoint-2000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd2de9749828adacdf103bf6e9592702bb7067a2c1df27dd62ab38c1eb8c070f
+size 557

checkpoint-2000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92ee0753920a1f5ae4ebe7eef9144f5df7cc16277682716fb4b4b52854d5d98c
+size 563

checkpoint-2500/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aace201ed4c097aee4ba690254c0ee33a1818d820fb3f789297373c1af06090e
+size 6591685

checkpoint-2500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c99be5af0c0bb2681917de4e5ac8161e7f0b94f73e40986c2ddf320f522873a3
+size 3285965

checkpoint-2500/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff321b0b6ea3c26b8f06e18a53f4eb0ea8eb6b2ff40b46a5189347385c98b8e1
+size 16719

checkpoint-2500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fbcebc8f5487b0c117b5dd47f2ea304af3eebf408d297118d9307e1223927e1
+size 557

checkpoint-2500/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb85608cdf9aeee6120a6bf917659b4cb4b80d07eaa66820bdf03b4aff3e77fd
+size 563

checkpoint-3000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29a053098abecdf72c7a212c14613d470fa8c567a6767218f7351747f8a04114
+size 6591685

checkpoint-3000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0ab9b129ce2eef2ebf283a95fa50f0d33a2de9b70085615b6df7703da78f6ad
+size 3285965

checkpoint-3000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07d2ac0cf64a1c5a61faa6e5407f98acb5da02355609ddc4c73f8e60b7acc6a4
+size 16719

checkpoint-3000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1f9398b77268202e8e1465734a63d123b1ef11c27f20f2473677e9883a6869
+size 557

checkpoint-3000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9405eb5fb0dbe2ed341f260e11bc4bfc31a90415d8dc221362ffa787977ae3e
+size 563

checkpoint-500/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97de3df572cb68542d73c9d0cd45952f59168570f52da0f1b970b84fb9f46a0e
+size 6591685

checkpoint-500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f19b17fb31608556888dbf87e3fb7166a6c9d040762a209c9b650b62c411d866
+size 3285965

checkpoint-500/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8892bf79d8077ede2fc4c5c130f413885e91a60cdfb8b857835e8053e0b4ff6c
+size 16719

checkpoint-500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3f196a54202bb4ba1220e8c59f42f9cda0702d68ea83147d814c2fb2f36b8f2
+size 557

checkpoint-500/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd3bf9a7a78805702b59d14af28a11ce03c798911f2afca318eff01a88547f4d
+size 563

pytorch_lora_weights.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20fe96ddaded71bc7f7465dfa40a7dedf7977d68870b5f2cd15d159d2ddcaa3e
+size 3287771

wandb/debug.log

@@ -0,0 +1,42 @@
+2023-07-12 00:03:08,079 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
+2023-07-12 00:03:08,080 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Configure stats pid to 3148323
+2023-07-12 00:03:08,080 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings
+2023-07-12 00:03:08,080 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings
+2023-07-12 00:03:08,080 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2023-07-12 00:03:08,080 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2023-07-12 00:03:08,080 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'}
+2023-07-12 00:03:08,080 INFO    MainThread:3148323 [wandb_init.py:_log_setup():507] Logging user logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/logs/debug.log
+2023-07-12 00:03:08,080 INFO    MainThread:3148323 [wandb_init.py:_log_setup():508] Logging internal logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/logs/debug-internal.log
+2023-07-12 00:03:08,081 INFO    MainThread:3148323 [wandb_init.py:init():547] calling init triggers
+2023-07-12 00:03:08,081 INFO    MainThread:3148323 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2023-07-12 00:03:08,081 INFO    MainThread:3148323 [wandb_init.py:init():596] starting backend
+2023-07-12 00:03:08,081 INFO    MainThread:3148323 [wandb_init.py:init():600] setting up manager
+2023-07-12 00:03:08,084 INFO    MainThread:3148323 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-07-12 00:03:08,087 INFO    MainThread:3148323 [wandb_init.py:init():606] backend started and connected
+2023-07-12 00:03:08,091 INFO    MainThread:3148323 [wandb_init.py:init():703] updated telemetry
+2023-07-12 00:03:08,092 INFO    MainThread:3148323 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
+2023-07-12 00:03:08,319 INFO    MainThread:3148323 [wandb_run.py:_on_init():2176] communicating current version
+2023-07-12 00:03:08,412 INFO    MainThread:3148323 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2023-07-12 00:03:08,412 INFO    MainThread:3148323 [wandb_init.py:init():787] starting run threads in backend
+2023-07-12 00:03:08,572 INFO    MainThread:3148323 [wandb_run.py:_console_start():2155] atexit reg
+2023-07-12 00:03:08,572 INFO    MainThread:3148323 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
+2023-07-12 00:03:08,572 INFO    MainThread:3148323 [wandb_run.py:_redirect():2075] Wrapping output streams.
+2023-07-12 00:03:08,572 INFO    MainThread:3148323 [wandb_run.py:_redirect():2100] Redirects installed.
+2023-07-12 00:03:08,573 INFO    MainThread:3148323 [wandb_init.py:init():828] run started, returning control to user process
+2023-07-12 00:03:18,853 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
+2023-07-12 00:03:18,854 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Configure stats pid to 3148323
+2023-07-12 00:03:18,854 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings
+2023-07-12 00:03:18,854 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings
+2023-07-12 00:03:18,854 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2023-07-12 00:03:18,854 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2023-07-12 00:03:18,854 INFO    MainThread:3148323 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'}
+2023-07-12 00:03:18,854 INFO    MainThread:3148323 [wandb_init.py:_log_setup():507] Logging user logs to /nfs/nfs2/home/sragas/demo/wandb/run-20230712_000318-e287bzfl/logs/debug.log
+2023-07-12 00:03:18,854 INFO    MainThread:3148323 [wandb_init.py:_log_setup():508] Logging internal logs to /nfs/nfs2/home/sragas/demo/wandb/run-20230712_000318-e287bzfl/logs/debug-internal.log
+2023-07-12 00:03:18,855 INFO    MainThread:3148323 [wandb_init.py:init():547] calling init triggers
+2023-07-12 00:03:18,855 INFO    MainThread:3148323 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2023-07-12 00:03:18,855 INFO    MainThread:3148323 [wandb_init.py:init():591] wandb.init() called when a run is still active
+2023-07-12 00:03:18,856 INFO    MainThread:3148323 [wandb_run.py:_config_callback():1283] config_cb None None {'pretrained_model_name_or_path': 'runwayml/stable-diffusion-v1-5', 'revision': None, 'dataset_name': None, 'dataset_config_name': None, 'train_data_dir': '/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/', 'image_column': 'image', 'caption_column': 'text', 'validation_prompt': 'tom cruise thumbs up', 'num_validation_images': 4, 'validation_epochs': 1, 'max_train_samples': None, 'output_dir': '/l/vision/v5/sragas/easel_ai/models_plain/', 'cache_dir': '/l/vision/v5/sragas/hf_models/', 'seed': 51, 'resolution': 512, 'center_crop': True, 'random_flip': True, 'train_batch_size': 2, 'num_train_epochs': 300, 'max_train_steps': 3000, 'gradient_accumulation_steps': 4, 'gradient_checkpointing': False, 'learning_rate': 1e-05, 'scale_lr': False, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'dataloader_num_workers': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'push_to_hub': True, 'hub_token': None, 'prediction_type': None, 'hub_model_id': 'person-thumbs-up-plain-lora', 'logging_dir': 'logs', 'mixed_precision': None, 'report_to': 'wandb', 'local_rank': 0, 'checkpointing_steps': 500, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'enable_xformers_memory_efficient_attention': False, 'noise_offset': 0, 'rank': 4}
+2023-07-12 12:35:49,765 WARNING MsgRouterThr:3148323 [router.py:message_loop():77] message_loop has been closed

wandb/run-20230711_182238-06r7b4m3/files/config.yaml

@@ -0,0 +1,39 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.8.10
+    cli_version: 0.15.4
+    framework: huggingface
+    huggingface_version: 4.30.2
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    start_time: 1689114158.539519
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      3:
+      - 23
+      4: 3.8.10
+      5: 0.15.4
+      6: 4.30.2
+      8:
+      - 2
+      - 5

wandb/run-20230711_182238-06r7b4m3/files/output.log

@@ -0,0 +1,7 @@
+07/11/2023 18:22:40 - INFO - __main__ - Distributed environment: MULTI_GPU  Backend: nccl
+Num processes: 4
+Process index: 1
+Local process index: 1
+Device: cuda:1
+Mixed precision type: fp16
+Resolving data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 44533.36it/s]

wandb/run-20230711_182238-06r7b4m3/files/requirements.txt

@@ -0,0 +1,165 @@
+absl-py==1.4.0
+accelerate==0.20.3
+aiohttp==3.8.4
+aiosignal==1.3.1
+anyio==3.7.0
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==21.3.0
+asttokens==2.2.1
+async-lru==2.0.2
+async-timeout==4.0.2
+attrs==23.1.0
+babel==2.12.1
+backcall==0.2.0
+beautifulsoup4==4.12.2
+bleach==6.0.0
+cachetools==5.3.1
+certifi==2023.5.7
+cffi==1.15.1
+charset-normalizer==3.1.0
+click==8.1.3
+cmake==3.26.4
+comm==0.1.3
+datasets==2.13.1
+debugpy==1.6.7
+decorator==5.1.1
+defusedxml==0.7.1
+diffusers==0.18.0.dev0
+dill==0.3.6
+docker-pycreds==0.4.0
+exceptiongroup==1.1.2
+executing==1.2.0
+fastjsonschema==2.17.1
+filelock==3.12.2
+frozenlist==1.3.3
+fsspec==2023.6.0
+ftfy==6.1.1
+gitdb==4.0.10
+gitpython==3.1.31
+google-auth-oauthlib==1.0.0
+google-auth==2.21.0
+grpcio==1.56.0
+huggingface-hub==0.15.1
+idna==3.4
+importlib-metadata==6.7.0
+importlib-resources==5.12.0
+ipykernel==6.24.0
+ipython==8.12.2
+jedi==0.18.2
+jinja2==3.1.2
+json5==0.9.14
+jsonschema==4.17.3
+jupyter-client==8.3.0
+jupyter-core==5.3.1
+jupyter-events==0.6.3
+jupyter-lsp==2.2.0
+jupyter-server-terminals==0.4.4
+jupyter-server==2.7.0
+jupyterlab-pygments==0.2.2
+jupyterlab-server==2.23.0
+jupyterlab==4.0.2
+lit==16.0.6
+markdown==3.4.3
+markupsafe==2.1.3
+matplotlib-inline==0.1.6
+mistune==3.0.1
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+mypy-extensions==1.0.0
+nbclient==0.8.0
+nbconvert==7.6.0
+nbformat==5.9.0
+nest-asyncio==1.5.6
+networkx==3.1
+notebook-shim==0.2.3
+numpy==1.24.4
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+oauthlib==3.2.2
+overrides==7.3.1
+packaging==23.1
+pandas==2.0.3
+pandocfilters==1.5.0
+parso==0.8.3
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==10.0.0
+pip==20.0.2
+pkg-resources==0.0.0
+pkgutil-resolve-name==1.3.10
+platformdirs==3.8.0
+prometheus-client==0.17.0
+prompt-toolkit==3.0.39
+protobuf==4.23.3
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==12.0.1
+pyasn1-modules==0.3.0
+pyasn1==0.5.0
+pycparser==2.21
+pygments==2.15.1
+pyre-extensions==0.0.29
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+pytz==2023.3
+pyyaml==6.0
+pyzmq==25.1.0
+regex==2023.6.3
+requests-oauthlib==1.3.1
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rsa==4.9
+safetensors==0.3.1
+send2trash==1.8.2
+sentry-sdk==1.27.0
+setproctitle==1.3.2
+setuptools==44.0.0
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+soupsieve==2.4.1
+stack-data==0.6.2
+sympy==1.12
+tensorboard-data-server==0.7.1
+tensorboard==2.13.0
+terminado==0.17.1
+tinycss2==1.2.1
+tokenizers==0.13.3
+tomli==2.0.1
+torch==2.0.1
+torchaudio==2.0.2
+torchvision==0.15.2
+tornado==6.3.2
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.30.2
+triton==2.0.0
+typing-extensions==4.7.1
+typing-inspect==0.9.0
+tzdata==2023.3
+urllib3==2.0.3
+wandb==0.15.4
+wcwidth==0.2.6
+webencodings==0.5.1
+websocket-client==1.6.1
+werkzeug==2.3.6
+wheel==0.40.0
+xformers==0.0.20
+xxhash==3.2.0
+yarl==1.9.2
+zipp==3.15.0

wandb/run-20230711_182238-06r7b4m3/files/wandb-metadata.json

@@ -0,0 +1,431 @@
+{
+    "os": "Linux-5.15.0-52-generic-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2023-07-11T22:22:40.405929",
+    "startedAt": "2023-07-11T22:22:38.515893",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5",
+        "--train_data_dir=/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/",
+        "--resolution=512",
+        "--center_crop",
+        "--random_flip",
+        "--train_batch_size=2",
+        "--gradient_accumulation_steps=4",
+        "--num_train_epochs=300",
+        "--learning_rate=1e-5",
+        "--max_grad_norm=1",
+        "--lr_scheduler=cosine",
+        "--lr_warmup_steps=500",
+        "--output_dir=/l/vision/v5/sragas/easel_ai/models_plain/",
+        "--checkpointing_steps=500",
+        "--report_to=wandb",
+        "--validation_prompt=tom cruise thumbs up",
+        "--seed=15",
+        "--push_to_hub",
+        "--hub_model_id=person-thumbs-up-plain-lora"
+    ],
+    "state": "running",
+    "program": "train_text_to_image_lora.py",
+    "codePath": "train_text_to_image_lora.py",
+    "host": "snorlax",
+    "username": "sragas",
+    "executable": "/nfs/blitzle/home/data/vision5/sragas/easel_venv/bin/python3",
+    "cpu_count": 36,
+    "cpu_count_logical": 72,
+    "cpu_freq": {
+        "current": 1158.2238333333332,
+        "min": 1000.0,
+        "max": 3900.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2700.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1100.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 3614.641,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1100.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1100.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2500.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2043.382,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1100.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1100.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2200.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 2400.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 1000.0,
+            "min": 1000.0,
+            "max": 3900.0
+        },
+        {
+            "current": 3700.0,
+            "min": 1000.0,
+            "max": 3900.0
+        }
+    ],
+    "disk": {
+        "total": 490.5871887207031,
+        "used": 102.94281387329102
+    },
+    "gpu": "Quadro RTX 6000",
+    "gpu_count": 4,
+    "gpu_devices": [
+        {
+            "name": "Quadro RTX 6000",
+            "memory_total": 25769803776
+        },
+        {
+            "name": "Quadro RTX 6000",
+            "memory_total": 25769803776
+        },
+        {
+            "name": "Quadro RTX 6000",
+            "memory_total": 25769803776
+        },
+        {
+            "name": "Quadro RTX 6000",
+            "memory_total": 25769803776
+        }
+    ],
+    "memory": {
+        "total": 502.5611343383789
+    }
+}

wandb/run-20230711_182238-06r7b4m3/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20230711_182238-06r7b4m3/logs/debug.log

@@ -0,0 +1,27 @@
+2023-07-11 18:22:38,531 INFO    MainThread:2301801 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
+2023-07-11 18:22:38,531 INFO    MainThread:2301801 [wandb_setup.py:_flush():76] Configure stats pid to 2301801
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'}
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_init.py:_log_setup():507] Logging user logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/logs/debug.log
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_init.py:_log_setup():508] Logging internal logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/logs/debug-internal.log
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_init.py:init():547] calling init triggers
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2023-07-11 18:22:38,532 INFO    MainThread:2301801 [wandb_init.py:init():596] starting backend
+2023-07-11 18:22:38,533 INFO    MainThread:2301801 [wandb_init.py:init():600] setting up manager
+2023-07-11 18:22:38,535 INFO    MainThread:2301801 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-07-11 18:22:38,539 INFO    MainThread:2301801 [wandb_init.py:init():606] backend started and connected
+2023-07-11 18:22:38,542 INFO    MainThread:2301801 [wandb_init.py:init():703] updated telemetry
+2023-07-11 18:22:38,543 INFO    MainThread:2301801 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
+2023-07-11 18:22:40,245 INFO    MainThread:2301801 [wandb_run.py:_on_init():2176] communicating current version
+2023-07-11 18:22:40,322 INFO    MainThread:2301801 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2023-07-11 18:22:40,322 INFO    MainThread:2301801 [wandb_init.py:init():787] starting run threads in backend
+2023-07-11 18:22:40,470 INFO    MainThread:2301801 [wandb_run.py:_console_start():2155] atexit reg
+2023-07-11 18:22:40,470 INFO    MainThread:2301801 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
+2023-07-11 18:22:40,470 INFO    MainThread:2301801 [wandb_run.py:_redirect():2075] Wrapping output streams.
+2023-07-11 18:22:40,470 INFO    MainThread:2301801 [wandb_run.py:_redirect():2100] Redirects installed.
+2023-07-11 18:22:40,471 INFO    MainThread:2301801 [wandb_init.py:init():828] run started, returning control to user process

wandb/run-20230711_182238-06r7b4m3/run-06r7b4m3.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9efbe5c7657be984345fe118e106fc277da1f713a6462629a49bba257c817235
+size 1247842

wandb/run-20230711_182238-k29fqnbt/files/config.yaml

@@ -0,0 +1,187 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.8.10
+    cli_version: 0.15.4
+    framework: huggingface
+    huggingface_version: 4.30.2
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    start_time: 1689114158.56903
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 83
+      3:
+      - 23
+      - 24
+      4: 3.8.10
+      5: 0.15.4
+      6: 4.30.2
+      8:
+      - 2
+      - 5
+pretrained_model_name_or_path:
+  desc: null
+  value: runwayml/stable-diffusion-v1-5
+revision:
+  desc: null
+  value: null
+dataset_name:
+  desc: null
+  value: null
+dataset_config_name:
+  desc: null
+  value: null
+train_data_dir:
+  desc: null
+  value: /l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/
+image_column:
+  desc: null
+  value: image
+caption_column:
+  desc: null
+  value: text
+validation_prompt:
+  desc: null
+  value: tom cruise thumbs up
+num_validation_images:
+  desc: null
+  value: 4
+validation_epochs:
+  desc: null
+  value: 1
+max_train_samples:
+  desc: null
+  value: null
+output_dir:
+  desc: null
+  value: /l/vision/v5/sragas/easel_ai/models_plain/
+cache_dir:
+  desc: null
+  value: /l/vision/v5/sragas/hf_models/
+seed:
+  desc: null
+  value: 15
+resolution:
+  desc: null
+  value: 512
+center_crop:
+  desc: null
+  value: true
+random_flip:
+  desc: null
+  value: true
+train_batch_size:
+  desc: null
+  value: 2
+num_train_epochs:
+  desc: null
+  value: 300
+max_train_steps:
+  desc: null
+  value: 2400
+gradient_accumulation_steps:
+  desc: null
+  value: 4
+gradient_checkpointing:
+  desc: null
+  value: false
+learning_rate:
+  desc: null
+  value: 1.0e-05
+scale_lr:
+  desc: null
+  value: false
+lr_scheduler:
+  desc: null
+  value: cosine
+lr_warmup_steps:
+  desc: null
+  value: 500
+snr_gamma:
+  desc: null
+  value: null
+use_8bit_adam:
+  desc: null
+  value: false
+allow_tf32:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_weight_decay:
+  desc: null
+  value: 0.01
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+push_to_hub:
+  desc: null
+  value: true
+hub_token:
+  desc: null
+  value: null
+prediction_type:
+  desc: null
+  value: null
+hub_model_id:
+  desc: null
+  value: person-thumbs-up-plain-lora
+logging_dir:
+  desc: null
+  value: logs
+mixed_precision:
+  desc: null
+  value: null
+report_to:
+  desc: null
+  value: wandb
+local_rank:
+  desc: null
+  value: 0
+checkpointing_steps:
+  desc: null
+  value: 500
+checkpoints_total_limit:
+  desc: null
+  value: null
+resume_from_checkpoint:
+  desc: null
+  value: null
+enable_xformers_memory_efficient_attention:
+  desc: null
+  value: false
+noise_offset:
+  desc: null
+  value: 0
+rank:
+  desc: null
+  value: 4