Update requirements.txt with new package versions

requirements.txt

@@ -1,16 +1,19 @@
 absl-py==2.0.0
-accelerate==0.25.0
+accelerate==0.24.0
 aiofiles==23.2.1
 aiohttp==3.9.1
+aioitertools==0.11.0
 aiosignal==1.3.1
 albumentations==1.3.1
 alembic==1.13.0
 altair==5.2.0
 annotated-types==0.6.0
 anyio==3.7.1
+appnope==0.1.3
 arrow==1.3.0
+asttokens==2.4.1
 attrs==23.1.0
-autotrain-advanced==0.6.59
+autotrain-advanced==0.6.51
 bitsandbytes==0.41.0
 Brotli==1.1.0
 cachetools==5.3.2
@@ -20,73 +23,128 @@ click==8.1.7
 cmaes==0.10.0
 codecarbon==2.2.3
 colorlog==6.8.0
+comm==0.2.0
 contourpy==1.2.0
 cycler==0.12.1
+dataclasses-json==0.6.3
 datasets==2.14.7
+debugpy==1.8.0
+decorator==5.1.1
 diffusers==0.21.4
 dill==0.3.7
+distro==1.8.0
+dnspython==2.4.2
 docstring-parser==0.15
 einops==0.6.1
 evaluate==0.3.0
+executing==2.0.1
 fastapi==0.104.1
 ffmpy==0.3.1
 filelock==3.13.1
-fonttools==4.47.0
-frozenlist==1.4.1
+fonttools==4.46.0
+frozenlist==1.4.0
 fsspec==2023.10.0
 fuzzywuzzy==0.18.0
-google-auth==2.25.2
-google-auth-oauthlib==1.2.0
+google-api-core==2.14.0
+google-auth==2.24.0
+google-auth-oauthlib==1.1.0
+google-cloud-core==2.3.3
+google-cloud-storage==2.13.0
+google-crc32c==1.5.0
+google-resumable-media==2.6.0
+googleapis-common-protos==1.61.0
 gradio==3.41.0
 gradio_client==0.5.0
-grpcio==1.60.0
+greenlet==3.0.2
+grpcio==1.59.3
+gunicorn==21.2.0
 h11==0.14.0
-hf_transfer==0.1.4
 httpcore==1.0.2
 httpx==0.25.2
 huggingface-hub==0.19.4
 idna==3.6
-imageio==2.33.1
+imageio==2.33.0
 importlib-metadata==7.0.0
 importlib-resources==6.1.1
 inflate64==1.0.0
 invisible-watermark==0.2.0
 ipadic==1.0.0
+ipykernel==6.27.1
+ipython==8.18.1
+ipywidgets==8.1.1
+jedi==0.19.1
 Jinja2==3.1.2
 jiwer==3.0.2
 joblib==1.3.1
+jsonlines==4.0.0
+jsonpatch==1.33
+jsonpointer==2.4
 jsonschema==4.20.0
 jsonschema-specifications==2023.11.2
+jupyter_client==8.6.0
+jupyter_core==5.5.0
+jupyterlab-widgets==3.0.9
 kiwisolver==1.4.5
+langchain==0.0.345
+langchain-core==0.0.9
+langsmith==0.0.69
 lazy_loader==0.3
 loguru==0.7.0
 Mako==1.3.0
 Markdown==3.5.1
 markdown-it-py==3.0.0
 MarkupSafe==2.1.3
+marshmallow==3.20.1
 matplotlib==3.8.2
+matplotlib-inline==0.1.6
 mdurl==0.1.2
+mmh3==3.1.0
 mpmath==1.3.0
 multidict==6.0.4
 multiprocess==0.70.15
 multivolumefile==0.2.3
+mypy-extensions==1.0.0
+nest-asyncio==1.5.8
 networkx==3.2.1
 nltk==3.8.1
-numpy==1.26.2
+numpy==1.25.2
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu12==12.1.105
 oauthlib==3.2.2
-# opencv-python==4.8.1.78
+openai==1.3.7
+opencv-python==4.8.1.78
 opencv-python-headless==4.8.1.78
 optuna==3.3.0
 orjson==3.9.10
 packaging==23.1
 pandas==2.1.4
-peft==0.7.1
+pandas-stubs==2.0.3.230814
+parso==0.8.3
+peft==0.6.2
+pexpect==4.9.0
+pi==0.1.2
 Pillow==10.0.0
+pinecone-client==2.2.4
+pinecone-text==0.7.1
+platformdirs==4.1.0
+prompt-toolkit==3.0.41
 protobuf==4.23.4
-psutil==5.9.7
+psutil==5.9.6
+ptyprocess==0.7.0
+pure-eval==0.2.2
 py-cpuinfo==9.0.0
 py7zr==0.20.6
-pyarrow==14.0.2
+pyarrow==11.0.0
 pyarrow-hotfix==0.6
 pyasn1==0.5.1
 pyasn1-modules==0.3.0
@@ -96,26 +154,28 @@ pydantic==2.4.2
 pydantic_core==2.10.1
 pydub==0.25.1
 Pygments==2.17.2
-pyngrok==7.0.3
 pynvml==11.5.0
 pyparsing==3.1.1
+pypdf==3.17.1
 pyppmd==1.0.0
 python-dateutil==2.8.2
+python-dotenv==1.0.0
 python-multipart==0.0.6
 pytz==2023.3.post1
 PyWavelets==1.5.0
 PyYAML==6.0.1
+pyzmq==25.1.2
 pyzstd==0.15.9
 qudida==0.0.4
 rapidfuzz==2.13.7
-referencing==0.32.0
+referencing==0.31.1
 regex==2023.10.3
 requests==2.31.0
 requests-oauthlib==1.3.1
 responses==0.18.0
 rich==13.7.0
 rouge-score==0.1.2
-rpds-py==0.15.2
+rpds-py==0.13.2
 rsa==4.9
 sacremoses==0.0.53
 safetensors==0.4.1
@@ -128,29 +188,44 @@ shtab==1.6.5
 six==1.16.0
 sniffio==1.3.0
 SQLAlchemy==2.0.23
+sse-starlette==1.8.2
+stack-data==0.6.3
 starlette==0.27.0
 sympy==1.12
+tenacity==8.2.3
 tensorboard==2.15.1
 tensorboard-data-server==0.7.2
 texttable==1.7.0
 threadpoolctl==3.2.0
-tifffile==2023.12.9
+tifffile==2023.9.26
 tiktoken==0.5.1
 tokenizers==0.15.0
 toolz==0.12.0
-torch==2.1.2
+torch==2.1.1
+tornado==6.4
 tqdm==4.65.0
-transformers==4.36.1
+traitlets==5.14.0
+transformers==4.35.2
+triton==2.1.0
 trl==0.7.4
+types-jsonschema==4.20.0.0
 types-python-dateutil==2.8.19.14
-typing_extensions==4.9.0
+types-pytz==2023.3.1.1
+types-PyYAML==6.0.12.12
+types-tqdm==4.66.0.5
+typing-inspect==0.9.0
+typing_extensions==4.8.0
 tyro==0.6.0
 tzdata==2023.3
-urllib3==2.1.0
+urllib3==2.0.7
 uvicorn==0.22.0
+wcwidth==0.2.12
 websockets==11.0.3
 Werkzeug==2.3.6
+wget==3.2
+widgetsnbextension==4.0.9
+wrapt==1.16.0
 xgboost==1.7.6
 xxhash==3.4.1
-yarl==1.9.4
+yarl==1.9.3
 zipp==3.17.0

train_llm.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 51,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -17,7 +17,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 52,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -35,7 +35,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 53,
    "metadata": {},
    "outputs": [
     {
@@ -57,7 +57,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 54,
    "metadata": {},
    "outputs": [
     {
@@ -201,28 +201,39 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 49,
+   "execution_count": 50,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "⚠️ WARNING | 2023-12-22 10:39:42 | autotrain.cli.run_dreambooth:<module>:14 - ❌ Some DreamBooth components are missing! Please run `autotrain setup` to install it. Ignore this warning if you are not using DreamBooth or running `autotrain setup` already.\n",
-      "usage: autotrain <command> [<args>]\n",
-      "AutoTrain advanced CLI: error: unrecognized arguments: --batch_size 2\n"
+      "⚠️ WARNING | 2023-12-22 10:41:00 | autotrain.cli.run_dreambooth:<module>:14 - ❌ Some DreamBooth components are missing! Please run `autotrain setup` to install it. Ignore this warning if you are not using DreamBooth or running `autotrain setup` already.\n",
+      "Traceback (most recent call last):\n",
+      "  File \"/home/dsmueller/Repositories/HuggingFace/autotrain-playground/.venv/bin/autotrain\", line 8, in <module>\n",
+      "    sys.exit(main())\n",
+      "             ^^^^^^\n",
+      "  File \"/home/dsmueller/Repositories/HuggingFace/autotrain-playground/.venv/lib/python3.11/site-packages/autotrain/cli/autotrain.py\", line 47, in main\n",
+      "    command = args.func(args)\n",
+      "              ^^^^^^^^^^^^^^^\n",
+      "  File \"/home/dsmueller/Repositories/HuggingFace/autotrain-playground/.venv/lib/python3.11/site-packages/autotrain/cli/run_llm.py\", line 14, in run_llm_command_factory\n",
+      "    return RunAutoTrainLLMCommand(args)\n",
+      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "  File \"/home/dsmueller/Repositories/HuggingFace/autotrain-playground/.venv/lib/python3.11/site-packages/autotrain/cli/run_llm.py\", line 473, in __init__\n",
+      "    raise ValueError(\"No GPU/MPS device found. LLM training requires an accelerator\")\n",
+      "ValueError: No GPU/MPS device found. LLM training requires an accelerator\n"
      ]
     },
     {
      "ename": "CalledProcessError",
-     "evalue": "Command '\nautotrain llm --train     --trainer sft     --project_name ./llms/ams_data_train-100_6abb23dc-cb9d-428e-9079-e47deee0edd9     --model mistralai/Mistral-7B-v0.1     --data_path ./fine_tune_data/     --train_split train_data     --valid_split validation_data     --repo_id ai-aerospace/ams-data-train-100-4601c8c8-0903-4f18-a6e8-1d2a40a697ce     --push_to_hub     --token HUGGINGFACE_TOKEN     --block_size 2242     --model_max_length 1121     --logging_steps -1     --evaluation_strategy epoch     --save_total_limit 1     --save_strategy epoch     --fp16     --lr 3e-05     --num_train_epochs 3     --batch_size 2     --warmup_ratio 0.1     --gradient_accumulation 1     --optimizer adamw_torch     --scheduler linear     --weight_decay 0     --max_grad_norm 1     --seed 42     --use_int4     --use-peft     --lora_r 16     --lora_alpha 32     --lora_dropout 0.05\n' returned non-zero exit status 2.",
+     "evalue": "Command '\nautotrain llm --train     --trainer sft     --project_name ./llms/ams_data_train-100_6abb23dc-cb9d-428e-9079-e47deee0edd9     --model mistralai/Mistral-7B-v0.1     --data_path ./fine_tune_data/     --train_split train_data     --valid_split validation_data     --repo_id ai-aerospace/ams-data-train-100-4601c8c8-0903-4f18-a6e8-1d2a40a697ce     --push_to_hub     --token HUGGINGFACE_TOKEN     --block_size 2242     --model_max_length 1121     --logging_steps -1     --evaluation_strategy epoch     --save_total_limit 1     --save_strategy epoch     --fp16     --lr 3e-05     --num_train_epochs 3     --train_batch_size 2     --warmup_ratio 0.1     --gradient_accumulation 1     --optimizer adamw_torch     --scheduler linear     --weight_decay 0     --max_grad_norm 1     --seed 42     --use_int4     --use-peft     --lora_r 16     --lora_alpha 32     --lora_dropout 0.05\n' returned non-zero exit status 1.",
      "output_type": "error",
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mCalledProcessError\u001b[0m                        Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[49], line 40\u001b[0m\n\u001b[1;32m      4\u001b[0m command\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;124mautotrain llm --train \u001b[39m\u001b[38;5;130;01m\\\u001b[39;00m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;124m    --trainer sft \u001b[39m\u001b[38;5;130;01m\\\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[38;5;124m    --lora_dropout \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_params[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlora_dropout\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m     37\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m     39\u001b[0m \u001b[38;5;66;03m# Use subprocess.run() to execute the command\u001b[39;00m\n\u001b[0;32m---> 40\u001b[0m \u001b[43msubprocess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshell\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
+      "Cell \u001b[0;32mIn[50], line 40\u001b[0m\n\u001b[1;32m      4\u001b[0m command\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;124mautotrain llm --train \u001b[39m\u001b[38;5;130;01m\\\u001b[39;00m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;124m    --trainer sft \u001b[39m\u001b[38;5;130;01m\\\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[38;5;124m    --lora_dropout \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_params[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlora_dropout\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m     37\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m     39\u001b[0m \u001b[38;5;66;03m# Use subprocess.run() to execute the command\u001b[39;00m\n\u001b[0;32m---> 40\u001b[0m \u001b[43msubprocess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshell\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
       "File \u001b[0;32m/usr/lib/python3.11/subprocess.py:571\u001b[0m, in \u001b[0;36mrun\u001b[0;34m(input, capture_output, timeout, check, *popenargs, **kwargs)\u001b[0m\n\u001b[1;32m    569\u001b[0m     retcode \u001b[38;5;241m=\u001b[39m process\u001b[38;5;241m.\u001b[39mpoll()\n\u001b[1;32m    570\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m check \u001b[38;5;129;01mand\u001b[39;00m retcode:\n\u001b[0;32m--> 571\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m CalledProcessError(retcode, process\u001b[38;5;241m.\u001b[39margs,\n\u001b[1;32m    572\u001b[0m                                  output\u001b[38;5;241m=\u001b[39mstdout, stderr\u001b[38;5;241m=\u001b[39mstderr)\n\u001b[1;32m    573\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m CompletedProcess(process\u001b[38;5;241m.\u001b[39margs, retcode, stdout, stderr)\n",
-      "\u001b[0;31mCalledProcessError\u001b[0m: Command '\nautotrain llm --train     --trainer sft     --project_name ./llms/ams_data_train-100_6abb23dc-cb9d-428e-9079-e47deee0edd9     --model mistralai/Mistral-7B-v0.1     --data_path ./fine_tune_data/     --train_split train_data     --valid_split validation_data     --repo_id ai-aerospace/ams-data-train-100-4601c8c8-0903-4f18-a6e8-1d2a40a697ce     --push_to_hub     --token HUGGINGFACE_TOKEN     --block_size 2242     --model_max_length 1121     --logging_steps -1     --evaluation_strategy epoch     --save_total_limit 1     --save_strategy epoch     --fp16     --lr 3e-05     --num_train_epochs 3     --batch_size 2     --warmup_ratio 0.1     --gradient_accumulation 1     --optimizer adamw_torch     --scheduler linear     --weight_decay 0     --max_grad_norm 1     --seed 42     --use_int4     --use-peft     --lora_r 16     --lora_alpha 32     --lora_dropout 0.05\n' returned non-zero exit status 2."
+      "\u001b[0;31mCalledProcessError\u001b[0m: Command '\nautotrain llm --train     --trainer sft     --project_name ./llms/ams_data_train-100_6abb23dc-cb9d-428e-9079-e47deee0edd9     --model mistralai/Mistral-7B-v0.1     --data_path ./fine_tune_data/     --train_split train_data     --valid_split validation_data     --repo_id ai-aerospace/ams-data-train-100-4601c8c8-0903-4f18-a6e8-1d2a40a697ce     --push_to_hub     --token HUGGINGFACE_TOKEN     --block_size 2242     --model_max_length 1121     --logging_steps -1     --evaluation_strategy epoch     --save_total_limit 1     --save_strategy epoch     --fp16     --lr 3e-05     --num_train_epochs 3     --train_batch_size 2     --warmup_ratio 0.1     --gradient_accumulation 1     --optimizer adamw_torch     --scheduler linear     --weight_decay 0     --max_grad_norm 1     --seed 42     --use_int4     --use-peft     --lora_r 16     --lora_alpha 32     --lora_dropout 0.05\n' returned non-zero exit status 1."
      ]
     }
    ],