{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "abba1d18-4a4d-4978-9766-82f3940e411e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: auto-gptq in /opt/conda/lib/python3.10/site-packages (0.7.1)\n",
      "Requirement already satisfied: accelerate>=0.26.0 in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (0.31.0)\n",
      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (2.20.0)\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (0.2.0)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (1.26.3)\n",
      "Requirement already satisfied: rouge in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (1.0.1)\n",
      "Requirement already satisfied: gekko in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (1.1.3)\n",
      "Requirement already satisfied: torch>=1.13.0 in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (2.2.0)\n",
      "Requirement already satisfied: safetensors in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (0.4.3)\n",
      "Requirement already satisfied: transformers>=4.31.0 in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (4.43.0.dev0)\n",
      "Requirement already satisfied: peft>=0.5.0 in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (0.11.1)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (4.66.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.26.0->auto-gptq) (23.1)\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.26.0->auto-gptq) (5.9.0)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.26.0->auto-gptq) (6.0.1)\n",
      "Requirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.26.0->auto-gptq) (0.23.4)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (3.13.1)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (4.9.0)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (1.12)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (3.1)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (2023.12.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.31.0->auto-gptq) (2024.5.15)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers>=4.31.0->auto-gptq) (2.32.3)\n",
      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.31.0->auto-gptq) (0.19.1)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (16.1.0)\n",
      "Requirement already satisfied: pyarrow-hotfix in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (0.3.8)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (2.2.2)\n",
      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (0.70.16)\n",
      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (3.9.5)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from rouge->auto-gptq) (1.16.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (4.0.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.31.0->auto-gptq) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.31.0->auto-gptq) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.31.0->auto-gptq) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.31.0->auto-gptq) (2023.11.17)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.13.0->auto-gptq) (2.1.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->auto-gptq) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->auto-gptq) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->auto-gptq) (2024.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.13.0->auto-gptq) (1.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mERROR: unknown command \"insall\" - maybe you meant \"install\"\n"
     ]
    }
   ],
   "source": [
    "!pip install auto-gptq\n",
    "!pip insall --upgrade transformers optimum accelerate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cefbcaa-9c28-4d68-b15e-9889f73332dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install parallelformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "bf65bf24-e67b-4af7-8890-fba4182584e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7abb0ecf7aa42b3a5ade8aa6644cbbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import HfApi, notebook_login\n",
    "\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fe3c29b7-1946-4f44-9d07-e9798e656bc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1bff45ce-9a64-4ceb-97e0-0e2ab5f15c45",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparams\n",
    "\n",
    "quant_model_repo = \"Granther/Gemma-2-9B-Instruct-4Bit-GPTQ\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5b060dc1-4a95-476f-8707-412a4452dd89",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/modeling_utils.py:4565: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fceb82427c58488da421d6415e0df516",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from optimum.gptq import load_quantized_model\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(quant_model_repo, device_map='auto')\n",
    "tokenizer = AutoTokenizer.from_pretrained(quant_model_repo)\n",
    "\n",
    "#model = load_quantized_model(model, device_map=\"auto\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "440a9d11-8fb0-4ded-885f-52a226e326d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[    2,   106,  1645,   108, 27445,   682,  1105, 20731,   107,   108,\n",
       "           106,  2516,   108]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_text = [\n",
    "    {\"role\": \"user\", \"content\": \"Tell me about paris\"},\n",
    "]\n",
    "\n",
    "prompt = tokenizer.apply_chat_template(input_text, tokenize=True, add_generation_prompt=True, return_tensors='pt')\n",
    "# >>> '<bos><start_of_turn>user\\nTell me about paris<end_of_turn>\\n<start_of_turn>model\\n' \n",
    "\n",
    "prompt\n",
    "\n",
    "#prompt = {key: value.to(model.device) for key, value in prompt}\n",
    "\n",
    "# with torch.no_grad():\n",
    "#     outputs = model.generate(**inputs, max_length=100, num_return_sequences=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9ed28a1b-2c65-4762-8b7c-4338195729af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Tell me about python \\n\\nLet's talk about Python!\\n\\n**What is Python?**\\n\\nPython is a high-level, interpreted, general-purpose programming language.  Let's break down what that means:\\n\\n* **High-level:** Python is designed to be easy for humans to read and write. It uses words and symbols that are closer to natural language than the low-level instructions computers directly understand.\\n* **Interpreted:** Python code is executed line\""
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f040240-5bb0-41a1-a823-678f5ffb3dc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "chat = [\n",
    "    { \"role\": \"user\", \"content\": \"Write a hello world program\" },\n",
    "]\n",
    "prompt = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9cf56d96-bf6c-43cd-a062-aa8368b3406b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "pipe = pipeline('text-generation', model=model, tokenizer=quant_model_repo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eb34f0bc-9a4b-4f8b-baf9-9869e2d1bef1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'generated_text': 'hello, how are you?\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n'}]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe(\"hello, how are you\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52a00f3b-e601-4800-9708-5f656c54ffc8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}