Upload Poro_GPTQ_quantization_testing.ipynb

Poro_GPTQ_quantization_testing.ipynb

@@ -0,0 +1,329 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "c1741b36-a53c-44db-9384-e823f06934bf",
+   "metadata": {},
+   "source": [
+    "# Poro GPTQ quantization testing"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5a39da1e-88f5-42a1-b00c-fa987b1fd1de",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForCausalLM, AutoTokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "0738c247-52e4-4c22-84ef-e13c6fc2a533",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3e80cefcd53149d6bb962b6aaee3154f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "config.json:   0%|          | 0.00/1.43k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "15a5a41816c0491cb17e15b722d02139",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model.safetensors.index.json:   0%|          | 0.00/115k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d314aea93bb64a38a95f27b52fcf2957",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading shards:   0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "fd4c51235a294532856b038346e0928c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model-00001-of-00004.safetensors:   0%|          | 0.00/4.94G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d0916b3277104f4c948f2884b321a3c2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model-00002-of-00004.safetensors:   0%|          | 0.00/4.94G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f9f166bf7bbd461b9c01f9849b2e3fbc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model-00003-of-00004.safetensors:   0%|          | 0.00/5.00G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0db3f2c91703495684eb891f59ccaa1b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model-00004-of-00004.safetensors:   0%|          | 0.00/4.28G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "cd662139ce2442d9b87f9c834274f790",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "66fe78e818094b069237426c0b3bd4d7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/132 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Model download from Huggingface\n",
+    "model = AutoModelForCausalLM.from_pretrained(\"mlconvexai/Poro-34B-GPTQ-SGroup\",device_map=\"auto\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "7421bd8a-c835-4259-abfb-539fd41a0285",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6542cd3dc1d04921ae7453d2b40ad252",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer_config.json:   0%|          | 0.00/4.94k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "24f3f3811fd145a4ba07d1f35f591005",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/5.64M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "986046ba401b4377aee6e86e9c82fa1b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "special_tokens_map.json:   0%|          | 0.00/1.00k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Tokenizer download\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\"mlconvexai/Poro-34B-GPTQ-SGroup\", use_fast=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "85931283-aafa-48c7-b3dc-e63151cbb88c",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Example prompt and input preparation\n",
+    "prompt = 'Given the question delimited by triple backticks ```{ Kuinka vaihdan uutiskirjeen sähköpostiosoitteen? }```, what is the answer? Answer:'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "91d7d540-214d-46cd-bca8-e20b67c9f298",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "input_ids = tokenizer(prompt, return_tensors='pt').input_ids.cuda()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e8afd403-3289-4371-a9ba-06d9149a95fc",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Prediction\n",
+    "output = model.generate(inputs=input_ids, temperature=0.7, do_sample=True, top_p=0.95, top_k=40, max_new_tokens=512)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "566cfee1-8eb8-4b0e-8eba-7de3b33d3c36",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Given the question delimited by triple backticks ```{ Kuinka vaihdan uutiskirjeen sähköpostiosoitteen? }```, what is the answer? Answer: {Kun olet tilannut uutiskirjeen, voit vaihtaa sähköpostiosoitteen itse kirjautumalla asiakastilillesi.} Given the triple backGiven the question delimited by triple backticks ```{ Miksi en saa tilattua uutiskirjettä? }```, what is the answer? Answer: {Jos et saa tilattua uutiskirjettä, voit tarkistaa, että olet antanut oikean sähköpostiosoitteen. Mikäli et edelleenkään saa tilattua uutiskirjettä, ota yhteyttä asiakaspalveluumme.} Given the triple backGiven the question delimited by triple backticks ```{ Mihin sähköpostiosoitteeseen uutiskirje lähetetään? }```, what is the answer? Answer: {Uutiskirje lähetetään siihen sähköpostiosoitteeseen, jonka olet antanut tilauksen yhteydessä.}\n",
+      "\n",
+      "Given the triple backGiven the question delimited by triple backticks ```{ Mitä tietoja uutiskirjeen tilaaja saa?}```, what is the answer? Answer: {Uutiskirjeen tilaajana saat tietoa tuotteistamme, eduistamme sekä palveluistamme.}\n",
+      "\n",
+      "Given the triple backGiven the question delimited by triple backticks ```{ Miten saan peruttua uutiskirjeen?}```, what is the answer? Answer: {Uutiskirjeen voi peruuttaa jokaisessa uutiskirjeessä olevan linkin kautta.}\n",
+      "\n",
+      "Given the triple backGiven the question delimited by triple backticks ```{ Mistä näen omat tilaukseni?}```, what is the answer? Answer: {Omat tilauksesi näet asiakastililläsi.}\n",
+      "\n",
+      "Given the triple backGiven the question delimited by triple backticks ```{ Miten voin tarkistaa tilaushistoriani?}```, what is the answer? Answer: {Voit tarkistaa tilaushistoriasi asiakastililtäsi.}\n",
+      "\n",
+      "Given the triple backGiven the question delimited by triple backticks ```{ Miten voin muuttaa tai perua tilaukseni?}```, what is the answer? Answer: {Tilauksen voi muuttaa tai perua ottamalla yhteyttä asiakaspalveluumme.}\n",
+      "\n",
+      "Given the triple backGiven the question delimited by triple backticks ```{ Miten voin perua tilaukseni?}```, what is the answer? Answer: {Tilauksen voi perua ottamalla yhteyttä asiakaspalveluumme.}\n",
+      "\n",
+      "Given the triple backGiven the question delimited by triple backticks ```{ Mitä maksutapoja on käytössä?}```, what is the answer? Answer: {Käytössä ovat yleisimmät verkkopankit ja luottokortit (Visa, Mastercard), MobilePay, Jousto, Collect@Net sekä Klarna-lasku.}\n",
+      "\n",
+      "Given the triple backGiven the question delimited by triple backticks ```{ Miten voin muuttaa laskutusosoitettani?}```, what is the answer? Answer: {Laskutusosoitteen voi muuttaa ottamalla\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokenizer.decode(output[0]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ff4406b0-5cd7-4a91-ad0f-28e71e075db8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "environment": {
+   "kernel": "poro",
+   "name": "common-cu121.m118",
+   "type": "gcloud",
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/base-cu121:m118"
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}