Upload openvino.ipynb

you need some dependencies to run the code.

openvino.ipynb

@@ -0,0 +1,153 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "1128b0b6-b3a1-4553-a284-b1206b3c5a09",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/anish/miniconda3/envs/pytorch_test/lib/python3.11/site-packages/transformers/utils/import_utils.py:519: FutureWarning: `is_torch_tpu_available` is deprecated and will be removed in 4.41.0. Please use the `is_torch_xla_available` instead.\n",
+      "  warnings.warn(\n",
+      "/home/anish/miniconda3/envs/pytorch_test/lib/python3.11/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
+      "  torch.utils._pytree._register_pytree_node(\n"
+     ]
+    }
+   ],
+   "source": [
+    "from optimum.intel import OVModelForCausalLM\n",
+    "from transformers import AutoConfig, AutoTokenizer, BitsAndBytesConfig\n",
+    "import torch\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d0f822f2-2bac-4b7e-bf6c-1117a836eb46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = AutoTokenizer.from_pretrained(\"/home/anish/dockerx/Buildarea/LLMs/Mistral-7B-Instruct-v0.2/int4-sym-g64\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "05cf6ff2-2aba-438d-ac78-73bfc9a50139",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Compiling the model to CPU ...\n"
+     ]
+    }
+   ],
+   "source": [
+    "model = OVModelForCausalLM.from_pretrained(\"/home/anish/dockerx/Buildarea/LLMs/Mistral-7B-Instruct-v0.2/int4-sym-g64\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "c3769b36-2a6e-48f9-9cb7-2a3df894bee3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"do you want phull sapport saar?\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "05e51bb7-5610-447b-b5e9-6882b1515ed3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = \"[INST]Write me hello world in python language[\\INST]\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "d50de6df-d452-4f7d-9bfb-25bb4a4137ad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_inputs = tokenizer([prompt], return_tensors=\"pt\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "59651efc-f6d1-48ff-900c-fff926f36728",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "generated_ids = model.generate(**model_inputs, max_length=1024, pad_token_id=2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "0e1e29bb-0fe0-420f-9e7a-3dd8996676dc",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'<s> do you want phull sapport saar?\\n\\nI\\'m not entirely sure I understand your question. Could you please clarify what you mean by \"phull sapport saar\" and what type of support you\\'re asking for? I\\'ll do my best to help you out.</s>'"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer.batch_decode(generated_ids)[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d6d2616b-ac6e-4996-95ab-4661a84bc75d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}