{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1128b0b6-b3a1-4553-a284-b1206b3c5a09",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anish/miniconda3/envs/pytorch_test/lib/python3.11/site-packages/transformers/utils/import_utils.py:519: FutureWarning: `is_torch_tpu_available` is deprecated and will be removed in 4.41.0. Please use the `is_torch_xla_available` instead.\n",
      "  warnings.warn(\n",
      "/home/anish/miniconda3/envs/pytorch_test/lib/python3.11/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  torch.utils._pytree._register_pytree_node(\n"
     ]
    }
   ],
   "source": [
    "from optimum.intel import OVModelForCausalLM\n",
    "from transformers import AutoConfig, AutoTokenizer, BitsAndBytesConfig\n",
    "import torch\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d0f822f2-2bac-4b7e-bf6c-1117a836eb46",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"/home/anish/dockerx/Buildarea/LLMs/Mistral-7B-Instruct-v0.2/int4-sym-g64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "05cf6ff2-2aba-438d-ac78-73bfc9a50139",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Compiling the model to CPU ...\n"
     ]
    }
   ],
   "source": [
    "model = OVModelForCausalLM.from_pretrained(\"/home/anish/dockerx/Buildarea/LLMs/Mistral-7B-Instruct-v0.2/int4-sym-g64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c3769b36-2a6e-48f9-9cb7-2a3df894bee3",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"do you want phull sapport saar?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "05e51bb7-5610-447b-b5e9-6882b1515ed3",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"[INST]Write me hello world in python language[\\INST]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d50de6df-d452-4f7d-9bfb-25bb4a4137ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_inputs = tokenizer([prompt], return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "59651efc-f6d1-48ff-900c-fff926f36728",
   "metadata": {},
   "outputs": [],
   "source": [
    "generated_ids = model.generate(**model_inputs, max_length=1024, pad_token_id=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0e1e29bb-0fe0-420f-9e7a-3dd8996676dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<s> do you want phull sapport saar?\\n\\nI\\'m not entirely sure I understand your question. Could you please clarify what you mean by \"phull sapport saar\" and what type of support you\\'re asking for? I\\'ll do my best to help you out.</s>'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.batch_decode(generated_ids)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6d2616b-ac6e-4996-95ab-4661a84bc75d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}