{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "from langchain_pinecone import PineconeVectorStore\n", "from langchain_google_genai import GoogleGenerativeAIEmbeddings\n", "from langchain_google_genai import ChatGoogleGenerativeAI\n", "from langchain.chains import RetrievalQA\n", "from langchain.document_loaders import PyPDFLoader\n", "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", "from langchain_openai import OpenAIEmbeddings\n", "# from sentence_transformers import SentenceTransformer\n", "from langchain_google_genai import GoogleGenerativeAIEmbeddings\n", "\n", "from langchain_google_genai import ChatGoogleGenerativeAI\n", "from langchain_pinecone import PineconeVectorStore\n", "from langchain.chains.question_answering import load_qa_chain\n", "from langchain import PromptTemplate\n", "from uuid import uuid4\n", "from langchain_core.documents import Document\n", "import getpass\n", "import os\n", "import google.generativeai as genai\n", "from langchain.document_loaders import PyPDFLoader" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Importing the documents needs to be loaded in Vector Database\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "base_dir = os.getcwd()\n", "file_name = \"Reliance.pdf\"\n", "file_path = os.path.join(base_dir,file_name )\n", "\n", "def create_chunks(doc_to_chunk):\n", " text_splitter = RecursiveCharacterTextSplitter(\n", " chunk_size=500,\n", " chunk_overlap=100,\n", " length_function=len\n", " )\n", " return text_splitter.split_documents(doc_to_chunk)\n", "\n", "def load_pdf(path):\n", " loader = PyPDFLoader(path)\n", " return loader.load()\n", "\n", "def load_chunk_file(path):\n", " doc = load_pdf(path)\n", " return create_chunks(doc)\n", "\n", "chunks = load_chunk_file(file_path)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Loading the document into Vector Database" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['0c2ac5d0-8ee3-4b7e-8846-2918154f1eb8',\n", " 'aa5fc10a-7127-4f21-889f-28f4416f476b',\n", " '1642ce19-52a7-47ba-b9f5-571427234765',\n", " 'ab61277d-6c32-4c2b-9f16-462e1d91920b',\n", " 'df66c977-49f0-4f0c-b15d-399bae4cd22f',\n", " '2a7a3525-6032-4c9d-8841-8ae7ebf56850',\n", " 'b43a8564-a85b-4119-b58d-575e45a42165',\n", " '30ab4aba-7d2c-47b4-834c-59bc92f9ed74',\n", " '623f618b-a7f2-4095-b115-8b410ea9eb9e',\n", " 'c2ea7159-4de7-448e-aa69-1e9accff1850',\n", " '5869be42-c13a-4370-bc3b-2007d09cb7bc',\n", " 'e00749b7-b2a5-42f1-a16e-d6ed85849c91',\n", " '3673b32b-14b0-479d-8791-12f22bd90ff8',\n", " '919227ed-2b17-4b9c-a7d8-1de38c14d626',\n", " 'e9af81a2-0753-4788-993d-b59cc2cada2c',\n", " '872a0391-ad12-4eb5-baa5-fe6fe8ee659b',\n", " '744cf651-e62d-4739-9531-a0dc8e4f1f07',\n", " '645c60da-b40a-453a-b18f-4c06d314e221',\n", " 'b4f052a5-475c-43ab-8252-b28793ccd5c4',\n", " 'c40aea8d-d4a2-4af4-9054-97613d23e4a0',\n", " 'ea57becd-f898-47bb-9ca5-9fe15cb20454',\n", " 'd6b72ea5-eb18-47c0-8662-d4d211dfd10b',\n", " '83cea095-1704-46b1-85bd-79888fdf7d86',\n", " '189bdbb1-a31a-4e66-b5e8-59b1b4655398',\n", " 'a34b2aed-81a9-4326-9ccb-1e3fcf3873a9',\n", " '00180119-f5cf-41de-84b7-610b701383bf',\n", " '53f9ecc4-679c-47f8-9627-f2f886ef42de',\n", " 'be45dc55-d9ff-470c-9393-6cf96dc73c32',\n", " '63d75c0b-020e-4f72-9cae-3f2e4e82b228',\n", " 'c59b133d-c867-45be-963b-cb8f5df49802',\n", " 'bdf3cd50-4d60-4395-95a4-cde7a5e32f81',\n", " '29de5ec1-9910-45bd-8e1e-df565254ef73',\n", " '92a57d8c-84d2-4c73-9b40-682ed42e3998',\n", " '9954113f-64d7-4077-9097-1a64f2e6f096',\n", " 'eced5907-1ec7-45c2-9f1f-036c4fb78d6c',\n", " '2405c386-79f3-4337-8c40-db8b84bfedc2',\n", " '144fde6f-710b-4e1b-ac2f-6f3af969375a',\n", " '4929fd1e-f559-479b-974e-250a2bf2ac56',\n", " 'a4671288-21f3-4210-adcc-82daba519c62',\n", " '33219ac4-e102-4fdd-b453-8d911b6a0814',\n", " '4945640d-595c-46b8-87dc-87182501d9bf',\n", " '0f85529e-11e6-4120-b50c-7054b817d3aa',\n", " 'af1e1941-2904-40f5-aabe-ae1f7901bc5a',\n", " '3cca1def-8a18-4267-b04d-23fceab5faed',\n", " '9a53a228-f505-46db-95cd-dba9f40663bf',\n", " '0a5d4944-1a01-40f4-b557-330ed2e8a18a',\n", " 'cd161a04-1ce2-406d-a43b-ea7efea0370c',\n", " '4eddd6ce-b482-4466-9465-cda8c7c67d29',\n", " '584a0ea1-20cd-46cb-b127-970449862997',\n", " '4edbba63-e967-4864-be57-c1e195128021',\n", " 'ed281465-721c-470a-8ae7-21cfcb8af944',\n", " '143f581d-b09b-4afc-8d9c-eadb1c98e7bb',\n", " 'ec300ec2-5e3a-49fb-a109-00ff9386fd51',\n", " 'b233a907-d9b6-4d42-94cc-19013ca2ee22',\n", " '96f478ba-d59c-4cec-b69b-f8153819c64b',\n", " 'dee19024-32c4-437c-814f-65435855bc7d',\n", " '35efa2fe-bfec-41b0-af06-c1421b618dfb',\n", " 'a7f09dd4-0808-431f-9962-7781cec5d768',\n", " '7dd9eb93-154f-49af-97cf-3acb30fe39f1',\n", " 'ffb9ef79-92e3-4c24-b70d-717f9bb486ab',\n", " 'd4d0fc60-eece-4cde-aebd-2d2eacb1a175',\n", " '89a3c27b-332f-4598-b662-88460ebfea84',\n", " '8b853629-e6a8-482c-9d6a-8100c8e11be1',\n", " 'a16cbe9e-78a5-4998-9022-43941f5651ef',\n", " '43198ea2-2cb6-41f9-968e-71aef1aa3092',\n", " '201e741d-a2da-4f1c-bc0e-413cc264e15f',\n", " '122b9da9-d278-4858-859f-a343d989c755',\n", " 'e62610e0-fce8-484a-bb3f-c0acba538a20',\n", " 'e33ce6e0-fa91-4ea2-895a-c5dfc57dda15',\n", " '2feabd6a-0a10-4567-a00d-a175165795b7',\n", " 'e2f71af2-c2f9-4949-825b-e9da02d33f56',\n", " '51959195-4c81-4b4b-a928-91ae25c32cb7',\n", " '40cbbbaa-0241-4ec7-8672-d9b4eec0b5e3',\n", " 'e06501b9-6e70-4b46-a206-2d1bdbf8323a',\n", " '09624885-5fd8-46e8-a984-5415088a2652',\n", " 'ef2e77e7-a139-4d15-9415-c010cc348a58',\n", " 'e0fde1e5-907b-4fd4-b7b4-76996ad804af',\n", " '66658ffb-4311-4de6-8391-c58aa0539449',\n", " '84bd7262-d5df-4b18-ae3f-0d16cfc9a305',\n", " 'b6a1b8ca-d1b8-43ff-b58c-c3bd051cd6d1',\n", " '4800b8ed-2d70-4251-a6b2-f3423c6179a0',\n", " 'aecb53ac-b605-408f-896f-1969586ec320',\n", " '6e67366e-95da-4cae-8efa-59f8a72de036',\n", " '2b15f99a-f5db-4501-ada1-13b5e3608e7a',\n", " 'cccc222d-79b9-48e0-b1ec-204e98eb57e3',\n", " 'f56aeeb8-f883-4bf8-b7ba-5d7fa5c210cb',\n", " '2aaa5c36-d138-48b3-bc79-5462a0b1d9f0',\n", " '9bc822df-5102-4600-becc-344ac58ff8f1',\n", " '485a6eeb-f603-4d68-88e5-17cb88385eee',\n", " '9e3b0eb4-067c-4895-97e6-a4940e30c6e5',\n", " 'd9ec6744-d756-4c70-8b22-a5d0ea1bc7f9',\n", " '8dac6d17-163a-4186-bf2e-13c8a9d2c68b',\n", " '6afeca7a-019f-44ed-99bc-4faaec6cea9d',\n", " 'f22359e9-7fb8-465e-90da-9c31b2bc7044',\n", " '10aefd98-f3cd-4a30-8153-ce4be52baa2e',\n", " '444d3318-4b08-45f7-8706-26493b2ddd52',\n", " '8247b43f-1ceb-45c3-95fa-96e33ce3f0ae',\n", " '1dbd882d-fe93-4e43-b101-a2170b560645',\n", " '661945bd-82da-4221-bd8c-189b28b8dd3c',\n", " '0b406988-2b45-4c48-bfab-b6174b46cdbc',\n", " 'fbd11f62-3d13-409b-9faa-887a305d6024',\n", " '3d618eb3-d955-41eb-b0e8-e653b4955e4b',\n", " 'b8ef62cf-2182-43cd-8035-d49efe89fc53',\n", " '34b81bfe-4a05-40d2-b97f-1e760a4f9470',\n", " '24e1dc11-8d7e-49ce-a68e-29424199f219',\n", " '24ee8198-f1be-462c-a62b-bf50daa46456',\n", " 'c7c11b34-7df2-477f-86ac-33499992c4c3',\n", " 'bae84011-8d52-4eea-ade4-7e55fd9b6834',\n", " '733a2c66-5857-4d0a-96fa-c14682d6ebfe',\n", " '5ee5902e-e091-4295-b101-cf1550a474c0',\n", " '417946bf-4757-499a-b6c9-1ff5c80e137f',\n", " '4f7eeca1-5cab-42f8-a5cc-d42646c05692',\n", " '1fcbfff3-c90a-403f-8253-9646b874fdae',\n", " 'dde8af51-308b-44cc-923f-a0f14c7fc82c',\n", " 'bd1410b3-a00d-40ae-99c1-232838ec2ca3',\n", " '63f4b459-156c-4a26-97a5-f34f098b0b6a',\n", " 'c3b75842-5879-4ff8-904d-045fe7918db6',\n", " 'acdda0ae-50d7-4e6b-afda-7442052af76e',\n", " 'de80e5c5-2dea-463b-85b5-2a43389b103b',\n", " '0a1f41f7-b34c-47ad-9e11-1d01aad61f06',\n", " '592a5b78-2901-4343-96c0-ed1cd08c218a',\n", " '26e3415c-5919-4734-9236-018d5ae772e2']" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Import the necessary modules\n", "from langchain_pinecone import PineconeVectorStore\n", "from langchain_core.documents import Document\n", "from langchain_google_genai import GoogleGenerativeAIEmbeddings\n", "from dotenv import load_dotenv\n", "import os\n", "from pinecone import Pinecone, ServerlessSpec\n", "load_dotenv()\n", "\n", "# Initialize the Pinecone vector store\n", "index_name = \"test-index\"\n", "pc = Pinecone(\n", " api_key=os.environ[\"PINECONE_API_KEY\"]) \n", "if index_name not in pc.list_indexes().names():\n", " pc.create_index(\n", " name=index_name,\n", " dimension=768,\n", " metric='cosine', \n", " spec=ServerlessSpec(\n", " cloud='aws', # Specify your preferred cloud provider\n", " region='us-east-1' # Specify your preferred region\n", " )\n", " )\n", "\n", "embeddings = GoogleGenerativeAIEmbeddings(model=\"models/embedding-001\", google_api_key=os.environ[\"GOOGLE_API_KEY\"])\n", "vectorstore = PineconeVectorStore(index_name=index_name, embedding=embeddings)\n", "uuids = [str(uuid4()) for _ in range(len(chunks))]\n", "vectorstore.add_documents(documents=chunks, ids=uuids)\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create questions for the performance evaluation of RAG" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "ename": "ModuleNotFoundError", "evalue": "No module named 'llama_index.embeddings.gemini'", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", "Cell \u001b[1;32mIn[8], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllama_index\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mllms\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgemini\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Gemini\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllama_index\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01membeddings\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgemini\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m GeminiEmbeddings\n", "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'llama_index.embeddings.gemini'" ] } ], "source": [ "from llama_index.llms.gemini import Gemini\n", "from llama_index.embeddings.gemini import GeminiEmbeddings\n" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Unexpected exception formatting exception. Falling back to standard exception\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Traceback (most recent call last):\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\IPython\\core\\interactiveshell.py\", line 3508, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Temp\\ipykernel_38684\\3905429830.py\", line 3, in \n", " resp = Gemini(Model = \"gemini-1.5-flash\").complete(\"Write a poem about a magic backpack\")\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\llama_index\\llms\\gemini\\base.py\", line 147, in __init__\n", " model_meta = genai.get_model(model)\n", " ^^^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\google\\generativeai\\models.py\", line 55, in get_model\n", " elif name.startswith(\"tunedModels/\"):\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\google\\generativeai\\types\\model_types.py\", line 357, in make_model_name\n", "TypeError: Invalid input type. Expected one of the following types: `str`, `Model`, or `TunedModel`.\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\IPython\\core\\interactiveshell.py\", line 2105, in showtraceback\n", " stb = self.InteractiveTB.structured_traceback(\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\IPython\\core\\ultratb.py\", line 1396, in structured_traceback\n", " return FormattedTB.structured_traceback(\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\IPython\\core\\ultratb.py\", line 1287, in structured_traceback\n", " return VerboseTB.structured_traceback(\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\IPython\\core\\ultratb.py\", line 1140, in structured_traceback\n", " formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\IPython\\core\\ultratb.py\", line 1055, in format_exception_as_a_whole\n", " frames.append(self.format_record(record))\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\IPython\\core\\ultratb.py\", line 955, in format_record\n", " frame_info.lines, Colors, self.has_colors, lvals\n", " ^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\IPython\\core\\ultratb.py\", line 778, in lines\n", " return self._sd.lines\n", " ^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\stack_data\\utils.py\", line 145, in cached_property_wrapper\n", " value = obj.__dict__[self.func.__name__] = self.func(obj)\n", " ^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\stack_data\\core.py\", line 734, in lines\n", " pieces = self.included_pieces\n", " ^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\stack_data\\utils.py\", line 145, in cached_property_wrapper\n", " value = obj.__dict__[self.func.__name__] = self.func(obj)\n", " ^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\stack_data\\core.py\", line 681, in included_pieces\n", " pos = scope_pieces.index(self.executing_piece)\n", " ^^^^^^^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\stack_data\\utils.py\", line 145, in cached_property_wrapper\n", " value = obj.__dict__[self.func.__name__] = self.func(obj)\n", " ^^^^^^^^^^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\stack_data\\core.py\", line 660, in executing_piece\n", " return only(\n", " ^^^^^\n", " File \"C:\\Users\\agshi\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\executing\\executing.py\", line 116, in only\n", " raise NotOneValueFound('Expected one value, found 0')\n", "executing.executing.NotOneValueFound: Expected one value, found 0\n" ] } ], "source": [ "from dotenv import load_dotenv\n", "load_dotenv()\n", "resp = Gemini(Model = \"gemini-1.5-flash\").complete(\"Write a poem about a magic backpack\")\n", "print(resp)" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "GOOGLE_API_KEY = os.environ[\"GOOGLE_API_KEY\"]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "questions = [\n", " \"What is the company's registered office address?\",\n", " \"What is the phone number of the company?\",\n", " \"When was the company established?\",\n", " \"Who is the regulatory authority for the company?\",\n", " \"What is the website of the company?\",\n", " \"What is the CIN (Corporate Identification Number) of the company?\",\n", " \"Who is the CEO of the company?\",\n", " \"What is the company's market capitalization?\",\n", " \"What are the major products or services offered by the company?\",\n", " \"What is the company's financial performance in the last fiscal year?\"\n", "]" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "✅ In Context Relevance, input source will be set to __record__.main_input or `Select.RecordInput` .\n", "✅ In Context Relevance, input statement will be set to __record__.app.query.rets.source_nodes[:].node.text .\n", "✅ In Answer Relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n", "✅ In Answer Relevance, input context will be set to __record__.main_output or `Select.RecordOutput` .\n", "✅ In Context Relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n", "✅ In Context Relevance, input context will be set to __record__.app.query.rets.source_nodes[:].node.text .\n" ] } ], "source": [ "from trulens_eval.feedback.provider.langchain import Langchain\n", "from trulens_eval.feedback.provider.hugs import Huggingface\n", "from trulens_eval import Feedback\n", "from trulens_eval import TruLlama\n", "context_selection = TruLlama.select_source_nodes().node.text\n", "provider = Huggingface()\n", "import numpy as np\n", "huggingface_provider = Huggingface()\n", "# Define a groundedness feedback function\n", "f_groundedness = (\n", " Feedback(huggingface_provider.groundedness_measure_with_nli,\n", " name=\"Context Relevance\")\n", " .on_input()\n", " .on(context_selection)\n", " .aggregate(np.mean)\n", ")\n", "# Question/answer relevance between overall question and answer.\n", "f_answer_relevance = Feedback(\n", " huggingface_provider.context_relevance,\n", " name=\"Answer Relevance\"\n", ").on_input_output()\n", "\n", "# Context relevance between question and each context chunk.\n", "f_context_relevance = (\n", " Feedback(huggingface_provider.context_relevance,\n", " name=\"Context Relevance\")\n", " .on_input()\n", " .on(context_selection)\n", " .aggregate(np.mean)\n", ")" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "from trulens_eval.feedback.provider.hugs import Huggingface\n", "huggingface_provider = Huggingface()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "def generate_answer(query, vector_store, template):\n", " # Use LangChain's RetrievalQA to query the vector store and generate the answer\n", " qa_chain = RetrievalQA.from_chain_type(\n", " llm=ChatGoogleGenerativeAI(model=\"gemini-1.5-flash\"),\n", " chain_type=\"stuff\",\n", " retriever=vector_store.as_retriever(search_type=\"similarity\")\n", " )\n", "\n", " # Format the prompt with the retrieved context and the query\n", " prompt = template.format(context=\"{context}\", question=query)\n", " \n", " # Generate the answer by running the chain with the combined prompt\n", " answer = qa_chain.run(query=query) # Pass 'input' as keyword argument with the formatted prompt\n", " return answer" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "generate_answer(\n", " \"How do you create your AI portfolio?\")" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "ename": "ValidationError", "evalue": "2 validation errors for TruLlama\napp.is-instance[BaseQueryEngine]\n Input should be an instance of BaseQueryEngine [type=is_instance_of, input_value=, input_type=function]\n For further information visit https://errors.pydantic.dev/2.8/v/is_instance_of\napp.is-instance[BaseChatEngine]\n Input should be an instance of BaseChatEngine [type=is_instance_of, input_value=, input_type=function]\n For further information visit https://errors.pydantic.dev/2.8/v/is_instance_of", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mValidationError\u001b[0m Traceback (most recent call last)", "Cell \u001b[1;32mIn[15], line 3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtrulens_eval\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TruLlama\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtrulens_eval\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m FeedbackMode\n\u001b[1;32m----> 3\u001b[0m tru_recorder \u001b[38;5;241m=\u001b[39m \u001b[43mTruLlama\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mgenerate_answer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mapp_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mApp_1\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43mfeedbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\n\u001b[0;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mf_groundedness\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mf_answer_relevance\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43mf_context_relevance\u001b[49m\n\u001b[0;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43m]\u001b[49m\n\u001b[0;32m 11\u001b[0m \u001b[43m)\u001b[49m\n", "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\trulens_eval\\tru_llama.py:314\u001b[0m, in \u001b[0;36mTruLlama.__init__\u001b[1;34m(self, app, **kwargs)\u001b[0m\n\u001b[0;32m 311\u001b[0m kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mroot_class\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m Class\u001b[38;5;241m.\u001b[39mof_object(app) \u001b[38;5;66;03m# TODO: make class property\u001b[39;00m\n\u001b[0;32m 312\u001b[0m kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minstrument\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m LlamaInstrument(app\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n\u001b[1;32m--> 314\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\trulens_eval\\app.py:564\u001b[0m, in \u001b[0;36mApp.__init__\u001b[1;34m(self, tru, feedbacks, **kwargs)\u001b[0m\n\u001b[0;32m 559\u001b[0m kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfeedbacks\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m feedbacks\n\u001b[0;32m 560\u001b[0m kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrecording_contexts\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m contextvars\u001b[38;5;241m.\u001b[39mContextVar(\n\u001b[0;32m 561\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrecording_contexts\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 562\u001b[0m )\n\u001b[1;32m--> 564\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 566\u001b[0m app \u001b[38;5;241m=\u001b[39m kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mapp\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m 567\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapp \u001b[38;5;241m=\u001b[39m app\n", "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\trulens_eval\\schema\\app.py:97\u001b[0m, in \u001b[0;36mAppDefinition.__init__\u001b[1;34m(self, app_id, tags, metadata, feedback_mode, app_extra_json, **kwargs)\u001b[0m\n\u001b[0;32m 94\u001b[0m kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m {}\n\u001b[0;32m 95\u001b[0m kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mapp_extra_json\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m app_extra_json \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mdict\u001b[39m()\n\u001b[1;32m---> 97\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 99\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m app_id \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m 100\u001b[0m app_id \u001b[38;5;241m=\u001b[39m obj_id_of_obj(obj\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_dump(), prefix\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapp\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\trulens_eval\\utils\\pyschema.py:686\u001b[0m, in \u001b[0;36mWithClassInfo.__init__\u001b[1;34m(self, class_info, obj, cls, *args, **kwargs)\u001b[0m\n\u001b[0;32m 682\u001b[0m class_info \u001b[38;5;241m=\u001b[39m Class\u001b[38;5;241m.\u001b[39mof_class(\u001b[38;5;28mcls\u001b[39m, with_bases\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m 684\u001b[0m kwargs[CLASS_INFO] \u001b[38;5;241m=\u001b[39m class_info\n\u001b[1;32m--> 686\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\pydantic\\main.py:193\u001b[0m, in \u001b[0;36mBaseModel.__init__\u001b[1;34m(self, **data)\u001b[0m\n\u001b[0;32m 191\u001b[0m \u001b[38;5;66;03m# `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks\u001b[39;00m\n\u001b[0;32m 192\u001b[0m __tracebackhide__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m--> 193\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__pydantic_validator__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_python\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mself_instance\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n", "\u001b[1;31mValidationError\u001b[0m: 2 validation errors for TruLlama\napp.is-instance[BaseQueryEngine]\n Input should be an instance of BaseQueryEngine [type=is_instance_of, input_value=, input_type=function]\n For further information visit https://errors.pydantic.dev/2.8/v/is_instance_of\napp.is-instance[BaseChatEngine]\n Input should be an instance of BaseChatEngine [type=is_instance_of, input_value=, input_type=function]\n For further information visit https://errors.pydantic.dev/2.8/v/is_instance_of" ] } ], "source": [ "from trulens_eval import TruLlama\n", "from trulens_eval import FeedbackMode\n", "tru_recorder = TruLlama(\n", " generate_answer,\n", " app_id=\"App_1\",\n", " feedbacks=[\n", " f_groundedness,\n", " f_answer_relevance,\n", " f_context_relevance\n", " ]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.9" } }, "nbformat": 4, "nbformat_minor": 2 }