Upload week2assignment_shakes.py

week2assignment_shakes.py

@@ -0,0 +1,277 @@
+# -*- coding: utf-8 -*-
+"""week2assignment_shakes.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1OfNmkHMwkuJONUG4yQHDwJiuzLJvF1kJ
+"""
+
+!pip install openai langchain python-dotenv -q
+
+!echo openai_api_key="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" > .env
+
+import os
+import openai
+from dotenv import load_dotenv
+
+load_dotenv(".env")
+
+openai.api_key = os.environ.get("openai_api_key")
+
+from IPython.display import display, Markdown
+
+def disp_markdown(text: str) -> None:
+  display(Markdown(text))
+
+from langchain.chat_models import ChatOpenAI
+from langchain.schema import HumanMessage
+
+chat_model = ChatOpenAI(model_name="gpt-3.5-turbo", openai_api_key=os.environ.get("openai_api_key"))
+
+from langchain.schema import (
+    AIMessage,
+    HumanMessage,
+    SystemMessage
+)
+
+# The SystemMessage is associated with the system role
+system_message = SystemMessage(content="You are a food critic.")
+
+# The HumanMessage is associated with the user role
+user_message = HumanMessage(content="Do you think Kraft Dinner constitues fine dining?")
+
+# The AIMessage is associated with the assistant role
+assistant_message = AIMessage(content="Egads! No, it most certainly does not!")
+
+second_user_message = HumanMessage(content="What about Red Lobster, surely that is fine dining!")
+
+# create the list of prompts
+list_of_prompts = [
+    system_message,
+    user_message,
+    assistant_message,
+    second_user_message
+]
+
+# we can just call our chat_model on the list of prompts!
+chat_model(list_of_prompts)
+
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate
+)
+
+# we can signify variables we want access to by wrapping them in {}
+system_prompt_template = "You are an expert in {SUBJECT}, and you're currently feeling {MOOD}"
+system_prompt_template = SystemMessagePromptTemplate.from_template(system_prompt_template)
+
+user_prompt_template = "{CONTENT}"
+user_prompt_template = HumanMessagePromptTemplate.from_template(user_prompt_template)
+
+# put them together into a ChatPromptTemplate
+chat_prompt = ChatPromptTemplate.from_messages([system_prompt_template, user_prompt_template])
+
+formatted_chat_prompt = chat_prompt.format_prompt(SUBJECT="cheeses", MOOD="quite tired", CONTENT="Hi, what are the finest cheeses?").to_messages()
+
+disp_markdown(chat_model(formatted_chat_prompt).content)
+
+from langchain.chains import LLMChain
+
+chain = LLMChain(llm=chat_model, prompt=chat_prompt)
+
+disp_markdown(chain.run(SUBJECT="classic cars", MOOD="angry", CONTENT="Is the 67 Chevrolet Impala a good vehicle?"))
+
+!wget https://erki.lap.ee/failid/raamatud/guide1.txt
+
+with open("guide1.txt") as f:
+    hitchhikersguide = f.read()
+
+from langchain.text_splitter import CharacterTextSplitter
+
+text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0, separator = "\n")
+texts = text_splitter.split_text(hitchhikersguide)
+
+from langchain.embeddings.openai import OpenAIEmbeddings
+
+os.environ["OPENAI_API_KEY"] = openai.api_key
+
+embeddings = OpenAIEmbeddings()
+
+# !pip install chromadb==0.3.22 tiktoken -q
+
+# !pip install chromadb -U
+
+!pip install pydantic -q
+import chromadb
+
+from langchain.vectorstores.chroma import Chroma
+docsearch = Chroma.from_texts(texts, embeddings, metadatas=[{"source": str(i)} for i in range(len(texts))]).as_retriever()
+
+query = "What makes towels important?"
+docs = docsearch.get_relevant_documents(query)
+
+docs[0]
+
+from langchain.chains.question_answering import load_qa_chain
+from langchain.llms import OpenAI
+
+chain = load_qa_chain(OpenAI(temperature=0), chain_type="stuff")
+query = "What makes towels important?"
+chain.run(input_documents=docs, question=query)
+
+"""# Assignment 2
+
+"""
+
+!git clone https://github.com/TheMITTech/shakespeare
+
+from glob import glob
+
+files = glob("./shakespeare/**/*.html")
+
+import shutil
+import os
+
+os.mkdir('./data')
+destination_folder = './data/'
+
+for html_file in files:
+  shutil.move(html_file, destination_folder + html_file.split("/")[-1])
+
+!pip install beautifulsoup4 -q
+
+from langchain.document_loaders import BSHTMLLoader, DirectoryLoader
+
+bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
+
+data = bshtml_dir_loader.load()
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size = 1000,
+    chunk_overlap  = 20,
+    length_function = len,
+)
+
+documents = text_splitter.split_documents(data)
+
+persist_directory = "vector_db"
+
+vectordb = Chroma.from_documents(documents=documents, embedding=embeddings, persist_directory=persist_directory)
+
+vectordb.persist()
+vectordb = None
+
+vectordb = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
+
+llm = ChatOpenAI(temperature=0, model="gpt-4")
+
+doc_retriever = vectordb.as_retriever()
+
+from langchain.chains import RetrievalQA
+
+shakespeare_qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=doc_retriever)
+
+shakespeare_qa.run("Who is Hamlet'?")
+
+!pip install google-search-results -q
+
+os.environ["SERPAPI_API_KEY"] = "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"
+
+from langchain.utilities import SerpAPIWrapper
+
+search = SerpAPIWrapper()
+
+from langchain.agents import initialize_agent, Tool
+from langchain.agents import AgentType
+from langchain.tools import BaseTool
+from langchain.llms import OpenAI
+from langchain import LLMMathChain, SerpAPIWrapper
+
+tools = [
+    Tool(
+        name = "Shakespeare QA System",
+        func=shakespeare_qa.run,
+        description="useful for when you need to answer questions about Shakespeare's works. Input should be a fully formed question."
+    ),
+    Tool(
+        name = "SERP API Search",
+        func=search.run,
+        description="useful for when you need to answer questions about ruff (a python linter). Input should be a fully formed question."
+    ),
+]
+
+agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True)
+
+agent.run("What is Hamlet and more importantly who is hamlet?")
+
+from langchain.memory import ConversationBufferMemory, ReadOnlySharedMemory
+
+memory = ConversationBufferMemory(memory_key="chat_history")
+readonlymemory = ReadOnlySharedMemory(memory=memory)
+
+shakespeare_qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=doc_retriever, memory=readonlymemory)
+
+tools = [
+    Tool(
+        name = "Shakespeare QA System",
+        func=shakespeare_qa.run,
+        description="useful for when you need to answer questions about Shakespeare's works. Input should be a fully formed question."
+    ),
+    Tool(
+        name = "SERP API Search",
+        func=search.run,
+        description="useful for when you need to answer questions about ruff (a python linter). Input should be a fully formed question."
+    ),
+]
+
+from langchain.agents import ZeroShotAgent, Tool, AgentExecutor
+
+prefix = """Have a conversation with a human, answering the following questions as best you can. You have access to the following tools:"""
+suffix = """Begin!"
+
+{chat_history}
+Question: {input}
+{agent_scratchpad}"""
+
+prompt = ZeroShotAgent.create_prompt(
+    tools,
+    prefix=prefix,
+    suffix=suffix,
+    input_variables=["input", "chat_history", "agent_scratchpad"]
+)
+
+from langchain import OpenAI, LLMChain, PromptTemplate
+
+llm_chain = LLMChain(llm=llm, prompt=prompt)
+
+agent = ZeroShotAgent(llm_chain=llm_chain, tools=tools, verbose=True)
+agent_chain = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True, memory=memory)
+
+agent_chain.run(input="Who is Hamlet and What is Hamlet?")
+
+agent_chain.run(input="What age was he in the play?")
+
+agent_chain.run(input="Did he live through the play?")
+
+agent_chain.run(input="What age did you think he was if you approximate without directly reading it from the play? You make the inference on his acts and ages of people in his life")
+
+!pip install gradio
+
+import gradio as gr
+
+def the_app(text):
+  return agent_chain.run(input=text)
+
+x=the_app('who is gertrude?')
+
+x
+
+iface = gr.Interface(fn= the_app, inputs= "text", outputs="text",title="Shakespearean")
+
+iface.launch()
+
+7688iu78u87u78i