langchain_pdf_QA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning about document loading, splitting, storage, retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "import sys\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_ = load_dotenv(find_dotenv()) # read local .env file\n",
    "\n",
    "openai.api_key  = os.environ['OPENAI_API_KEY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import PyPDFLoader\n",
    "loader = PyPDFLoader(\"/Users/rwang/Library/CloudStorage/OneDrive-RMI/report_pdfs/rmi_know_your_oil_and_gas.pdf\")\n",
    "pages = loader.load()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size = 1500,\n",
    "    chunk_overlap = 150\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = text_splitter.split_documents(pages)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "embedding = OpenAIEmbeddings()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorstores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import Chroma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "persist_directory = 'docs/chroma/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf ./docs/chroma  # remove old database files if any"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectordb = Chroma.from_documents(\n",
    "    documents=splits,\n",
    "    embedding=embedding,\n",
    "    persist_directory=persist_directory\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "110\n"
     ]
    }
   ],
   "source": [
    "print(vectordb._collection.count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Similarity Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"what did they say about green jobs?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = vectordb.similarity_search(question,k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs[2].metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.retrievers import ContextualCompressionRetriever\n",
    "from langchain.retrievers.document_compressors import LLMChainExtractor\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.retrievers.self_query.base import SelfQueryRetriever\n",
    "from langchain.chains.query_constructor.base import AttributeInfo\n",
    "\n",
    "def pretty_print_docs(docs):\n",
    "    print(f\"\\n{'-' * 100}\\n\".join([f\"Document {i+1}:\\n\\n\" + d.page_content for i, d in enumerate(docs)]))\n",
    "\n",
    "# Wrap our vectorstore\n",
    "llm = OpenAI(temperature=0)\n",
    "compressor = LLMChainExtractor.from_llm(llm)\n",
    "\n",
    "\n",
    "compression_retriever = ContextualCompressionRetriever(\n",
    "    base_compressor=compressor,\n",
    "    base_retriever=vectordb.as_retriever(search_type = \"mmr\")\n",
    ")\n",
    "\n",
    "question = \"what did they say about methane?\"\n",
    "compressed_docs = compression_retriever.get_relevant_documents(question)\n",
    "pretty_print_docs(compressed_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start QA with the document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt-3.5-turbo\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "current_date = datetime.datetime.now().date()\n",
    "if current_date < datetime.date(2023, 9, 2):\n",
    "    llm_name = \"gpt-3.5-turbo-0301\"\n",
    "else:\n",
    "    llm_name = \"gpt-3.5-turbo\"\n",
    "print(llm_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import Chroma\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "persist_directory = 'docs/chroma/'\n",
    "embedding = OpenAIEmbeddings()\n",
    "vectordb = Chroma(persist_directory=persist_directory, embedding_function=embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question = \"Why is methane important for climate mitigation?\"\n",
    "docs = vectordb.similarity_search(question,k=3)\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='will facilitate imposing penalties, quantifying offsets, and encouraging their prevention. The integration of \\nmodeled and satellite data can offer new insights and more powerful tools to determine climate damage \\nwhen oil and gas upsets occur. \\nxxiii One cow burps up 220 pounds of methane per year, which amounts to 12 pounds over 20 days — the duration of this blowout \\nthat emitted 4,800 metric tons of methane. (See: Amy Quinton, “Cows and Climate Change, ” UC Davis, 2019,  \\nhttps://www.ucdavis.edu/food/news/making-cattle-more-sustainable .)', metadata={'page': 44, 'source': '/Users/rwang/Library/CloudStorage/OneDrive-RMI/report_pdfs/rmi_know_your_oil_and_gas.pdf'})"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "llm = ChatOpenAI(model_name=llm_name, temperature=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RetrievalQA chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import RetrievalQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "qa_chain = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = qa_chain({\"query\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Methane is important for climate mitigation because it is a potent '\n",
      " 'greenhouse gas that contributes to global warming. It has a much higher '\n",
      " 'warming potential than carbon dioxide over a shorter time frame. Therefore, '\n",
      " 'reducing methane emissions can have a significant impact on mitigating '\n",
      " 'climate change. The oil and gas sector is a major source of methane '\n",
      " 'emissions, and targeting methane reduction is considered the highest '\n",
      " 'priority for this sector. By preventing methane leakage from production '\n",
      " 'equipment, curtailing flaring, and maintaining flare efficiency, significant '\n",
      " 'reductions in methane emissions can be achieved.')\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "pprint(result[\"result\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"What actions can be taken to manage high-emitting oil and gas resource?\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Some actions that can be taken to manage high-emitting oil and gas resources '\n",
      " 'include:\\n'\n",
      " '\\n'\n",
      " '1. Prohibiting methane venting and routine flaring during light oil '\n",
      " 'production.\\n'\n",
      " '2. Tightly managing wet and dry gas to prevent production emissions.\\n'\n",
      " '3. Installing solar, wind, and other renewable electricity sources in oil '\n",
      " 'and gas operations.\\n'\n",
      " '4. Curbing flaring emissions by improving flare efficiency and preventing '\n",
      " 'operators from turning off pilots that keep flares lit.\\n'\n",
      " '5. Prohibiting the development of high-CO2 gas and continuously monitoring '\n",
      " 'for corrosion in legacy assets for acid gas.\\n'\n",
      " '6. Establishing protocols for decommissioning and tracking energy return on '\n",
      " 'investments for depleted oil and gas.\\n'\n",
      " '7. Performing routine LDAR (Leak Detection and Repair) to ensure no '\n",
      " 'leakage.\\n'\n",
      " '8. Tracking asset ownership transfers.\\n'\n",
      " '\\n'\n",
      " 'These are just a few examples of strategies that can be implemented to '\n",
      " 'reduce the climate footprints of high-emitting oil and gas resources.')\n"
     ]
    }
   ],
   "source": [
    "pprint(qa_chain({\"query\": question})['result'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Yes, according to the information provided, there is no geologic shortage of '\n",
      " 'oil and gas. Current projections suggest that there are many trillions of '\n",
      " 'barrels of oil equivalent stored in untapped oil and gas reservoirs '\n",
      " 'worldwide. Additionally, at current consumption rates, hydrocarbons in place '\n",
      " 'are projected to last for approximately 500 more years.')\n"
     ]
    }
   ],
   "source": [
    "question = \"Are oil and gas abundant?\"\n",
    "pprint(qa_chain({\"query\": question})['result'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts import PromptTemplate\n",
    "\n",
    "# Build prompt\n",
    "template = \"\"\"Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. Use three sentences maximum. Keep the answer as concise as possible. Always say \"thanks for asking!\" at the end of the answer. \n",
    "{context}\n",
    "Question: {question}\n",
    "Helpful Answer:\"\"\"\n",
    "QA_CHAIN_PROMPT = PromptTemplate.from_template(template)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run chain\n",
    "qa_chain = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever(),\n",
    "    return_source_documents=True,\n",
    "    chain_type_kwargs={\"prompt\": QA_CHAIN_PROMPT}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Yes, oil and gas are abundant resources with many trillions of barrels of '\n",
      " 'oil equivalent stored in untapped reservoirs worldwide. Thanks for asking!')\n"
     ]
    }
   ],
   "source": [
    "question = \"Are oil and gas abundant?\"\n",
    "result = qa_chain({\"query\": question})\n",
    "pprint(result['result'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'page': 18,\n",
       " 'source': '/Users/rwang/Library/CloudStorage/OneDrive-RMI/report_pdfs/rmi_know_your_oil_and_gas.pdf'}"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['source_documents'][0].metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RetrievalQA chain types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "qa_chain_mr = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever(),\n",
    "    chain_type=\"map_reduce\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = qa_chain_mr({\"query\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Yes, according to the document, oil and gas are described as abundant resources.'"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[\"result\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Based on the additional context provided, it is important to note that while oil and gas are abundant resources, their production and consumption levels can vary over time due to various factors such as global recessions, pandemics, and oil price hikes. The specific data points provided in the context, such as global oil and gas production and consumption, can help provide a more accurate assessment of their abundance. However, without specific data points or trends, it is difficult to determine the current abundance of oil and gas.'"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_chain_mr = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever(),\n",
    "    chain_type=\"refine\"\n",
    ")\n",
    "result = qa_chain_mr({\"query\": question})\n",
    "result[\"result\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chatbox "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the old retrival chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import Chroma\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "persist_directory = 'docs/chroma/'\n",
    "embedding = OpenAIEmbeddings()\n",
    "vectordb = Chroma(persist_directory=persist_directory, embedding_function=embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question = \"As oil and gas resources age, do their emissions increase or decrease?\"\n",
    "docs = vectordb.similarity_search(question,k=3)\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='rmi.org  / 18\\nKnow Your Oil and GasOil and Gas Emissions Increase as Resources Age\\nOil and gas are inherently heterogeneous, and their composition can change markedly as they age. Over \\ntime, oils can become solid, watery, gassy, and contaminated. Gases can get wetter or acquire impurities. \\nBoth can get trapped in fissures. \\nThe resources modeled show an upward trend between upstream emissions intensity and the asset’s years \\nin production in Exhibit 9. And the general finding that emissions intensity increases the longer oil and gas \\nare produced from an asset is supported by decades-long time-series data. As oil and gas reservoirs are \\ndepleted and production volumes decline, new recovery methods are employed that typically require more \\nenergy and result in higher emissions. The same is true of refining less conventional resources that evolve \\nas they age. Simulation studies show an expected doubling in average emissions over 25 years.12 These \\ntrends call for policies governing the transfer of aging resources and the need to establish decommissioning \\nincentives.\\nExhibit 9  Upstream Emissions Intensities Increase with Field Age\\n0\\n25\\n50\\n75\\n100\\n125\\n0\\n100\\n200\\n300\\n400\\n500\\n600\\nYears in Production\\nAll modeled oil and gas resourcesUpstream Emissions Intensity (kg CO/two.subse/boe)', metadata={'page': 17, 'source': '/Users/rwang/Library/CloudStorage/OneDrive-RMI/report_pdfs/rmi_know_your_oil_and_gas.pdf'})"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello! How can I assist you today?'"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "llm = ChatOpenAI(model_name=llm_name, temperature=0)\n",
    "llm.predict(\"Hello world!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'As oil and gas resources age, their emissions increase. Thanks for asking!'"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Build prompt\n",
    "from langchain.prompts import PromptTemplate\n",
    "template = \"\"\"Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. Use three sentences maximum. Keep the answer as concise as possible. Always say \"thanks for asking!\" at the end of the answer. \n",
    "{context}\n",
    "Question: {question}\n",
    "Helpful Answer:\"\"\"\n",
    "QA_CHAIN_PROMPT = PromptTemplate(input_variables=[\"context\", \"question\"],template=template,)\n",
    "\n",
    "# Run chain\n",
    "from langchain.chains import RetrievalQA\n",
    "question = \"As oil and gas resources age, do their emissions increase or decrease?\"\n",
    "qa_chain = RetrievalQA.from_chain_type(llm,\n",
    "                                       retriever=vectordb.as_retriever(),\n",
    "                                       return_source_documents=True,\n",
    "                                       chain_type_kwargs={\"prompt\": QA_CHAIN_PROMPT})\n",
    "\n",
    "\n",
    "result = qa_chain({\"query\": question})\n",
    "result[\"result\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'page': 17,\n",
       " 'source': '/Users/rwang/Library/CloudStorage/OneDrive-RMI/report_pdfs/rmi_know_your_oil_and_gas.pdf'}"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['source_documents'][0].metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.memory import ConversationBufferMemory\n",
    "memory = ConversationBufferMemory(\n",
    "    memory_key=\"chat_history\",\n",
    "    return_messages=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ConverstionalRetrivalChain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import ConversationalRetrievalChain\n",
    "retriever=vectordb.as_retriever()\n",
    "qa = ConversationalRetrievalChain.from_llm(\n",
    "    llm,\n",
    "    retriever=retriever,\n",
    "    memory=memory\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"What does the report tell us about oil and gas upstream emissions?\"\n",
    "result = qa({\"question\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('The report provides information about oil and gas upstream emissions. It '\n",
      " 'states that the most emissions-intensive oil or gas resource emits more than '\n",
      " '10 times as much as the least intensive resource in the production phase. '\n",
      " 'The average emissions intensity for oil resources is 175 kg CO2e/boe, while '\n",
      " 'for gas resources it is 95 kg CO2e/boe. The emissions drivers in the '\n",
      " 'upstream phase vary depending on factors such as the characteristics of the '\n",
      " 'resource, energy required for extraction and processing, and methane '\n",
      " 'leakage. The major upstream stages that drive emissions include drilling and '\n",
      " 'development, production and extraction, surface processing, small sources, '\n",
      " 'offsite emissions, and crude transport. The report also mentions that '\n",
      " 'greater data transparency can reduce uncertainty in emissions estimates.')\n"
     ]
    }
   ],
   "source": [
    "pprint(result['answer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"how are the emissions calculated\"\n",
    "result = qa({\"question\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('The emissions in the report are calculated using the OCI+ model, which '\n",
      " 'assesses life-cycle emissions from the wellhead through end use. It takes '\n",
      " 'into account the production, refining, processing, and shipping of oil and '\n",
      " 'gas resources. The model uses a bottom-up engineering systems approach, as '\n",
      " 'well as top-down measurements, to compare the emissions intensities of '\n",
      " 'different oil and gas resources. The emissions are measured in kilograms of '\n",
      " 'CO2 equivalent (kg CO2e) and methane leakage rates are also taken into '\n",
      " 'account.')\n"
     ]
    }
   ],
   "source": [
    "pprint(result['answer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"what models compose OCI+ and how are they connected\"\n",
    "result = qa({\"question\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('The OCI+ includes three interconnected models: OPGEE, PRELIM, and OPEM. '\n",
      " 'These models work together to estimate current and project future emissions '\n",
      " 'based on changing operating conditions.\\n'\n",
      " '\\n'\n",
      " '1. OPGEE (Oil Production Greenhouse Gas Emissions Estimator): This model is '\n",
      " 'used to estimate the greenhouse gas emissions associated with oil '\n",
      " 'production. It takes into account various factors such as well '\n",
      " 'characteristics, production techniques, and energy consumption.\\n'\n",
      " '\\n'\n",
      " '2. PRELIM (Petroleum Refining Life-cycle Inventory Model): This model '\n",
      " 'focuses on estimating the greenhouse gas emissions associated with petroleum '\n",
      " 'refining. It considers factors such as feedstock composition, energy '\n",
      " 'sources, and refining processes.\\n'\n",
      " '\\n'\n",
      " '3. OPEM (Oil Production Emissions Model): This model estimates the emissions '\n",
      " 'associated with oil production operations, including upstream activities '\n",
      " 'such as drilling, well completion, and maintenance.\\n'\n",
      " '\\n'\n",
      " 'These models are interconnected in the sense that the outputs of one model '\n",
      " 'serve as inputs for the others. For example, the emissions estimates from '\n",
      " 'OPGEE and PRELIM are used as inputs for OPEM to calculate the overall '\n",
      " 'emissions from oil production operations. This interconnectedness allows for '\n",
      " 'a comprehensive assessment of emissions throughout the oil and gas value '\n",
      " 'chain.')\n"
     ]
    }
   ],
   "source": [
    "pprint(result['answer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"Is that how life cycle emissions are calculated?\"\n",
    "result = qa({\"question\": question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(\"Life cycle emissions are calculated by considering all stages of a product's \"\n",
      " 'life cycle, from extraction or production to end-use and disposal. This '\n",
      " 'includes emissions from activities such as extraction, processing, refining, '\n",
      " 'transportation, and combustion. \\n'\n",
      " '\\n'\n",
      " 'To calculate life cycle emissions, various factors and data inputs are taken '\n",
      " 'into account, such as the carbon content of the fuel, energy consumption '\n",
      " 'during production and transportation, and the emissions associated with each '\n",
      " 'stage of the life cycle. These calculations can be complex and require '\n",
      " 'detailed data on the specific processes and inputs involved in the '\n",
      " 'production and use of the product.\\n'\n",
      " '\\n'\n",
      " 'Advanced models, operational inputs, and satellite data are often used to '\n",
      " 'estimate life cycle emissions. These models consider factors such as '\n",
      " 'resource category, region, operation, pollutant, and more to identify '\n",
      " 'significant reduction potential and variations in emissions intensities.\\n'\n",
      " '\\n'\n",
      " \"It's important to note that life cycle emissions calculations go beyond just \"\n",
      " 'considering direct emissions from fuel combustion. They take into account '\n",
      " 'the entire supply chain and provide a more comprehensive understanding of '\n",
      " 'the emissions associated with a particular product or industry.')\n"
     ]
    }
   ],
   "source": [
    "pprint(result['answer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('There are several improvements that can be made to midstream operations in '\n",
      " 'order to reduce emissions:\\n'\n",
      " '\\n'\n",
      " '1. Optimizing refinery configuration: Updating and reoptimizing old and '\n",
      " 'inefficient processes in refineries can significantly cut emissions. This '\n",
      " 'includes reconfiguring refineries to match the type of crude oil being '\n",
      " 'processed, as different crudes have different emissions intensities.\\n'\n",
      " '\\n'\n",
      " '2. Generating renewable hydrogen: Replacing steam methane reforming (SMR) '\n",
      " 'with renewable hydrogen production can reduce emissions. SMR is an '\n",
      " 'energy-intensive process used to produce hydrogen in refineries.\\n'\n",
      " '\\n'\n",
      " '3. Using renewable electricity in refining: Integrating renewable '\n",
      " 'electricity sources, such as solar and wind, into oil and gas operations can '\n",
      " 'reduce emissions. This includes using renewable electricity for heat '\n",
      " 'generation and other utilities essential to refinery operation.\\n'\n",
      " '\\n'\n",
      " '4. Curtailing flaring and maintaining flares: Implementing policies to '\n",
      " 'prevent operators from turning off pilots that keep flares lit and venting '\n",
      " 'pressurized gas into the atmosphere can reduce emissions. Improving flare '\n",
      " 'efficiency and monitoring can also help reduce emissions from flaring '\n",
      " 'operations.\\n'\n",
      " '\\n'\n",
      " 'These improvements can have significant climate benefits, considering the '\n",
      " 'large amount of oil and gas that is refined and processed daily.')\n"
     ]
    }
   ],
   "source": [
    "question = \"How midstream operations can be improved to reduce emissions?\"\n",
    "result = qa({\"question\": question})\n",
    "pprint(result['answer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('According to the provided context, there are several ways to improve '\n",
      " 'downstream operations and reduce emissions:\\n'\n",
      " '\\n'\n",
      " '1. Sequestering petcoke or finding noncombustive uses for it: Petcoke, a '\n",
      " 'by-product of heavy oils, is often exported to countries with less-stringent '\n",
      " 'environmental regulations and blended with coal to generate power. Banning '\n",
      " 'the sale of petcoke and finding alternative uses for it can help reduce '\n",
      " 'downstream emissions by as much as 24%.\\n'\n",
      " '\\n'\n",
      " '2. Favoring local use of natural gas: Promoting the use of natural gas '\n",
      " 'locally instead of shipping it globally can help reduce emissions. Natural '\n",
      " 'gas combustion emissions dominate gas fields, and by utilizing it locally, '\n",
      " 'transportation emissions can be minimized.\\n'\n",
      " '\\n'\n",
      " '3. Shipping petroleum products over shorter distances: Shipping all '\n",
      " 'petroleum products over shorter distances can help reduce emissions. The '\n",
      " 'transport of petroleum products has minimal emissions intensity per unit '\n",
      " 'processed, so reducing the distance traveled can have a positive impact.\\n'\n",
      " '\\n'\n",
      " '4. Optimizing refinery configurations: Refinery configurations can be '\n",
      " 'changed to alter product slates and reduce emissions. Carefully pairing '\n",
      " 'crudes to the optimal refinery can cut emissions. Processing a light oil '\n",
      " 'through a medium-conversion refinery is estimated to nearly double the '\n",
      " 'midstream climate footprint, while processing a medium oil in a '\n",
      " 'deep-conversion refinery can increase emissions intensity by an estimated '\n",
      " '35%.\\n'\n",
      " '\\n'\n",
      " '5. Curtailing flaring and maintaining flares: Flaring, the burning of excess '\n",
      " 'gas, can be curtailed to reduce emissions. Flares should ideally operate at '\n",
      " '99.99% efficiency, fully combusting all but less than 0.01% of the gas. '\n",
      " 'Policies are needed to prevent operators from turning off pilots that keep '\n",
      " 'flares lit and simply venting pressurized gas into the atmosphere.\\n'\n",
      " '\\n'\n",
      " 'It is important to note that these are just a few examples of how downstream '\n",
      " 'operations can be improved to reduce emissions. There may be other '\n",
      " 'strategies and technologies available as well.')\n"
     ]
    }
   ],
   "source": [
    "question = \"What about downstream?\"\n",
    "result = qa({\"question\": question})\n",
    "pprint(result['answer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a chatbot with UI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter\n",
    "from langchain.vectorstores import DocArrayInMemorySearch\n",
    "from langchain.document_loaders import TextLoader\n",
    "from langchain.chains import RetrievalQA,  ConversationalRetrievalChain\n",
    "from langchain.memory import ConversationBufferMemory\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.document_loaders import TextLoader\n",
    "from langchain.document_loaders import PyPDFLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_db(file, chain_type, k):\n",
    "    # load documents\n",
    "    loader = PyPDFLoader(file)\n",
    "    documents = loader.load()\n",
    "    # split documents\n",
    "    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)\n",
    "    docs = text_splitter.split_documents(documents)\n",
    "    # define embedding\n",
    "    embeddings = OpenAIEmbeddings()\n",
    "    # create vector database from data\n",
    "    db = DocArrayInMemorySearch.from_documents(docs, embeddings)\n",
    "    # define retriever\n",
    "    retriever = db.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": k})\n",
    "    # create a chatbot chain. Memory is managed externally.\n",
    "    qa = ConversationalRetrievalChain.from_llm(\n",
    "        llm=ChatOpenAI(model_name=llm_name, temperature=0), \n",
    "        chain_type=chain_type, \n",
    "        retriever=retriever, \n",
    "        return_source_documents=True,\n",
    "        return_generated_question=True,\n",
    "    )\n",
    "    return qa \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "import panel as pn\n",
    "import param\n",
    "\n",
    "class cbfs(param.Parameterized):\n",
    "    chat_history = param.List([])\n",
    "    answer = param.String(\"\")\n",
    "    db_query  = param.String(\"\")\n",
    "    db_response = param.List([])\n",
    "    \n",
    "    def __init__(self,  **params):\n",
    "        super(cbfs, self).__init__( **params)\n",
    "        self.panels = []\n",
    "        self.loaded_file = \"/Users/rwang/Library/CloudStorage/OneDrive-RMI/report_pdfs/rmi_know_your_oil_and_gas.pdf\"\n",
    "        self.qa = load_db(self.loaded_file,\"stuff\", 4)\n",
    "    \n",
    "    def call_load_db(self, count):\n",
    "        if count == 0 or file_input.value is None:  # init or no file specified :\n",
    "            return pn.pane.Markdown(f\"Loaded File: {self.loaded_file}\")\n",
    "        else:\n",
    "            file_input.save(\"temp.pdf\")  # local copy\n",
    "            self.loaded_file = file_input.filename\n",
    "            button_load.button_style=\"outline\"\n",
    "            self.qa = load_db(\"temp.pdf\", \"stuff\", 4)\n",
    "            button_load.button_style=\"solid\"\n",
    "        self.clr_history()\n",
    "        return pn.pane.Markdown(f\"Loaded File: {self.loaded_file}\")\n",
    "\n",
    "    def convchain(self, query):\n",
    "        if not query:\n",
    "            return pn.WidgetBox(pn.Row('User:', pn.pane.Markdown(\"\", width=600)), scroll=True)\n",
    "        result = self.qa({\"question\": query, \"chat_history\": self.chat_history})\n",
    "        self.chat_history.extend([(query, result[\"answer\"])])\n",
    "        self.db_query = result[\"generated_question\"]\n",
    "        self.db_response = result[\"source_documents\"]\n",
    "        self.answer = result['answer'] \n",
    "        self.panels.extend([\n",
    "            pn.Row('User:', pn.pane.Markdown(query, width=600)),\n",
    "            pn.Row('ChatBot:', pn.pane.Markdown(self.answer, width=600, style={'background-color': '#F6F6F6'}))\n",
    "        ])\n",
    "        inp.value = ''  #clears loading indicator when cleared\n",
    "        return pn.WidgetBox(*self.panels,scroll=True)\n",
    "\n",
    "    @param.depends('db_query ', )\n",
    "    def get_lquest(self):\n",
    "        if not self.db_query :\n",
    "            return pn.Column(\n",
    "                pn.Row(pn.pane.Markdown(f\"Last question to DB:\", styles={'background-color': '#F6F6F6'})),\n",
    "                pn.Row(pn.pane.Str(\"no DB accesses so far\"))\n",
    "            )\n",
    "        return pn.Column(\n",
    "            pn.Row(pn.pane.Markdown(f\"DB query:\", styles={'background-color': '#F6F6F6'})),\n",
    "            pn.pane.Str(self.db_query )\n",
    "        )\n",
    "\n",
    "    @param.depends('db_response', )\n",
    "    def get_sources(self):\n",
    "        if not self.db_response:\n",
    "            return \n",
    "        rlist=[pn.Row(pn.pane.Markdown(f\"Result of DB lookup:\", styles={'background-color': '#F6F6F6'}))]\n",
    "        for doc in self.db_response:\n",
    "            rlist.append(pn.Row(pn.pane.Str(doc)))\n",
    "        return pn.WidgetBox(*rlist, width=600, scroll=True)\n",
    "\n",
    "    @param.depends('convchain', 'clr_history') \n",
    "    def get_chats(self):\n",
    "        if not self.chat_history:\n",
    "            return pn.WidgetBox(pn.Row(pn.pane.Str(\"No History Yet\")), width=600, scroll=True)\n",
    "        rlist=[pn.Row(pn.pane.Markdown(f\"Current Chat History variable\", styles={'background-color': '#F6F6F6'}))]\n",
    "        for exchange in self.chat_history:\n",
    "            rlist.append(pn.Row(pn.pane.Str(exchange)))\n",
    "        return pn.WidgetBox(*rlist, width=600, scroll=True)\n",
    "\n",
    "    def clr_history(self,count=0):\n",
    "        self.chat_history = []\n",
    "        return \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: langchain[docarray] in /Users/rwang/Library/Python/3.9/lib/python/site-packages (0.0.242)\n",
      "Requirement already satisfied: PyYAML>=5.4.1 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (2.0.20)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (3.8.5)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (4.0.2)\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (0.5.14)\n",
      "Requirement already satisfied: langsmith<0.1.0,>=0.0.11 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (0.0.26)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (2.8.5)\n",
      "Requirement already satisfied: numpy<2,>=1 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (1.25.1)\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (1.2.4)\n",
      "Requirement already satisfied: pydantic<2,>=1 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (1.10.12)\n",
      "Requirement already satisfied: requests<3,>=2 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from langchain[docarray]) (8.2.3)\n",
      "Collecting docarray[hnswlib]<0.33.0,>=0.32.0 (from langchain[docarray])\n",
      "  Obtaining dependency information for docarray[hnswlib]<0.33.0,>=0.32.0 from https://files.pythonhosted.org/packages/34/80/c6f9330b386ff76db35148cbd09fd882401b5d0468090b2bd8fb184254a4/docarray-0.32.1-py3-none-any.whl.metadata\n",
      "  Downloading docarray-0.32.1-py3-none-any.whl.metadata (30 kB)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain[docarray]) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain[docarray]) (3.2.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain[docarray]) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain[docarray]) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain[docarray]) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain[docarray]) (1.3.1)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain[docarray]) (3.20.1)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain[docarray]) (0.9.0)\n",
      "Collecting orjson>=3.8.2 (from docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray])\n",
      "  Obtaining dependency information for orjson>=3.8.2 from https://files.pythonhosted.org/packages/8c/a5/c0c1ecab00c2c4bec414ab4d4be7c20203181b0ae8ba24692ebfae4fc405/orjson-3.9.7-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl.metadata\n",
      "  Downloading orjson-3.9.7-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl.metadata (49 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.2/49.2 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: rich>=13.1.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray]) (13.5.2)\n",
      "Collecting types-requests>=2.28.11.6 (from docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray])\n",
      "  Obtaining dependency information for types-requests>=2.28.11.6 from https://files.pythonhosted.org/packages/06/9b/04bb62f11a6824df5d4568439cf0715118c265d0ffbebeb7cf4b8c9caa15/types_requests-2.31.0.2-py3-none-any.whl.metadata\n",
      "  Downloading types_requests-2.31.0.2-py3-none-any.whl.metadata (1.4 kB)\n",
      "Collecting hnswlib>=0.6.2 (from docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray])\n",
      "  Downloading hnswlib-0.7.0.tar.gz (33 kB)\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: protobuf>=3.19.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray]) (4.24.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from pydantic<2,>=1->langchain[docarray]) (4.7.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from requests<3,>=2->langchain[docarray]) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from requests<3,>=2->langchain[docarray]) (1.26.16)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from requests<3,>=2->langchain[docarray]) (2023.7.22)\n",
      "Requirement already satisfied: packaging>=17.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json<0.6.0,>=0.5.7->langchain[docarray]) (23.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from rich>=13.1.0->docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray]) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from rich>=13.1.0->docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray]) (2.14.0)\n",
      "Collecting types-urllib3 (from types-requests>=2.28.11.6->docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray])\n",
      "  Obtaining dependency information for types-urllib3 from https://files.pythonhosted.org/packages/11/7b/3fc711b2efea5e85a7a0bbfe269ea944aa767bbba5ec52f9ee45d362ccf3/types_urllib3-1.26.25.14-py3-none-any.whl.metadata\n",
      "  Downloading types_urllib3-1.26.25.14-py3-none-any.whl.metadata (1.7 kB)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain[docarray]) (1.0.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /Users/rwang/Library/Python/3.9/lib/python/site-packages (from markdown-it-py>=2.2.0->rich>=13.1.0->docarray[hnswlib]<0.33.0,>=0.32.0->langchain[docarray]) (0.1.2)\n",
      "Downloading orjson-3.9.7-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl (241 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m241.8/241.8 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading types_requests-2.31.0.2-py3-none-any.whl (14 kB)\n",
      "Downloading docarray-0.32.1-py3-none-any.whl (215 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m215.3/215.3 kB\u001b[0m \u001b[31m20.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading types_urllib3-1.26.25.14-py3-none-any.whl (15 kB)\n",
      "Building wheels for collected packages: hnswlib\n",
      "  Building wheel for hnswlib (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for hnswlib: filename=hnswlib-0.7.0-cp39-cp39-macosx_10_9_universal2.whl size=387012 sha256=a9286cdff716985c58d0f153a7a7d115c0290bc21f294c50aa9f30da22ae9a78\n",
      "  Stored in directory: /Users/rwang/Library/Caches/pip/wheels/ba/26/61/fface6c407f56418b3140cd7645917f20ba6b27d4e32b2bd20\n",
      "Successfully built hnswlib\n",
      "Installing collected packages: types-urllib3, types-requests, orjson, hnswlib, docarray\n",
      "Successfully installed docarray-0.32.1 hnswlib-0.7.0 orjson-3.9.7 types-requests-2.31.0.2 types-urllib3-1.26.25.14\n"
     ]
    }
   ],
   "source": [
    "! pip3 install \"langchain[docarray]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "cb = cbfs()\n",
    "\n",
    "file_input = pn.widgets.FileInput(accept='.pdf')\n",
    "button_load = pn.widgets.Button(name=\"Load DB\", button_type='primary')\n",
    "button_clearhistory = pn.widgets.Button(name=\"Clear History\", button_type='warning')\n",
    "button_clearhistory.on_click(cb.clr_history)\n",
    "inp = pn.widgets.TextInput( placeholder='Enter text here…')\n",
    "\n",
    "bound_button_load = pn.bind(cb.call_load_db, button_load.param.clicks)\n",
    "conversation = pn.bind(cb.convchain, inp) \n",
    "\n",
    "jpg_pane = pn.pane.Image( 'thul-A25BD42E-3285-437E-84E101FB0E946E65.png')\n",
    "\n",
    "tab1 = pn.Column(\n",
    "    pn.Row(inp),\n",
    "    pn.layout.Divider(),\n",
    "    pn.panel(conversation,  loading_indicator=True, height=300),\n",
    "    pn.layout.Divider(),\n",
    ")\n",
    "tab2= pn.Column(\n",
    "    pn.panel(cb.get_lquest),\n",
    "    pn.layout.Divider(),\n",
    "    pn.panel(cb.get_sources ),\n",
    ")\n",
    "tab3= pn.Column(\n",
    "    pn.panel(cb.get_chats),\n",
    "    pn.layout.Divider(),\n",
    ")\n",
    "tab4=pn.Column(\n",
    "    pn.Row( file_input, button_load, bound_button_load),\n",
    "    pn.Row( button_clearhistory, pn.pane.Markdown(\"Clears chat history. Can use to start a new topic\" )),\n",
    "    pn.layout.Divider(),\n",
    "    pn.Row(jpg_pane.clone(width=400))\n",
    ")\n",
    "dashboard = pn.Column(\n",
    "    pn.Row(pn.pane.Markdown('# ChatWithYourData_Bot')),\n",
    "    pn.Tabs(('Conversation', tab1), ('Database', tab2), ('Chat History', tab3),('Configure', tab4))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}