RAG_PDF.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Install necessary libraries"
      ],
      "metadata": {
        "id": "7fHl_Wk7OMVx"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MOlxttXFOJp9",
        "outputId": "8a694a1e-f3d9-4a36-dcd3-15f25c645db6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/806.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.4/806.7 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m806.7/806.7 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m225.1/225.1 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m23.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m284.0/284.0 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.6/17.6 MB\u001b[0m \u001b[31m48.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m59.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m237.0/237.0 kB\u001b[0m \u001b[31m21.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.4/54.4 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.9.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mCollecting langchain-openai\n",
            "  Downloading langchain_openai-0.0.5-py3-none-any.whl (29 kB)\n",
            "Requirement already satisfied: langchain-core<0.2,>=0.1.16 in /usr/local/lib/python3.10/dist-packages (from langchain-openai) (0.1.18)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain-openai) (1.23.5)\n",
            "Requirement already satisfied: openai<2.0.0,>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from langchain-openai) (1.10.0)\n",
            "Requirement already satisfied: tiktoken<0.6.0,>=0.5.2 in /usr/local/lib/python3.10/dist-packages (from langchain-openai) (0.5.2)\n",
            "Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.16->langchain-openai) (6.0.1)\n",
            "Requirement already satisfied: anyio<5,>=3 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.16->langchain-openai) (3.7.1)\n",
            "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.16->langchain-openai) (1.33)\n",
            "Requirement already satisfied: langsmith<0.1,>=0.0.83 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.16->langchain-openai) (0.0.86)\n",
            "Requirement already satisfied: packaging<24.0,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.16->langchain-openai) (23.2)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.16->langchain-openai) (1.10.14)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.16->langchain-openai) (2.31.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.16->langchain-openai) (8.2.3)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (1.7.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (0.26.0)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (1.3.0)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (4.66.1)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.10.0->langchain-openai) (4.9.0)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.6.0,>=0.5.2->langchain-openai) (2023.12.25)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.16->langchain-openai) (3.6)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.16->langchain-openai) (1.2.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.10.0->langchain-openai) (2023.11.17)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.10.0->langchain-openai) (1.0.2)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.10.0->langchain-openai) (0.14.0)\n",
            "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.2,>=0.1.16->langchain-openai) (2.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain-core<0.2,>=0.1.16->langchain-openai) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain-core<0.2,>=0.1.16->langchain-openai) (2.0.7)\n",
            "Installing collected packages: langchain-openai\n",
            "Successfully installed langchain-openai-0.0.5\n"
          ]
        }
      ],
      "source": [
        "!pip -q install langchain openai tiktoken pypdf faiss-cpu langchainhub\n",
        "!pip install -U langchain-openai"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Declare OPENAI API KEY"
      ],
      "metadata": {
        "id": "oFzNP6H7O5Bw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"\""
      ],
      "metadata": {
        "id": "c6IjHy_OOgZ9"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_community.document_loaders import PyPDFLoader\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "from langchain_openai import ChatOpenAI,OpenAIEmbeddings\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "from langchain.vectorstores import FAISS\n",
        "from langchain import hub\n",
        "from langchain_core.output_parsers import StrOutputParser\n",
        "from langchain_core.runnables import RunnablePassthrough\n",
        "from langchain_core.runnables import RunnableParallel\n",
        "from langchain_core.runnables import RunnableLambda"
      ],
      "metadata": {
        "id": "psjBPM0KO3N2"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "PATH = \"/content/1.pdf\" #your PDF path\n",
        "\n",
        "loader = PyPDFLoader(PATH)\n",
        "pages = loader.load_and_split()\n",
        "\n",
        "#pages will contain metadata like page and source pdf as well"
      ],
      "metadata": {
        "id": "gsjVIfZwRj5Y"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now split the full text into chunks"
      ],
      "metadata": {
        "id": "npUzwWRGULd5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Splitting up the text into smaller chunks for indexing\n",
        "text_splitter = CharacterTextSplitter(\n",
        "    separator = \"\\n\",\n",
        "    chunk_size = 1000,\n",
        "    chunk_overlap  = 200, #striding over the text\n",
        "    length_function = len,\n",
        ")\n"
      ],
      "metadata": {
        "id": "kJt1qCEApk4i"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "docs_new = text_splitter.split_documents(pages)\n"
      ],
      "metadata": {
        "id": "fORTC1But9gi"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(docs_new)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4V0LPTpbL3NL",
        "outputId": "ee5760b2-6c77-4e7a-a241-b74cd26383e3"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "49"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "docs_new[17].page_content"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 103
        },
        "id": "k3J6MXGs3erS",
        "outputId": "5da81382-1b6a-49fd-b354-3791ebfa7f45"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 8\\n5Functiondescription\\n5.1Fixedtransmission\\n5.2Broadcastingtransmission\\n5.3Broadcastingaddress\\n\\uf06cForexample:SettheaddressofmoduleAas0xFFFF,andthechannelas0x04;\\n\\uf06cWhenmoduleisthetransmitter(transparenttransmission),allmodulesunderchannel0x04willreceivethedata,the'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "docs_new[17].metadata"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xhV9JZH4uMWn",
        "outputId": "1dbd9cf8-83fd-4ce5-e28b-d57411f3c83d"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'source': '/content/1.pdf', 'page': 8}"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now Embeddings of the data"
      ],
      "metadata": {
        "id": "LIuH9IpP5VPh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "embeddings = OpenAIEmbeddings()"
      ],
      "metadata": {
        "id": "1DnejlSM4Lyl"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "db = FAISS.from_documents(docs_new, embeddings)"
      ],
      "metadata": {
        "id": "i32IkH2v5iY7"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Retriever"
      ],
      "metadata": {
        "id": "xunOSMcNXA9d"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "retriever = db.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": 6})"
      ],
      "metadata": {
        "id": "Hd7u7xaY4xmN"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "retriever"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4lUvHEfvRnEJ",
        "outputId": "b3345802-9e07-4bfc-eeae-991978538892"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "VectorStoreRetriever(tags=['FAISS', 'OpenAIEmbeddings'], vectorstore=<langchain_community.vectorstores.faiss.FAISS object at 0x7e3269b2b580>, search_kwargs={'k': 6})"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query = \"What is E32?\"\n",
        "retrieved_docs = retriever.invoke(query)\n",
        "print((retrieved_docs))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wLsyHVapQx9V",
        "outputId": "fdabe17d-a82e-49d1-c4e2-b8fe51ffd4fd"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Document(page_content='E32-915T30DSX1276915M 30 80.3~19.2kDIP24*43SMA-K\\nE32-170T30DSX1278170M 30 80.3k~9.6kDIP24*43SMA-K\\nE32-868T20DSX1276868M 20 30.3~19.2kDIP21*36SMA-K\\nE32-915T20DSX1276915M 20 30.3~19.2kDIP21*36SMA-K\\nE32-433T20DCSX1278433M 20 30.3k~19.2kDIP21*36SMA-K\\nE32-433T30DSX1278433M 30 80.3k~19.2kDIP24*43SMA-K', metadata={'source': '/content/1.pdf', 'page': 18}), Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 3\\nDisclaimer\\nEBYTEreservesallrightstothisdocumentandtheinformationcontainedherein.\\nProducts,names,logosanddesignsdescribedhereinmayinwholeorinpartbe\\nsubjecttointellectualpropertyrights.Reproduction,use,modificationordisclosureto\\nthirdpartiesofthisdocumentoranypartthereofwithouttheexpresspermissionof\\nEBYTEisstrictlyprohibited.\\nTheinformationcontainedhereinisprovided“asis”andEBYTEassumesno\\nliabilityfortheuseoftheinformation.Nowarranty,eitherexpressorimplied,isgiven,\\nincludingbutnotlimited,withrespecttotheaccuracy,correctness,reliabilityand\\nfitnessforaparticularpurposeoftheinformation.Thisdocumentmayberevisedby\\nEBYTEatanytime.Formostrecentdocuments,visitwww.ebyte.com.', metadata={'source': '/content/1.pdf', 'page': 3}), Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 2\\nREVISIONHISTORY.......................................................................................................................21\\nABOUTUS..........................................................................................................................................21', metadata={'source': '/content/1.pdf', 'page': 2}), Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 21\\nRevisionhistory\\nVersion Date Description Issuedby\\n1.00 2017-11-10 Initialversion huaa\\n1.10 2018-01-11 UpdatingE32(868T30S)/E32(915T30S) huaa\\n1.20 2018-01-15 UpdatingE32(868T20S)/E32(915T20S)/E32(400T20S) huaa\\n1.30 2018-01-22UpdatingE32(868T20D)/E32(868T30D)\\nE32(915T20D)/E32(915T30D)/E32(170T30D)huaa\\n1.40 2018-05-24 Updatingantennaoption huaa\\n1.50 2018-10-11 Manualdividing huaa\\n1.60 2020-04-13 Ren\\n1.8 2020-10-23 Parametercorrection Linson\\nAboutus\\nTechnicalsupport:support@cdebyte.com\\nDocumentsandRFSettingdownloadlink:www.ebyte.com\\nThankyouforusingEbyteproducts!Pleasecontactuswithanyquestionsorsuggestions:info@cdebyte.com\\n------------------------------------------------------------------------------------------------------------\\nFax:028-64146160ext.821\\nWeb:www.ebyte.com\\nAddress:BuildingB5,MouldIndustrialPark,199#XiquAvenue,Chengdu,Sichuan', metadata={'source': '/content/1.pdf', 'page': 21}), Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 1\\nCONTENT\\n1.OVERVIEW...............................................................................................................................3\\n1.1INTRODUCTION..............................................................................................................................................................4\\n1.2FEATURES......................................................................................................................................................................4\\n2.SPECIFICATIONANDPARAMETER.........................................................................................5\\n2.1LIMITPARAMETER.........................................................................................................................................................5', metadata={'source': '/content/1.pdf', 'page': 1}), Document(page_content='\\uf06cTheextensionlineandfeederqualityarepoorortoolong,sothebiterrorrateishigh;\\n10Productionguidance\\nThistypeisDIPmodule,whenthewelderweldsthemodule,hemustbeweldingaccordingtotheanti-static\\nregulation.Thisproductisallergictostatic,randomlyweldingthemodulewillhavethechanceofdamagingit\\npermanently.\\n11E32series\\nModelNo.CoreICFrequency\\nHzTxpower\\ndBmDistance\\nkmDataRatePackageSize\\nmmInterface\\nE32-868T20SSX1276868M 20 30.3k~19.2kSMD16*26UART\\nE32-915T20SSX1276915M 20 30.3k~19.2kSMD16*26UART\\nE32-400T20SSX1278433M470M 20 30.3k~19.2kSMD16*26UART\\nE32-915T30SSX1276915M 30 80.3k~19.2kSMD25*40.3UART\\nE32-868T30SSX1276868M 30 80.3k~19.2kSMD25*40.3UART\\nE32-433T30SSX1278433M 30 80.3k~19.2kSMD25*40.3UART\\nE32-433T20S2TSX1278433M 20 30.3k~19.2kSMD17*30UART\\nE32-868T30DSX1276868M 30 80.3~19.2kDIP24*43SMA-K\\nE32-915T30DSX1276915M 30 80.3~19.2kDIP24*43SMA-K\\nE32-170T30DSX1278170M 30 80.3k~9.6kDIP24*43SMA-K\\nE32-868T20DSX1276868M 20 30.3~19.2kDIP21*36SMA-K\\nE32-915T20DSX1276915M 20 30.3~19.2kDIP21*36SMA-K', metadata={'source': '/content/1.pdf', 'page': 18})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Select prompt"
      ],
      "metadata": {
        "id": "g4WbzcBR5Yky"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = hub.pull(\"rlm/rag-prompt\")"
      ],
      "metadata": {
        "id": "KDtQ0uphN-OK"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Define LLM"
      ],
      "metadata": {
        "id": "ZVsKOLZqXV42"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "llm = ChatOpenAI(model_name = \"gpt-3.5-turbo\", temperature = 0)"
      ],
      "metadata": {
        "id": "512vYfxFOeBw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def format_docs(docs):\n",
        "    return \"\\n\\n\".join(doc.page_content for doc in docs)"
      ],
      "metadata": {
        "id": "78FEbIyPXI7h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rag_chain_from_docs = (\n",
        "    RunnablePassthrough.assign(context = (lambda x: format_docs(x[\"context\"])))\n",
        "    | prompt\n",
        "    | llm\n",
        "    | StrOutputParser()\n",
        ")\n",
        "\n",
        "rag_chain_from_source = RunnableParallel(\n",
        "    {\"context\": retriever, \"question\": RunnablePassthrough()}\n",
        ").assign(answer = rag_chain_from_docs)"
      ],
      "metadata": {
        "id": "dGi-lQnY2I4Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rag_chain_from_source.invoke(\"what is E32?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pFz1i-ZoNQ8H",
        "outputId": "dd0055cd-6fc0-40a4-e761-941cfc27168e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'context': [Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 3\\nDisclaimer\\nEBYTEreservesallrightstothisdocumentandtheinformationcontainedherein.\\nProducts,names,logosanddesignsdescribedhereinmayinwholeorinpartbe\\nsubjecttointellectualpropertyrights.Reproduction,use,modificationordisclosureto\\nthirdpartiesofthisdocumentoranypartthereofwithouttheexpresspermissionof\\nEBYTEisstrictlyprohibited.\\nTheinformationcontainedhereinisprovided“asis”andEBYTEassumesno\\nliabilityfortheuseoftheinformation.Nowarranty,eitherexpressorimplied,isgiven,\\nincludingbutnotlimited,withrespecttotheaccuracy,correctness,reliabilityand\\nfitnessforaparticularpurposeoftheinformation.Thisdocumentmayberevisedby\\nEBYTEatanytime.Formostrecentdocuments,visitwww.ebyte.com.', metadata={'source': '/content/1.pdf', 'page': 3}),\n",
              "  Document(page_content='E32-915T30DSX1276915M 30 80.3~19.2kDIP24*43SMA-K\\nE32-170T30DSX1278170M 30 80.3k~9.6kDIP24*43SMA-K\\nE32-868T20DSX1276868M 20 30.3~19.2kDIP21*36SMA-K\\nE32-915T20DSX1276915M 20 30.3~19.2kDIP21*36SMA-K\\nE32-433T20DCSX1278433M 20 30.3k~19.2kDIP21*36SMA-K\\nE32-433T30DSX1278433M 30 80.3k~19.2kDIP24*43SMA-K', metadata={'source': '/content/1.pdf', 'page': 18}),\n",
              "  Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 2\\nREVISIONHISTORY.......................................................................................................................21\\nABOUTUS..........................................................................................................................................21', metadata={'source': '/content/1.pdf', 'page': 2}),\n",
              "  Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 21\\nRevisionhistory\\nVersion Date Description Issuedby\\n1.00 2017-11-10 Initialversion huaa\\n1.10 2018-01-11 UpdatingE32(868T30S)/E32(915T30S) huaa\\n1.20 2018-01-15 UpdatingE32(868T20S)/E32(915T20S)/E32(400T20S) huaa\\n1.30 2018-01-22UpdatingE32(868T20D)/E32(868T30D)\\nE32(915T20D)/E32(915T30D)/E32(170T30D)huaa\\n1.40 2018-05-24 Updatingantennaoption huaa\\n1.50 2018-10-11 Manualdividing huaa\\n1.60 2020-04-13 Ren\\n1.8 2020-10-23 Parametercorrection Linson\\nAboutus\\nTechnicalsupport:support@cdebyte.com\\nDocumentsandRFSettingdownloadlink:www.ebyte.com\\nThankyouforusingEbyteproducts!Pleasecontactuswithanyquestionsorsuggestions:info@cdebyte.com\\n------------------------------------------------------------------------------------------------------------\\nFax:028-64146160ext.821\\nWeb:www.ebyte.com\\nAddress:BuildingB5,MouldIndustrialPark,199#XiquAvenue,Chengdu,Sichuan', metadata={'source': '/content/1.pdf', 'page': 21}),\n",
              "  Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 1\\nCONTENT\\n1.OVERVIEW...............................................................................................................................3\\n1.1INTRODUCTION..............................................................................................................................................................4\\n1.2FEATURES......................................................................................................................................................................4\\n2.SPECIFICATIONANDPARAMETER.........................................................................................5\\n2.1LIMITPARAMETER.........................................................................................................................................................5', metadata={'source': '/content/1.pdf', 'page': 1}),\n",
              "  Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 20\\n13Packageforbatchorder', metadata={'source': '/content/1.pdf', 'page': 20})],\n",
              " 'question': 'what is E32?',\n",
              " 'answer': 'E32 refers to a series of products manufactured by Chengdu Ebyte Electronic Technology Co., Ltd. The specific model mentioned in the context is E32-433T30D.'}"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Extras\n"
      ],
      "metadata": {
        "id": "uk_Zdx1MKzz-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "To understand RunnableParallel and RunnablePassthrough in LCEL"
      ],
      "metadata": {
        "id": "uZF0hbthKz14"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_core.runnables import RunnableParallel, RunnablePassthrough\n",
        "\n",
        "def add(a):\n",
        "  return a+1\n",
        "\n",
        "chain = RunnablePassthrough.assign(pass_3 = lambda x: x[\"check\"])\n",
        "\n",
        "run = RunnableParallel(\n",
        "    {\"check\" : RunnablePassthrough()},\n",
        "    passed = RunnablePassthrough(),\n",
        "\n",
        "    pass_2 = RunnablePassthrough.assign(mul = lambda x: x[\"num\"]*3),\n",
        "    pass_3 = RunnablePassthrough.assign(mul_1 = lambda x: add(x[\"num\"]))\n",
        "\n",
        ").assign(ans = chain)"
      ],
      "metadata": {
        "id": "cLnhPA6-K0H7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "run.invoke({\"num\" : 3})"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IGqvrrJHL2pc",
        "outputId": "efaf2db0-30f0-47a1-be1d-f226edecd400"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'check': {'num': 3},\n",
              " 'passed': {'num': 3},\n",
              " 'pass_2': {'num': 3, 'mul': 9},\n",
              " 'pass_3': {'num': 3, 'mul_1': 4},\n",
              " 'ans': {'check': {'num': 3},\n",
              "  'passed': {'num': 3},\n",
              "  'pass_2': {'num': 3, 'mul': 9},\n",
              "  'pass_3': {'num': 3}}}"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rag_chain_from_docs = (\n",
        "    RunnablePassthrough.assign(context_1 = (lambda x: format_docs(x[\"context\"])))\n",
        "    #| prompt\n",
        "    #| llm\n",
        "    #| StrOutputParser()\n",
        ")\n",
        "\n",
        "rag_chain_from_source = RunnableParallel(\n",
        "    {\"context\": retriever, \"question\": RunnablePassthrough()}\n",
        ").assign(answer = rag_chain_from_docs)"
      ],
      "metadata": {
        "id": "xCGlQjarL8Fc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rag_chain_from_source.invoke(\"what is E32?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bxhv8ZBuSPjD",
        "outputId": "de6d9778-7ca1-43b0-84bd-e8082a1e772a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'context': [Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 3\\nDisclaimer\\nEBYTEreservesallrightstothisdocumentandtheinformationcontainedherein.\\nProducts,names,logosanddesignsdescribedhereinmayinwholeorinpartbe\\nsubjecttointellectualpropertyrights.Reproduction,use,modificationordisclosureto\\nthirdpartiesofthisdocumentoranypartthereofwithouttheexpresspermissionof\\nEBYTEisstrictlyprohibited.\\nTheinformationcontainedhereinisprovided“asis”andEBYTEassumesno\\nliabilityfortheuseoftheinformation.Nowarranty,eitherexpressorimplied,isgiven,\\nincludingbutnotlimited,withrespecttotheaccuracy,correctness,reliabilityand\\nfitnessforaparticularpurposeoftheinformation.Thisdocumentmayberevisedby\\nEBYTEatanytime.Formostrecentdocuments,visitwww.ebyte.com.', metadata={'source': '/content/1.pdf', 'page': 3}),\n",
              "  Document(page_content='E32-915T30DSX1276915M 30 80.3~19.2kDIP24*43SMA-K\\nE32-170T30DSX1278170M 30 80.3k~9.6kDIP24*43SMA-K\\nE32-868T20DSX1276868M 20 30.3~19.2kDIP21*36SMA-K\\nE32-915T20DSX1276915M 20 30.3~19.2kDIP21*36SMA-K\\nE32-433T20DCSX1278433M 20 30.3k~19.2kDIP21*36SMA-K\\nE32-433T30DSX1278433M 30 80.3k~19.2kDIP24*43SMA-K', metadata={'source': '/content/1.pdf', 'page': 18}),\n",
              "  Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 2\\nREVISIONHISTORY.......................................................................................................................21\\nABOUTUS..........................................................................................................................................21', metadata={'source': '/content/1.pdf', 'page': 2}),\n",
              "  Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 21\\nRevisionhistory\\nVersion Date Description Issuedby\\n1.00 2017-11-10 Initialversion huaa\\n1.10 2018-01-11 UpdatingE32(868T30S)/E32(915T30S) huaa\\n1.20 2018-01-15 UpdatingE32(868T20S)/E32(915T20S)/E32(400T20S) huaa\\n1.30 2018-01-22UpdatingE32(868T20D)/E32(868T30D)\\nE32(915T20D)/E32(915T30D)/E32(170T30D)huaa\\n1.40 2018-05-24 Updatingantennaoption huaa\\n1.50 2018-10-11 Manualdividing huaa\\n1.60 2020-04-13 Ren\\n1.8 2020-10-23 Parametercorrection Linson\\nAboutus\\nTechnicalsupport:support@cdebyte.com\\nDocumentsandRFSettingdownloadlink:www.ebyte.com\\nThankyouforusingEbyteproducts!Pleasecontactuswithanyquestionsorsuggestions:info@cdebyte.com\\n------------------------------------------------------------------------------------------------------------\\nFax:028-64146160ext.821\\nWeb:www.ebyte.com\\nAddress:BuildingB5,MouldIndustrialPark,199#XiquAvenue,Chengdu,Sichuan', metadata={'source': '/content/1.pdf', 'page': 21}),\n",
              "  Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 1\\nCONTENT\\n1.OVERVIEW...............................................................................................................................3\\n1.1INTRODUCTION..............................................................................................................................................................4\\n1.2FEATURES......................................................................................................................................................................4\\n2.SPECIFICATIONANDPARAMETER.........................................................................................5\\n2.1LIMITPARAMETER.........................................................................................................................................................5', metadata={'source': '/content/1.pdf', 'page': 1}),\n",
              "  Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 20\\n13Packageforbatchorder', metadata={'source': '/content/1.pdf', 'page': 20})],\n",
              " 'question': 'what is E32?',\n",
              " 'answer': {'context': [Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 3\\nDisclaimer\\nEBYTEreservesallrightstothisdocumentandtheinformationcontainedherein.\\nProducts,names,logosanddesignsdescribedhereinmayinwholeorinpartbe\\nsubjecttointellectualpropertyrights.Reproduction,use,modificationordisclosureto\\nthirdpartiesofthisdocumentoranypartthereofwithouttheexpresspermissionof\\nEBYTEisstrictlyprohibited.\\nTheinformationcontainedhereinisprovided“asis”andEBYTEassumesno\\nliabilityfortheuseoftheinformation.Nowarranty,eitherexpressorimplied,isgiven,\\nincludingbutnotlimited,withrespecttotheaccuracy,correctness,reliabilityand\\nfitnessforaparticularpurposeoftheinformation.Thisdocumentmayberevisedby\\nEBYTEatanytime.Formostrecentdocuments,visitwww.ebyte.com.', metadata={'source': '/content/1.pdf', 'page': 3}),\n",
              "   Document(page_content='E32-915T30DSX1276915M 30 80.3~19.2kDIP24*43SMA-K\\nE32-170T30DSX1278170M 30 80.3k~9.6kDIP24*43SMA-K\\nE32-868T20DSX1276868M 20 30.3~19.2kDIP21*36SMA-K\\nE32-915T20DSX1276915M 20 30.3~19.2kDIP21*36SMA-K\\nE32-433T20DCSX1278433M 20 30.3k~19.2kDIP21*36SMA-K\\nE32-433T30DSX1278433M 30 80.3k~19.2kDIP24*43SMA-K', metadata={'source': '/content/1.pdf', 'page': 18}),\n",
              "   Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 2\\nREVISIONHISTORY.......................................................................................................................21\\nABOUTUS..........................................................................................................................................21', metadata={'source': '/content/1.pdf', 'page': 2}),\n",
              "   Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 21\\nRevisionhistory\\nVersion Date Description Issuedby\\n1.00 2017-11-10 Initialversion huaa\\n1.10 2018-01-11 UpdatingE32(868T30S)/E32(915T30S) huaa\\n1.20 2018-01-15 UpdatingE32(868T20S)/E32(915T20S)/E32(400T20S) huaa\\n1.30 2018-01-22UpdatingE32(868T20D)/E32(868T30D)\\nE32(915T20D)/E32(915T30D)/E32(170T30D)huaa\\n1.40 2018-05-24 Updatingantennaoption huaa\\n1.50 2018-10-11 Manualdividing huaa\\n1.60 2020-04-13 Ren\\n1.8 2020-10-23 Parametercorrection Linson\\nAboutus\\nTechnicalsupport:support@cdebyte.com\\nDocumentsandRFSettingdownloadlink:www.ebyte.com\\nThankyouforusingEbyteproducts!Pleasecontactuswithanyquestionsorsuggestions:info@cdebyte.com\\n------------------------------------------------------------------------------------------------------------\\nFax:028-64146160ext.821\\nWeb:www.ebyte.com\\nAddress:BuildingB5,MouldIndustrialPark,199#XiquAvenue,Chengdu,Sichuan', metadata={'source': '/content/1.pdf', 'page': 21}),\n",
              "   Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 1\\nCONTENT\\n1.OVERVIEW...............................................................................................................................3\\n1.1INTRODUCTION..............................................................................................................................................................4\\n1.2FEATURES......................................................................................................................................................................4\\n2.SPECIFICATIONANDPARAMETER.........................................................................................5\\n2.1LIMITPARAMETER.........................................................................................................................................................5', metadata={'source': '/content/1.pdf', 'page': 1}),\n",
              "   Document(page_content='ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 20\\n13Packageforbatchorder', metadata={'source': '/content/1.pdf', 'page': 20})],\n",
              "  'question': 'what is E32?',\n",
              "  'context_1': 'ChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 3\\nDisclaimer\\nEBYTEreservesallrightstothisdocumentandtheinformationcontainedherein.\\nProducts,names,logosanddesignsdescribedhereinmayinwholeorinpartbe\\nsubjecttointellectualpropertyrights.Reproduction,use,modificationordisclosureto\\nthirdpartiesofthisdocumentoranypartthereofwithouttheexpresspermissionof\\nEBYTEisstrictlyprohibited.\\nTheinformationcontainedhereinisprovided“asis”andEBYTEassumesno\\nliabilityfortheuseoftheinformation.Nowarranty,eitherexpressorimplied,isgiven,\\nincludingbutnotlimited,withrespecttotheaccuracy,correctness,reliabilityand\\nfitnessforaparticularpurposeoftheinformation.Thisdocumentmayberevisedby\\nEBYTEatanytime.Formostrecentdocuments,visitwww.ebyte.com.\\n\\nE32-915T30DSX1276915M 30 80.3~19.2kDIP24*43SMA-K\\nE32-170T30DSX1278170M 30 80.3k~9.6kDIP24*43SMA-K\\nE32-868T20DSX1276868M 20 30.3~19.2kDIP21*36SMA-K\\nE32-915T20DSX1276915M 20 30.3~19.2kDIP21*36SMA-K\\nE32-433T20DCSX1278433M 20 30.3k~19.2kDIP21*36SMA-K\\nE32-433T30DSX1278433M 30 80.3k~19.2kDIP24*43SMA-K\\n\\nChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 2\\nREVISIONHISTORY.......................................................................................................................21\\nABOUTUS..........................................................................................................................................21\\n\\nChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 21\\nRevisionhistory\\nVersion Date Description Issuedby\\n1.00 2017-11-10 Initialversion huaa\\n1.10 2018-01-11 UpdatingE32(868T30S)/E32(915T30S) huaa\\n1.20 2018-01-15 UpdatingE32(868T20S)/E32(915T20S)/E32(400T20S) huaa\\n1.30 2018-01-22UpdatingE32(868T20D)/E32(868T30D)\\nE32(915T20D)/E32(915T30D)/E32(170T30D)huaa\\n1.40 2018-05-24 Updatingantennaoption huaa\\n1.50 2018-10-11 Manualdividing huaa\\n1.60 2020-04-13 Ren\\n1.8 2020-10-23 Parametercorrection Linson\\nAboutus\\nTechnicalsupport:support@cdebyte.com\\nDocumentsandRFSettingdownloadlink:www.ebyte.com\\nThankyouforusingEbyteproducts!Pleasecontactuswithanyquestionsorsuggestions:info@cdebyte.com\\n------------------------------------------------------------------------------------------------------------\\nFax:028-64146160ext.821\\nWeb:www.ebyte.com\\nAddress:BuildingB5,MouldIndustrialPark,199#XiquAvenue,Chengdu,Sichuan\\n\\nChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 1\\nCONTENT\\n1.OVERVIEW...............................................................................................................................3\\n1.1INTRODUCTION..............................................................................................................................................................4\\n1.2FEATURES......................................................................................................................................................................4\\n2.SPECIFICATIONANDPARAMETER.........................................................................................5\\n2.1LIMITPARAMETER.........................................................................................................................................................5\\n\\nChengduEbyteElectronicTechnologyCo,;Ltd E32-433T30Dusermanual\\nCopyright©2012–2019，ChengduEbyteElectronicTechnologyCo.,Ltd. 20\\n13Packageforbatchorder'}}"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    }
  ]
}