start.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install project dependencies \n",
    "# %pip install python-dotenv\n",
    "# %pip install openai\n",
    "# %pip install langchain\n",
    "# %pip install google-search-results\n",
    "# %pip install deeplake tiktoken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "import textwrap\n",
    "# Set the maximum line width\n",
    "max_line_width = 80\n",
    "\n",
    "# Load environment variables from .env file\n",
    "load_dotenv()\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.vectorstores import DeepLake\n",
    "embeddings = OpenAIEmbeddings()\n",
    "\n",
    "# Access the variables\n",
    "openai_api_key = os.getenv('OPENAI_API_KEY')\n",
    "activeloop_api_key = os.getenv('ACTIVELOOP_API_KEY')\n",
    "# !activeloop login -t \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.vectorstores import DeepLake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize OpenAI wrapper\n",
    "from langchain.llms import OpenAI\n",
    "llm = OpenAI(temperature=0.9)\n",
    "\n",
    "embeddings = OpenAIEmbeddings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load all files from a directory\n",
    "from langchain.document_loaders import TextLoader\n",
    "\n",
    "root_dir = './clone-nayms'\n",
    "docs = []\n",
    "for dirpath, dirnames, filenames in os.walk(root_dir):\n",
    "    for file in filenames:\n",
    "        try: \n",
    "            loader = TextLoader(os.path.join(dirpath, file), encoding='utf-8')\n",
    "            docs.extend(loader.load_and_split())\n",
    "        except Exception as e: \n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chunk the documents into smaller pieces\n",
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "\n",
    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
    "texts = text_splitter.split_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = DeepLake.from_documents(\n",
    "    texts, embeddings, dataset_path='hub://kevinpark/ds1', skip_ok=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deeplake_path = 'hub://kevinpark/ds1'\n",
    "db.ingest(deeplake_path, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import deeplake\n",
    "ds = deeplake.load('mem://langchain')\n",
    "# ds.visualize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = db.as_retriever()\n",
    "retriever.search_kwargs['distance_metric'] = 'cos'\n",
    "retriever.search_kwargs['fetch_k'] = 100\n",
    "retriever.search_kwargs['maximal_marginal_relevance'] = True\n",
    "retriever.search_kwargs['k'] = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.chains import ConversationalRetrievalChain\n",
    "\n",
    "# model = ChatOpenAI(model='gpt-4') # 'gpt-3.5-turbo',\n",
    "# model = ChatOpenAI(model='gpt-3.5-turbo') # 'gpt-3.5-turbo',\n",
    "model = ChatOpenAI(temperature=0) # 'gpt-3.5-turbo',\n",
    "qa = ConversationalRetrievalChain.from_llm(model,retriever=retriever)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_questions(qa, questions, chat_history=None, max_line_width=80):\n",
    "    # Initialize chat_history if it's not provided\n",
    "    if chat_history is None:\n",
    "        chat_history = []\n",
    "    # Iterate over the questions\n",
    "    for question in questions:  \n",
    "        # Process the question using the given qa object (ConversationalRetrievalChain)\n",
    "        result = qa({\"question\": question, \"chat_history\": chat_history})\n",
    "        # Append the question-answer pair to chat_history\n",
    "        chat_history.append((question, result['answer']))\n",
    "\n",
    "        # Wrap the text\n",
    "        wrapped_answer = textwrap.fill(result['answer'], width=max_line_width)\n",
    " \n",
    "        # Print the question\n",
    "        print(f\"-> **Question**: {question}\\n\")\n",
    "        \n",
    "        # Print the wrapped answer\n",
    "        print(f\"**Answer**:\\n{wrapped_answer}\\n\")\n",
    "    # Return the wrapped answer text and updated chat_history\n",
    "    # return wrapped_answer, chat_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = [\n",
    "    \"What does AppStorage do?\",\n",
    "] \n",
    "\n",
    "ask_questions(qa, questions, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Usage example:\n",
    "questions = [\n",
    "    \"Give me a list of all of the methods in AdminFacet.\",\n",
    "] \n",
    "\n",
    "ask_questions(qa, questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = [\n",
    "    \"What does withdrawDividend() do and how does it work? Can you show me an example of where it has a bug?\",\n",
    "] \n",
    "\n",
    "ask_questions(qa, questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = [\n",
    "    \"Are dividends correctly calculated?\",\n",
    "] \n",
    "\n",
    "ask_questions(qa, questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = [\n",
    "    \"How does the access control in Nayms work? The logic for the Nayms access control is called ACL and the logic is in the ACLFacet.sol file.\",\n",
    "] \n",
    "\n",
    "ask_questions(qa, questions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}