cymbal_ingest_to_vector_database.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "id": "Ay8BBE3DXPYJDhq2z66xi2sh",
      "metadata": {
        "tags": [],
        "id": "Ay8BBE3DXPYJDhq2z66xi2sh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727767498172,
          "user_tz": -330,
          "elapsed": 50597,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "6f35a396-a82f-4a9f-dc3f-3f16c0cef431"
      },
      "source": [
        "!pip install --quiet --upgrade google_cloud_firestore google_cloud_aiplatform langchain langchain-google-vertexai langchain_community langchain_experimental pymupdf"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.4/149.4 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m336.8/336.8 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.3/5.3 MB\u001b[0m \u001b[31m58.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m52.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.3/87.3 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m83.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m208.1/208.1 kB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.5/3.5 MB\u001b[0m \u001b[31m81.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.9/15.9 MB\u001b[0m \u001b[31m78.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.5/130.5 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m400.9/400.9 kB\u001b[0m \u001b[31m32.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m292.2/292.2 kB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m434.9/434.9 kB\u001b[0m \u001b[31m30.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m56.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.3/49.3 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import vertexai\n",
        "from vertexai.language_models import TextEmbeddingModel\n",
        "from vertexai.generative_models import GenerativeModel\n",
        "\n",
        "import pickle\n",
        "from IPython.display import display, Markdown\n",
        "\n",
        "from langchain_google_vertexai import VertexAIEmbeddings\n",
        "from langchain_community.document_loaders import PyMuPDFLoader\n",
        "from langchain_experimental.text_splitter import SemanticChunker\n",
        "\n",
        "from google.cloud import firestore\n",
        "from google.cloud.firestore_v1.vector import Vector\n",
        "from google.cloud.firestore_v1.base_vector_query import DistanceMeasure"
      ],
      "metadata": {
        "id": "camOcS7b9Xt_"
      },
      "id": "camOcS7b9Xt_",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "PROJECT_ID = \"qwiklabs-gcp-02-3fa385ad1d3d\"  # @param {type:\"string\"}\n",
        "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
        "\n",
        "import vertexai\n",
        "\n",
        "vertexai.init(project=PROJECT_ID, location=LOCATION)"
      ],
      "metadata": {
        "id": "4JMV9eXU9naj"
      },
      "id": "4JMV9eXU9naj",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_model = VertexAIEmbeddings(model_name=\"text-embedding-004\")"
      ],
      "metadata": {
        "id": "IYwB5kxV9prt"
      },
      "id": "IYwB5kxV9prt",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!gcloud storage cp gs://partner-genai-bucket/genai069/nyc_food_safety_manual.pdf ."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7z_nROB29rs5",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727767506804,
          "user_tz": -330,
          "elapsed": 2840,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "2c5a2ff1-6703-48ad-bd86-8bd19b62f3fb"
      },
      "id": "7z_nROB29rs5",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Copying gs://partner-genai-bucket/genai069/nyc_food_safety_manual.pdf to file://./nyc_food_safety_manual.pdf\n",
            "\n",
            "Average throughput: 27.0MiB/s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loader = PyMuPDFLoader(\"./nyc_food_safety_manual.pdf\")\n",
        "data = loader.load()"
      ],
      "metadata": {
        "id": "bXUS4V1S9wjP"
      },
      "id": "bXUS4V1S9wjP",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def clean_page(page):\n",
        "  return page.page_content.replace(\"-\\n\",\"\")\\\n",
        "                          .replace(\"\\n\",\" \")\\\n",
        "                          .replace(\"\\x02\",\"\")\\\n",
        "                          .replace(\"\\x03\",\"\")\\\n",
        "                          .replace(\"fo d P R O T E C T I O N  T R A I N I N G  M A N U A L\",\"\")\\\n",
        "                          .replace(\"N E W  Y O R K  C I T Y  D E P A R T M E N T  O F  H E A L T H  &  M E N T A L  H Y G I E N E\",\"\")\n"
      ],
      "metadata": {
        "id": "UC6VJN1B9y17"
      },
      "id": "UC6VJN1B9y17",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "cleaned_pages = []\n",
        "for pages in data:\n",
        "  cleaned_pages.append(clean_page(pages))\n",
        "\n",
        "text_splitter = SemanticChunker(embedding_model)\n",
        "docs = text_splitter.create_documents(cleaned_pages[0:4])\n",
        "chunked_content = [doc.page_content for doc in docs]"
      ],
      "metadata": {
        "id": "nmvdOMAE95VJ"
      },
      "id": "nmvdOMAE95VJ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chunked_embeddings = embedding_model.embed_documents(chunked_content)"
      ],
      "metadata": {
        "id": "ZBY3Pinc-AqR"
      },
      "id": "ZBY3Pinc-AqR",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!gsutil cp gs://partner-genai-bucket/genai069/chunked_content.pkl .\n",
        "!gsutil cp gs://partner-genai-bucket/genai069/chunked_embeddings.pkl .\n",
        "\n",
        "chunked_content = pickle.load(open(\"chunked_content.pkl\", \"rb\"))\n",
        "chunked_embeddings = pickle.load(open(\"chunked_embeddings.pkl\", \"rb\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2K9Eb515-Cmz",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727767606843,
          "user_tz": -330,
          "elapsed": 5519,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "d54be618-a985-46fe-a070-0d71258c4e57"
      },
      "id": "2K9Eb515-Cmz",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Copying gs://partner-genai-bucket/genai069/chunked_content.pkl...\n",
            "/ [1 files][280.7 KiB/280.7 KiB]                                                \n",
            "Operation completed over 1 objects/280.7 KiB.                                    \n",
            "Copying gs://partner-genai-bucket/genai069/chunked_embeddings.pkl...\n",
            "/ [1 files][  1.8 MiB/  1.8 MiB]                                                \n",
            "Operation completed over 1 objects/1.8 MiB.                                      \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Populate a db variable with a Firestore Client.\n",
        "db = firestore.Client(project=PROJECT_ID)"
      ],
      "metadata": {
        "id": "hwGYuzsJ-Eak",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727767996675,
          "user_tz": -330,
          "elapsed": 9,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "hwGYuzsJ-Eak",
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Use a variable called collection to create a reference to a collection named food-safety.\n",
        "collection = db.collection('food-safety')"
      ],
      "metadata": {
        "id": "Y9Fz7W_1_2HH",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727768005773,
          "user_tz": -330,
          "elapsed": 5,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "Y9Fz7W_1_2HH",
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Using a combination of our lists chunked_content and chunked_embeddings,\n",
        "# add a document to your collection for each of your chunked documents.\n",
        "for i, (content, embedding) in enumerate(zip(chunked_content, chunked_embeddings)):\n",
        "    doc_ref = collection.document(f\"doc_{i}\")\n",
        "    doc_ref.set({\n",
        "        \"content\": content,\n",
        "        \"embedding\": Vector(embedding)\n",
        "    })\n"
      ],
      "metadata": {
        "id": "BwE3-9mN_4Nh",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727768064972,
          "user_tz": -330,
          "elapsed": 23498,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "BwE3-9mN_4Nh",
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!gcloud firestore indexes composite create \\\n",
        "--collection-group=food-safety \\\n",
        "--query-scope=COLLECTION \\\n",
        "--field-config field-path=embedding,vector-config='{\"dimension\":\"768\", \"flat\": \"{}\"}' \\\n",
        "--project=\"qwiklabs-gcp-02-3fa385ad1d3d\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z_FnRLmsAA5H",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727768244316,
          "user_tz": -330,
          "elapsed": 149778,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "fbe4b2c5-4240-45e2-9541-fff6df3ffd48"
      },
      "id": "z_FnRLmsAA5H",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Create request issued\n",
            "Created index [CICAgOjXh4EK].\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def search_vector_database(query: str):\n",
        "  context = \"\"\n",
        "  query_embedding = embedding_model.embed_query(query)\n",
        "  vector_query = collection.find_nearest(\n",
        "    vector_field=\"embedding\",\n",
        "    query_vector=Vector(query_embedding),\n",
        "    distance_measure=DistanceMeasure.EUCLIDEAN,\n",
        "    limit=5,\n",
        "  )\n",
        "  docs = vector_query.stream()\n",
        "  context = [result.to_dict()['content'] for result in docs]\n",
        "  return context"
      ],
      "metadata": {
        "id": "yP6ITM_tAOLl",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727768244317,
          "user_tz": -330,
          "elapsed": 13,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "yP6ITM_tAOLl",
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "search_vector_database(\"How should I store food?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m-oe552JAh2z",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1727768244317,
          "user_tz": -330,
          "elapsed": 10,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "cab7852d-c501-4d74-b049-e35428525a80"
      },
      "id": "m-oe552JAh2z",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[' Store foods away from dripping condensate , at least six inches above the floor and with enough space between items to encourage air circulation. Freezer Storage Freezing is an excellent method for prolonging the shelf life of foods. By keeping foods frozen solid, the bacterial growth is minimal at best. However, if frozen foods are thawed and then refrozen, then harmful bacteria can reproduce to dangerous levels when thawed for the second time. In addition to that, the quality of the food is also affected. Never refreeze thawed foods, instead use them immediately. Keep the following rules in mind for freezer storage:  Use First In First Out method of stock rotation. All frozen foods should be frozen solid with temperature at 0°F or lower. Always use clean containers that are clearly labeled and marked, and have proper and secure lids. Allow adequate spacing between food containers to allow for proper air circulation. Never use the freezer for cooling hot foods. * * Tip: When receiving multiple items, always store the frozen foods first, then foods that are to be refrigerated, and finally the non perishable dry goods. Dry Storage Proper storage of dry foods such as cereals, flour, rice, starches, spices, canned goods, packaged foods and vegetables that do not require refrigeration ensures that these foods will still be usable when needed. Adequate storage space as well as low humidity (50% or less), and low temperatures (70 °F or less) are strongly recommended.',\n",
              " 'Only use food containers that are clean, non-absorbent and are made from food-grade material intended for such use. Containers made from metal may react with certain type of high acid foods such as sauerkraut, citrus juices, tomato sauce, etc. Plastic food-grade containers are the best choice for these types of foods. Containers made of copper, brass, tin and galvanized metal should not be used. The use of such products is prohibited. Re-using cardboard containers to store cooked foods is also a source of contamination. Lining containers with newspapers, menus or other publication before placing foods is also prohibited as chemical dyes from these can easily leach into foods. Storage Areas Foods should only be stored in designated areas. Storing foods in passageways, rest rooms, garbage areas, utility rooms, etc. would subject these to contamination. Raw foods must always be stored below and away from cooked foods to avoid cross contamination. Refrigerated Storage This type of storage is typically used for holding potentially hazardous foods as well as perishable foods for short periods of time—a few hours to a few days. An adequate number of efficient refrigerated units are required to store potentially hazardous cold foods. By keeping cold foods cold, the microorganisms that are found naturally on these foods are kept to a minimum. Cold temperature does not kill microorganisms, however, it slows down their growth. Pre-packaged cold foods must be stored at temperatures recommended by the manufacturer. This is especially important when dealing with vacuum packed foods, modified atmosphere packages and sous vide foods. Smoked fish is required by the Health Code to be stored at 38°F or below. Fresh meat, poultry and other potentially hazardous foods must be stored at 41°F or below, while frozen foods must be stored at 0°F or below.',\n",
              " 'In addition to the above, avoid sunlight as it may affect the quality of some foods. Following are some of the guidelines:  Use First In First Out method of stock rotation. Keep foods at least 6 inches off the floor. This allows for proper cleaning and to detect vermin activity. Keep foods in containers with tightly fitted lids. Keep dry storage areas well lighted and ventilated. Install shades on windows to prevent exposure from sunlight. Do not store foods under overhead water lines that may drip due to leaks or condensation. Do not store garbage in dry food storage areas. Make sure that dry storage area is vermin proof by sealing walls and baseboards and by repairing holes and other openings. * * Safety Tip: Storage of harmful chemicals in the food storage areas can create hazardous situations and hence is prohibited by law. All chemicals must be labeled properly and used in accordance to the instructions on the label. Pesticide use is prohibited unless used by a licensed pest control officer. Storage in Ice Whenever food items are to be stored in ice, care must be taken to ensure that water from the melted ice is constantly being drained so that the food remains on ice and not immersed in iced water.',\n",
              " 'Furthermore, it is improper to store food in ice machines or ice that will be later used for human consumption. Food should be stored at least six inches off the floor, away from walls and dripping pipes. Keep all food, bulk or otherwise, covered and safe from contamination. Check food daily and throw away any spoiled or contaminated food. Store cleaning, disinfecting, and other chemicals away from foods, clearly marked and in their original containers. Keep food refrigerated at a temperature of 41°F or below. Monitor temperatures regularly with a thermometer placed in the warmest part of the refrigerator. Keep all cooling compartments closed except when you are using them. Store food in a refrigerator in such a way that the air inside can circulate freely. Keep all refrigerated foods covered, and use up stored leftovers quickly. When dishes and utensils are sparkling clean, keep them that way by proper storage.',\n",
              " 'l Store food in vermin-proof containers — metal or glass  containers, with tightly fitted lids. l Remove dented, leaking, rusted, swollen or unlabeled canned goods. Cold Storage: l All PHFs must be stored at 41° F (Except smoked fish at 38° F and raw shell eggs at 45 ° F). l All cooked and ready-to-eat food must be stored away from and above raw food. l Do not store foods in quantities that exceed the storage unit’s  capacity. l Place a refrigeration thermometer in the warmest spot in the unit to measure ambient air temperature of the unit l Check for condensation that may contaminate food. l Keep frozen foods frozen at 0° F or lower. STORAGE ']"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "kLdCA5FDAlLP"
      },
      "id": "kLdCA5FDAlLP",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.10"
    },
    "colab": {
      "provenance": [],
      "name": "cymbal_ingest_to_vector_database"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}