Homegate-Geneva scraping code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e1f09a91",
   "metadata": {},
   "source": [
    "# Homegate.ch scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "46eacd24",
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing necessary packages\n",
    "# using bs and requests modules for scraping\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import csv\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d3b923a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Page -> 1\n",
      "20 0\n",
      "Page -> 2\n",
      "21 19\n",
      "Page -> 3\n",
      "21 39\n",
      "Page -> 4\n",
      "21 59\n",
      "Page -> 5\n",
      "21 79\n",
      "Page -> 6\n",
      "21 99\n",
      "Page -> 7\n",
      "21 119\n",
      "Page -> 8\n",
      "21 139\n",
      "Page -> 9\n",
      "21 159\n",
      "Page -> 10\n",
      "21 179\n",
      "Page -> 11\n",
      "21 199\n",
      "Page -> 12\n",
      "21 219\n",
      "Page -> 13\n",
      "21 239\n",
      "Page -> 14\n",
      "21 259\n",
      "Page -> 15\n",
      "21 279\n",
      "Page -> 16\n",
      "21 299\n",
      "Page -> 17\n",
      "21 319\n",
      "Page -> 18\n",
      "21 339\n",
      "Page -> 19\n",
      "21 359\n",
      "Page -> 20\n",
      "21 379\n",
      "Page -> 21\n",
      "21 380\n",
      "Page -> 22\n",
      "Found 401apartments\n"
     ]
    }
   ],
   "source": [
    "# We define page number here for looping pagination\n",
    "cur_page = 1\n",
    "\n",
    "\n",
    "# After carefully isnpecting website, realized apt rental advertisement goes in 2 categories:\n",
    "# Premium one (paid subscription) and simple ones (without subscription) therefore create 2 empty list\n",
    "\n",
    "premium = []\n",
    "simple = []\n",
    "\n",
    "# Creating function for changing page numbers and While True loop has to run until it breaks\n",
    "def getLink(page):\n",
    "    return f\"https://www.homegate.ch/rent/apartment/city-geneva/matching-list?ep={page}\"\n",
    "    \n",
    "while True:\n",
    "    print(\"Page ->\", cur_page)\n",
    "    link = getLink(cur_page)\n",
    "    \n",
    "    res = requests.get(link)\n",
    "    \n",
    "    \n",
    "## we parse HTML with Beautiful Soup\n",
    "    bs = BeautifulSoup(res.text, features='html.parser')\n",
    "    \n",
    "## we define 2 variable a for premium and b for simple annoucements\n",
    "## and with find_all() we returns all div containers with mentioned class names that match our filters\n",
    "\n",
    "    a = bs.find_all('div', {'class': 'ListItemTopPremium_item_K9dLF'})\n",
    "    b = bs.find_all('div', {'class': 'ListItem_item_1GcIZ'})\n",
    "\n",
    "## if we get zero results then we break the while loop defined earlier (we have 21 pages of results)\n",
    "\n",
    "    if len(a) == 0 and len(b) == 0:\n",
    "        break\n",
    "        \n",
    "## otherwise we run for loop for findings in a and b then append according to the empty list created earlier\n",
    "\n",
    "    for offer in a:\n",
    "        premium.append(offer)\n",
    "    for offer in b:\n",
    "        simple.append(offer)\n",
    "\n",
    "    \n",
    "# printing results and incrementing page number by one \n",
    "\n",
    "    print(len(premium), len(simple))\n",
    "    cur_page += 1\n",
    "\n",
    "# Defining function block with result dic keys price,size,rooms,address and for each of them we use try except to get \n",
    "# within span tag with mentioned class name info needed  and add them to created list\n",
    "\n",
    "def extractPremiumInfo(block):\n",
    "    result = {\n",
    "        'price': None,\n",
    "        'size': None,\n",
    "        'rooms': None,\n",
    "        'address': None\n",
    "    }\n",
    "    try:\n",
    "        price = block.find('span', {'class': 'ListItemPrice_price_1o0i3'}).find_all('span')[1].text\n",
    "        result['price'] = price\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "    try:\n",
    "        m2 = block.find('span', {'class': 'ListItemLivingSpace_value_2zFir'}).text\n",
    "        result['size'] = m2\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "    try:\n",
    "        rn = block.find('span', {'class': 'ListItemRoomNumber_value_Hpn8O'}).text\n",
    "        result['rooms'] = rn\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "    address = block.find('div', {'class': 'ListItemTopPremium_data_3i7Ca'})\n",
    "    if address is None:\n",
    "        address = block.find('div', {'class': 'ListItem_data_18_z_'})\n",
    "\n",
    "    address = address.find_all('p')[1].text\n",
    "\n",
    "    result['address'] = address\n",
    "\n",
    "    return result\n",
    "\n",
    "# Again with for loop we go over simple and premium list and append results to fnish list.\n",
    "\n",
    "finish = []\n",
    "\n",
    "for i in premium:\n",
    "    finish.append(extractPremiumInfo(i))\n",
    "\n",
    "for i in simple:\n",
    "    finish.append(extractPremiumInfo(i))\n",
    "\n",
    "print(f\"Found {len(finish)}apartments\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fd40313e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(finish)\n",
    "df.to_csv('Geneva_listings_src.csv', index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed56ec60",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}