002_CondoScraping_gh.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib\n",
    "from bs4 import BeautifulSoup\n",
    "import pickle as pk\n",
    "import openpyxl\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('CondoLinkList.pkl','rb') as f:  # Python 3: open(..., 'rb')\n",
    "    (CondoLinkList) = pk.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(CondoLinkList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Write function to retrive general info for each condo\n",
    "def func_Scrape_GenInfo(QuotePage):\n",
    "    print(QuotePage)\n",
    "    Page = urllib.request.urlopen(QuotePage)\n",
    "    Soup = BeautifulSoup(Page, 'html.parser')\n",
    "\n",
    "    Name  = Soup.find('div', attrs={'class': 'alt'}).text\n",
    "    print(Name)\n",
    "    \n",
    "    PriceMin = Soup.find('div', attrs={'class': 'price'}).text\n",
    "\n",
    "# 1. Sales Information header section\n",
    "\n",
    "    InfoMainTag = Soup.find('ul', attrs={'class': 'main-info-list'})\n",
    "    InfoTagObj  = InfoMainTag.findAll('div', attrs={'class': 'value'})\n",
    "    InfoCatTagObj  = InfoMainTag.findAll('div', attrs={'class': 'title'})\n",
    "    \n",
    "    print('---Info---')\n",
    "    \n",
    "    Location = ''\n",
    "    Status   = ''\n",
    "    Type     = ''\n",
    "    Segment  = ''\n",
    "    Build    = ''\n",
    "    Developer= ''\n",
    "    \n",
    "    for k in range(0,len(InfoCatTagObj)):\n",
    "        if InfoCatTagObj[k].text == 'Address':              Location = InfoTagObj[k].text\n",
    "        if InfoCatTagObj[k].text == 'Status':               Status = InfoTagObj[k].text\n",
    "        if InfoCatTagObj[k].text == 'Property Type':        Type = InfoTagObj[k].text\n",
    "        if InfoCatTagObj[k].text == 'Segment':              Segment = InfoTagObj[k].text\n",
    "        if InfoCatTagObj[k].text == 'Project Start Date':   Build = InfoTagObj[k].text\n",
    "        if InfoCatTagObj[k].text == 'Developed By':         Developer = InfoTagObj[k].text\n",
    "        \n",
    "# 2. Basic Information section\n",
    "    print('---Basic---')\n",
    "    BasicMainTag = Soup.find('ul', attrs={'class': 'basic-info-list'})\n",
    "    \n",
    "    try:\n",
    "        AreaTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item area-total'})\n",
    "        Area = AreaTag.text.strip().split('\\n')[1]\n",
    "        print('Area = ' +  Area)\n",
    "    except AttributeError :\n",
    "        Area = ''\n",
    "        \n",
    "        \n",
    "    try:        \n",
    "        NTypeTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item num-unit-type'})\n",
    "        NType = NTypeTag.text.strip().split('\\n')[1]\n",
    "        print('NType = ' + NType)\n",
    "    except AttributeError :\n",
    "        NType = ''\n",
    "\n",
    "    try:          \n",
    "        NUnitTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item num-unit'})\n",
    "        NUnit = NUnitTag.text.strip().split('\\n')[1]    \n",
    "        print('NUnit = ' + NUnit)\n",
    "    except AttributeError :\n",
    "        NUnit = ''        \n",
    "\n",
    "    try:    \n",
    "        NFloorTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item num-floor'})\n",
    "        NFloor = NFloorTag.text.strip().split('\\n')[1]\n",
    "        print('NFloor = ' + NFloor)\n",
    "    except AttributeError :\n",
    "        NFloor = ''\n",
    "\n",
    "    try:    \n",
    "        NParkingTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item num-parking'})\n",
    "        NParking = NParkingTag.text.strip().split('\\n')[1]\n",
    "        print('NParking = ' + NParking)\n",
    "    except AttributeError :\n",
    "        NParking = ''        \n",
    "        \n",
    "    try:    \n",
    "        RParkingTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item ratio-parking'})\n",
    "        RParking = RParkingTag.text.strip().split('\\n')[1]\n",
    "        print('RParking = ' + RParking)\n",
    "    except AttributeError :\n",
    "        RParking = ''        \n",
    "\n",
    "    try:    \n",
    "        NLiftTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item num-lift'})\n",
    "        NLift = NLiftTag.text.strip().split('\\n')[1]\n",
    "        print('NLift = ' + NLift)\n",
    "    except AttributeError :\n",
    "        NLift = ''                \n",
    "        \n",
    "    try:         \n",
    "        PriceStartTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item price-start'})\n",
    "        PriceStart = PriceStartTag.text.strip().split('\\n')[1].split()[0]    \n",
    "        print('PriceStart = ' + PriceStart)\n",
    "    except AttributeError :\n",
    "        PriceStart = ''\n",
    "        \n",
    "    try:        \n",
    "        PriceHighTag = BasicMainTag.find('li', attrs={'class': 'basic-info-item price-end'})\n",
    "        PriceHigh = PriceHighTag.text.strip().split('\\n')[1].split()[0]       \n",
    "        print('PriceHigh = ' + PriceHigh)\n",
    "    except AttributeError :\n",
    "        PriceHigh = ''\n",
    "         \n",
    "\n",
    "# 3. Image\n",
    "    ImageUrl = Soup.find('div', attrs = {'class':'entity-main-image'}).find('img').get('src')\n",
    "        \n",
    "# 4. Location (Lat/Long)\n",
    "    LocationTag = Soup.find('living-score-widget-map-canvas')\n",
    "    if LocationTag == None:\n",
    "        LocationTag = Soup.find('baania-map-streetview-overlay')\n",
    "    \n",
    "    try:\n",
    "        Latitude = LocationTag.get('latitude')\n",
    "    except  AttributeError :\n",
    "        Latitude = ''\n",
    "    \n",
    "    try:\n",
    "        Longitude = LocationTag.get('longitude')\n",
    "    except AttributeError :\n",
    "        Longitude = ''\n",
    "    \n",
    "# 5. Project Progress\n",
    "    try:\n",
    "        ProgressSect=Soup.find('div', attrs={'class': 'overall col-sm-3 col-lg-4'})\n",
    "        Progress=ProgressSect.find('div', attrs={'class': 'value'}).text\n",
    "    except AttributeError :\n",
    "        Progress=''\n",
    "        \n",
    "    condoinfo=(Name, PriceMin, Location, Status,  Type, Segment, Build, Developer,\n",
    "             Area, NType, NUnit,NFloor,NParking,RParking, NLift, PriceStart, PriceHigh,\n",
    "             ImageUrl, Latitude, Longitude,Progress)\n",
    "    \n",
    "    return condoinfo,Soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func_find_RoomResale(Soup):  \n",
    "    # find all 'Resale' rooms in this page    \n",
    "    AllRoomsTag= Soup.find('ul', attrs= {'class': 'listing-list item-list'})\n",
    "    AllRooms= AllRoomsTag.findAll('li', attrs= {'class': 'listing-row item-row pie-clearfix'})\n",
    "    print('------ Number of resale rooms inside this page is ' + str(len(AllRooms)))\n",
    "    return AllRooms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract all info in each 'Resale' room\n",
    "def func_Resale_info(AllRooms,i): \n",
    "    # extract data for each room\n",
    "    try: Link = AllRooms[i].find('h4', attrs={'class': 'title'}).find('a').get('href') # ดึง href ของแต่ละ room\n",
    "    except AttributeError : Link =''\n",
    "        \n",
    "    try: Title = AllRooms[i].find('h4', attrs={'class': 'title'}).text.strip().split('\\n')[0]\n",
    "    except AttributeError : Title =''\n",
    "        \n",
    "    try: Area = AllRooms[i].find('div', attrs={'class': 'usable-area'}).text.strip().split(' ', 2)[0]\n",
    "    except AttributeError : Area =''\n",
    "        \n",
    "    try: Price = AllRooms[i].find('div', attrs={'class': 'price'}).text.strip().split('\\n')[0].split()[0].replace(',', '')\n",
    "    except AttributeError : Price =''\n",
    "        \n",
    "    try: Bed = AllRooms[i].find('div', attrs={'class': 'col column-bed'}).text.strip().split('\\n')[0]\n",
    "    except AttributeError : Bed =''\n",
    "            \n",
    "    try: Bath = AllRooms[i].find('div', attrs={'class': 'col column-bath'}).text.strip().split('\\n')[0]\n",
    "    except AttributeError : Bath =''\n",
    "            \n",
    "    try: Parking = AllRooms[i].find('div', attrs={'class': 'col column-size'}).text.strip().split('\\n')[0]\n",
    "    except AttributeError : Parking =''  \n",
    "            \n",
    "    roominfo= (Link,Title,Area,Price,Bed,Bath,Parking)\n",
    "       \n",
    "    return roominfo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func_find_RoomRent(Soup):  \n",
    "\n",
    "    # find all 'Rental' rooms in this page    \n",
    "    AllRoomsTag= Soup.find('ul', attrs= {'class': 'rent-list item-list'})\n",
    "    AllRooms= AllRoomsTag.findAll('li', attrs= {'class': 'rent-row item-row pie-clearfix'})\n",
    "    print('------ Number of rental rooms inside this page is ' + str(len(AllRooms)))\n",
    "    return AllRooms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# extract all info in each 'Rental' room\n",
    "def func_Rent_info(AllRooms,i): \n",
    "    # extract data for each room\n",
    "    try: Link = AllRooms[i].find('h4', attrs={'class': 'title'}).find('a').get('href') # ดึง href ของแต่ละ room\n",
    "    except AttributeError : Link =''\n",
    "        \n",
    "    try: Title = AllRooms[i].find('h4', attrs={'class': 'title'}).text.strip().split('\\n')[0]\n",
    "    except AttributeError : Title =''\n",
    "        \n",
    "    try: Area = AllRooms[i].find('div', attrs={'class': 'usable-area'}).text.strip().split(' ', 2)[0]\n",
    "    except AttributeError : Area =''\n",
    "        \n",
    "    try: Price = AllRooms[i].find('div', attrs={'class': 'price'}).text.strip().split('\\n')[0].split()[0].replace(',', '')\n",
    "    except AttributeError : Price =''\n",
    "        \n",
    "    try: Bed = AllRooms[i].find('div', attrs={'class': 'col column-bed'}).text.strip().split('\\n')[0]\n",
    "    except AttributeError : Bed =''\n",
    "            \n",
    "    try: Bath = AllRooms[i].find('div', attrs={'class': 'col column-bath'}).text.strip().split('\\n')[0]\n",
    "    except AttributeError : Bath =''\n",
    "            \n",
    "    try: Parking = AllRooms[i].find('div', attrs={'class': 'col column-size'}).text.strip().split('\\n')[0]\n",
    "    except AttributeError : Parking =''  \n",
    "            \n",
    "    roominfo= (Link,Title,Area,Price,Bed,Bath,Parking)\n",
    "        \n",
    "    return roominfo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make excel database header (first row of excel database file)\n",
    "    \n",
    "wb = openpyxl.Workbook()\n",
    "ws = wb.active\n",
    "ws.title = \"Condo_info\"\n",
    "dest_filename = 'Condo_info.xlsx'\n",
    "\n",
    "current_col=0\n",
    "allinfo=['Name', 'PriceMin', 'Location', 'Status','Type', 'Segment', 'Build', 'Developer',\n",
    "         'CondoArea', 'NType', 'NUnit','NFloor','NParking','RParking', 'NLift', 'PriceStart', 'PriceHigh',\n",
    "         'ImageUrl', 'Latitude', 'Longitude','Progress',\n",
    "        'LinkS','TitleS','RoomAreaS','PriceS','BedS','BathS','ParkingS', # Resale Room info\n",
    "        'LinkR','TitleR','RoomAreaR','PriceR','BedR','BathR','ParkingR'] # Rental Room info\n",
    "for i in range(0,len(allinfo)):\n",
    "    ws.cell(row=1,column=i+1).value = str(allinfo[i])\n",
    "    current_col=current_col+1\n",
    "    \n",
    "wb.save(filename = dest_filename)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get info for the page\n",
    "start_time = time.time()\n",
    "start_row = 2\n",
    "r=start_row\n",
    "\n",
    "for c in range(0,len(CondoLinkList)): #len(CondoLinkList):\n",
    "    print('-------------- Current condo # is :' + str(c))\n",
    "    condoinfo,Soup = func_Scrape_GenInfo(CondoLinkList[c])     \n",
    "\n",
    "    # Some condos do not have Resale or Rental posting... Use try and except\n",
    "\n",
    "    try: # check if resale room is available                \n",
    "        AllRooms=func_find_RoomResale(Soup)            \n",
    "        for i in range(0,len(AllRooms)): # loop all rooms in condo\n",
    "            roominfo = func_Resale_info(AllRooms,i)\n",
    "            for i in range(0,len(condoinfo)):\n",
    "                ws.cell(row=r,column=i+1).value = condoinfo[i]              \n",
    "            for i in range(0,len(roominfo)):\n",
    "                ws.cell(row=r,column=len(condoinfo)+i+1).value = roominfo[i]                       \n",
    "            r+=1          \n",
    "        try: # check if rental room is available                \n",
    "            AllRooms=func_find_RoomRent(Soup)            \n",
    "            for i in range(0,len(AllRooms)): # loop all rooms in condo             \n",
    "                roominfo = func_Rent_info(AllRooms,i)\n",
    "                for i in range(0,len(condoinfo)):\n",
    "                    ws.cell(row=r,column=i+1).value = condoinfo[i]              \n",
    "                for i in range(0,len(roominfo)):\n",
    "                    ws.cell(row=r,column=len(roominfo)+len(condoinfo)+i+1).value = roominfo[i]                       \n",
    "                r+=1     \n",
    "        except AttributeError: \n",
    "            pass       \n",
    "\n",
    "        \n",
    "    except AttributeError: # if no resale for renting\n",
    "        try: # check if rental room is available                \n",
    "            AllRooms=func_find_RoomRent(Soup)            \n",
    "            for i in range(0,len(AllRooms)): # loop all rooms in condo                \n",
    "                roominfo = func_Rent_info(AllRooms,i)\n",
    "                for i in range(0,len(condoinfo)):\n",
    "                    ws.cell(row=r,column=i+1).value = condoinfo[i]              \n",
    "                for i in range(0,len(roominfo)):\n",
    "                    ws.cell(row=r,column=len(roominfo)+len(condoinfo)+i+1).value = roominfo[i]                       \n",
    "                r+=1 \n",
    "                \n",
    "        except AttributeError: # add only condo info            \n",
    "            for i in range(0,len(condoinfo)): # add condo info for each room\n",
    "                ws.cell(row=r,column=i+1).value = condoinfo[i]\n",
    "            r+=1 \n",
    "        \n",
    "    wb.save(filename = dest_filename)         \n",
    "    print('-------------- Completed Condo # '+str(c))\n",
    "    print('Total time = ' + str(time.time() - start_time) + ' seconds')\n",
    "    time.sleep(3)    #Add sleep time    \n",
    "print('---------------- Scraping Completed ----------------')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}