001_CondoScraping_gh.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib\n",
    "from bs4 import BeautifulSoup\n",
    "import pickle as pk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Select Bangkok, Resale condos\n",
    "#This block will retrive all urls for each page and save to 'urllist' list.\n",
    "\n",
    "#Add url of the first page\n",
    "urllist = [\n",
    "'https://baania.com/en/projects?ptype=2&province=3781&stype=resale-project'      \n",
    "]\n",
    "\n",
    "#Check the url patterns, last page is page=67\n",
    "#Loop from page 1 to page 67, append url into QuotePageList. For new condos, last page is 19.\n",
    "\n",
    "for i in range(1,68):\n",
    "    url=\"https://baania.com/en/projects?page=\"+str(i)+\"&ptype=2&province=3781&stype=resale-project\"\n",
    "    urllist.append(url)\n",
    "print(urllist[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "#This block will retrive the link for all condos that show up in each page saved in 'urllist'.\n",
    "\n",
    "CondoLinkList = [] \n",
    "\n",
    "#Loop in the urllist\n",
    "for url in urllist:\n",
    "\n",
    "    print(\"Current Page is \"+ url)\n",
    "    \n",
    "    Page = urllib.request.urlopen(url)\n",
    "    Soup = BeautifulSoup(Page, 'html.parser')\n",
    "\n",
    "    MainTag = Soup.find('ul', attrs={'data-ls-event': 'scroll'})    \n",
    "    LinkTags = MainTag.findAll('h3', attrs={'class': 'header-title'})\n",
    "    print('Number of house inside this page is ' + str(len(LinkTags)))\n",
    "\n",
    "    #Loop to get link for each condo in each url\n",
    "    for LinkTag in LinkTags:\n",
    "        Link = LinkTag.find('a').get('href') # อันนี้คือ keyword ของการดึง href จาก Tag (link ของแต่ละ Condo)\n",
    "        CondoLinkList.append(Link)\n",
    "    time.sleep(3) #Add sleep time (3 sec)\n",
    "\n",
    "print('CondoLinkList Completed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(CondoLinkList[0])\n",
    "print(CondoLinkList[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(CondoLinkList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save the CondoLinkList as pickle \n",
    "with open('CondoLinkList.pkl', 'wb') as f:\n",
    "    pk.dump(CondoLinkList, f)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(CondoLinkList[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}