main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_count = 0\n",
    "\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "chrome_options = Options()\n",
    "chrome_options.add_argument(\"--headless\")\n",
    "browser = webdriver.Chrome(options=chrome_options)\n",
    "veri = \"covid19\"\n",
    "browser.get(\"https://twitter.com/search?q=\"+veri+\"%20lang%3Aen&src=typed_query&f=live\")\n",
    "time.sleep(2)\n",
    "\n",
    "tweets = []\n",
    "tweet = browser.find_elements(\"xpath\", \"//div[@data-testid='tweetText']\")\n",
    "time.sleep(2)\n",
    "for i in tweet:\n",
    "    tweets.append(i.text)\n",
    "    tweet_count += 1\n",
    "\n",
    "sayac = 0\n",
    "last = browser.execute_script(\"return document.documentElement.scrollHeight\") #We are taking the mouse scrolling height\n",
    "while True: #loop\n",
    "    if sayac > 10 : #if the counter is more than 10, the program is terminated.\n",
    "        break\n",
    "    browser.execute_script(\"window.scrollTo(0,document.documentElement.scrollHeight)\") #We're scrolling so we can access the tweets below.\n",
    "    new = browser.execute_script(\"return document.documentElement.scrollHeight\") #we get the new length we need to scroll with the mouse\n",
    "    if last == new : #If our translation is the same as the old translation, if nothing has changed, the program closes because there are no tweets left.\n",
    "        break\n",
    "    last = new\n",
    "    sayac += 1\n",
    "    tweet = browser.find_elements(\"xpath\", \"//div[@data-testid='tweetText']\") #scraping tweets from the page which in front of us\n",
    "    time.sleep(2) #2 seconds waiting\n",
    "    for i in tweet: #for each tweet that we scraped\n",
    "        tweets.append(i.text) #adding that tweet to list\n",
    "        tweet_count += 1 #adding 1 to tweet_count\n",
    "\n",
    "tweets_dataframe = pd.DataFrame(tweets, columns=['tweet'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_dataframe"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c7195dcf1fd822bb1e147b13e9e3020352b54981afaa83e0bfb01da98ffec2b2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}