Streaming Twitter Data into MySQL Database.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tweepy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "consumer_key = \"AAAA\"\n",
    "consumer_secret = \"BBBB\"\n",
    "access_token = \"CCCC\"\n",
    "access_token_secret = \"DDDD\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth =  tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "api = tweepy.API(auth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "public_tweets = api.home_timeline()\n",
    "print(len(public_tweets))\n",
    "for tweet in public_tweets:\n",
    "    print(tweet.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pymysql\n",
    "import pymysql.cursors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "connection = pymysql.connect(host='localhost',\n",
    "                             user='root',\n",
    "                             password='Password',\n",
    "                             db='twitter_test_proj',\n",
    "                             charset='utf8mb4',\n",
    "                             cursorclass=pymysql.cursors.DictCursor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sql = \"SHOW DATABASES\"\n",
    "cur = connection.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'Database': 'information_schema'}, {'Database': 'mysql'}, {'Database': 'performance_schema'}, {'Database': 'sys'}, {'Database': 'twitter_test_proj'}]\n"
     ]
    }
   ],
   "source": [
    "cur.execute(sql)\n",
    "result = cur.fetchall()\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\owner\\anaconda3\\lib\\site-packages\\pymysql\\cursors.py:329: Warning: (1051, \"Unknown table 'twitter_test_proj.twitter_tweets'\")\n",
      "  self._do_get_result()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drop_table = \"\"\"DROP TABLE IF EXISTS twitter_tweets\"\"\"\n",
    "cur.execute(drop_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "create_table = \"\"\" CREATE TABLE IF NOT EXISTS twitter_tweets(\n",
    "    twitter_id INT(20) NOT NULL AUTO_INCREMENT,\n",
    "    user_name VARCHAR(255),\n",
    "    create_at VARCHAR(255),\n",
    "    tweet VARCHAR(255),\n",
    "    retweet_count INT(11),\n",
    "    location VARCHAR(255),\n",
    "    place VARCHAR(255),\n",
    "    PRIMARY KEY (twitter_id)\n",
    ")\"\"\"\n",
    "cur.execute(create_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "connection.commit()\n",
    "connection.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def connect(username, create_at, tweet, retweet_count, place, location):\n",
    "    \n",
    "    data_insert_values = [username, create_at, tweet, retweet_count, place, location]\n",
    "    \n",
    "    try:\n",
    "        connection = pymysql.connect(host='localhost',\n",
    "                             user='root',\n",
    "                             password='Password',\n",
    "                             db='twitter_test_proj',\n",
    "                             charset='utf8mb4',\n",
    "                             cursorclass=pymysql.cursors.DictCursor)\n",
    "        cur = connection.cursor()\n",
    "        \n",
    "        data_insert_query = \"\"\" INSERT INTO twitter_tweets (\n",
    "                            user_name, \n",
    "                            create_at, \n",
    "                            tweet, \n",
    "                            retweet_count, \n",
    "                            place, \n",
    "                            location)\n",
    "                            VALUES(%s, %s, %s, %s, %s, %s)\"\"\"\n",
    "        \n",
    "        cur.execute(data_insert_query, data_insert_values)\n",
    "        connection.commit()\n",
    "        \n",
    "    except pymysql.Error as e:\n",
    "        print(e)\n",
    "    \n",
    "    cur.close()\n",
    "    connection.close()\n",
    "    \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from dateutil import parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StreamListener(tweepy.StreamListener):\n",
    "    \n",
    "    def on_connect(self):\n",
    "        print(\"You are connected to Twitter API\")\n",
    "        \n",
    "    def on_error(self):\n",
    "        if status_code != 200:\n",
    "            print(\"Error found\")\n",
    "            return False\n",
    "        \n",
    "    \n",
    "    def on_data(self, data):\n",
    "        \n",
    "        try:\n",
    "            raw_data = json.loads(data)\n",
    "            \n",
    "            if 'text' in raw_data:\n",
    "                username = raw_data['user']['screen_name']\n",
    "                created_at = raw_data['created_at']\n",
    "                tweet = raw_data['text']\n",
    "                retweet_count = raw_data['retweet_count']\n",
    "                \n",
    "                if raw_data['place'] is not None:\n",
    "                    place = raw_data['place']['country']\n",
    "                    print(place)\n",
    "                else:\n",
    "                    place = ''\n",
    "                    \n",
    "                location = raw_data['user']['location']\n",
    "                \n",
    "                connect(username, created_at, tweet, retweet_count, place, location)\n",
    "                print(\"Tweet collected at: {}\".format(str(created_at)))\n",
    "                \n",
    "        except pymysql.Error as e:\n",
    "            print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are connected to Twitter API\n",
      "Tweet collected at: Wed Jun 05 04:46:08 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:18 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:18 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:20 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:20 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:26 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:28 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:38 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:55 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:56 +0000 2019\n",
      "Tweet collected at: Wed Jun 05 04:46:58 +0000 2019\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      ]
    }
   ],
   "source": [
    "listener = StreamListener(api = api)\n",
    "stream = tweepy.Stream(auth, listener = listener)\n",
    "\n",
    "\n",
    "track = ['airflow', 'spark', 'aws']\n",
    "stream.filter(track = track, languages=['en'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cur.execute(\"SELECT COUNT(*) FROM twitter_tweets\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'COUNT(*)': 11}]\n"
     ]
    }
   ],
   "source": [
    "result = cur.fetchall()\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}