report-202005.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code related to the 202005 progress report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import re\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIR = \"../data/text/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count the tweets per month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileNames = os.listdir(DATADIR)\n",
    "for filePattern in \"2020022[789] 202003 202004 202005\".split():\n",
    "    nbrOfHours = 0\n",
    "    dates = {}\n",
    "    nbrOfTweets = 0\n",
    "    for fileName in fileNames:\n",
    "        if re.search(filePattern,fileName):\n",
    "            nbrOfHours += 1\n",
    "            date = fileName[0:8]\n",
    "            dates[date] = True\n",
    "            nbrOfTweets += len(pd.read_csv(DATADIR+fileName))\n",
    "    print(filePattern,nbrOfTweets,round(nbrOfTweets/len(dates)),round(nbrOfTweets/nbrOfHours))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count tweets per month containing keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERY = \"corona|covid|flattenthecurve|blijfthuis|rivm|mondkapje|huisarts|houvol\"\n",
    "TEXT = \"text\"\n",
    "\n",
    "fileNames = os.listdir(DATADIR)\n",
    "for filePattern in \"2020022[789] 202003 202004 202005\".split():\n",
    "    nbrOfHours = 0\n",
    "    dates = {}\n",
    "    nbrOfTweets = 0\n",
    "    for fileName in fileNames:\n",
    "        if re.search(filePattern,fileName):\n",
    "            nbrOfHours += 1\n",
    "            date = fileName[0:8]\n",
    "            dates[date] = True\n",
    "            nbrOfTweets += len([tweet for index,tweet in pd.read_csv(DATADIR+fileName).iterrows() if re.search(QUERY,tweet[TEXT],flags=re.IGNORECASE)])\n",
    "    print(filePattern,nbrOfTweets,round(nbrOfTweets/len(dates)),round(nbrOfTweets/nbrOfHours))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}