01_Scraping_French_newspaper_crawler.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraping french news with newspaper-crawler\n",
    "This notebook can be used to scrap french news from RSS feed of these newspapers : \n",
    "- Figaro\n",
    "- Futura Sciences\n",
    "- Liberation\n",
    "- Le Monde\n",
    "\n",
    "Doesn't work for : \n",
    "- Nouvel Obs\n",
    "\n",
    "This notebook can be executed several times to add new news to database : newspaper_db\n",
    "\n",
    "## installation : \n",
    "pip install feedparser\n",
    "\n",
    "pip install scrapy\n",
    "\n",
    "pip install git+https://github.com/gbolmier/newspaper-crawler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# your path folder to save results\n",
    "PATH_FOLDER_SAVE = '../../data'\n",
    "\n",
    "PATH_DF_TRUE_NEWS = PATH_FOLDER_SAVE + '/df_true_news.pkl'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "# importing os module  \n",
    "import os\n",
    "# import datetime module\n",
    "import datetime\n",
    "    \n",
    "def save_results_newpaper_crawler(your_source_folder, your_save_folder=None):\n",
    "    '''\n",
    "    Move results to \"pressarticles_ #current date time#\"\n",
    "    '''\n",
    "    PATH_SOURCE_ARTICLES =  \"pressarticles\"\n",
    "    \n",
    "    if your_source_folder is None:\n",
    "        your_source_folder_articles = PATH_SOURCE_ARTICLES\n",
    "    else:\n",
    "        your_source_folder_articles = your_source_folder + \\\n",
    "            '/' + PATH_SOURCE_ARTICLES\n",
    "    \n",
    "    try:\n",
    "        d = datetime.datetime.now()\n",
    "        str_date = d.strftime(\"%Y%m%d_%H_%M_%S\")\n",
    "        # Directory \n",
    "        # saved files paths\n",
    "        if your_save_folder is None:\n",
    "            your_save_folder_articles =  your_source_folder_articles \\\n",
    "                + \"_\" + str_date\n",
    "        # move directory\n",
    "        shutil.move(your_source_folder_articles, your_save_folder_articles) \n",
    "        print(\"Directory '%s' created\" %your_save_folder_articles)\n",
    "    except:\n",
    "        print('Folder {} does not exist!'.format(your_source_folder_articles))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stop process after around 5 min manually...\n"
     ]
    }
   ],
   "source": [
    "print(\"Stop process after around 5 min manually...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Folder:  /Users/gregory/Documents/CloudStationSinchon/Applications/python/OC_IML_projet_7/code/FakeNewsDetectionFr\n",
      "Scraping Results Folder:  ../../data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-31 08:30:40 [scrapy.crawler] INFO: Overridden settings: {'DOWNLOAD_DELAY': 2}\n",
      "2019-12-31 08:30:40 [scrapy.extensions.telnet] INFO: Telnet Password: 1690a3858b6b3796\n",
      "2019-12-31 08:30:40 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.memusage.MemoryUsage',\n",
      " 'scrapy.extensions.logstats.LogStats']\n",
      "2019-12-31 08:30:41 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n",
      "2019-12-31 08:30:41 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2019-12-31 08:30:41 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "[]\n",
      "2019-12-31 08:30:41 [scrapy.core.engine] INFO: Spider opened\n",
      "2019-12-31 08:30:41 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:30:41 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n",
      "2019-12-31 08:31:08 [scrapy.crawler] INFO: Overridden settings: {'DOWNLOAD_DELAY': 2}\n",
      "2019-12-31 08:31:08 [scrapy.extensions.telnet] INFO: Telnet Password: 67158ab20ebd67cf\n",
      "2019-12-31 08:31:08 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.memusage.MemoryUsage',\n",
      " 'scrapy.extensions.logstats.LogStats']\n",
      "2019-12-31 08:31:08 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n",
      "2019-12-31 08:31:08 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2019-12-31 08:31:08 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "[]\n",
      "2019-12-31 08:31:08 [scrapy.core.engine] INFO: Spider opened\n",
      "2019-12-31 08:31:08 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:31:08 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6024\n",
      "2019-12-31 08:31:15 [scrapy.crawler] INFO: Overridden settings: {'DOWNLOAD_DELAY': 2}\n",
      "2019-12-31 08:31:15 [scrapy.extensions.telnet] INFO: Telnet Password: 6293ac863e28a198\n",
      "2019-12-31 08:31:15 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.memusage.MemoryUsage',\n",
      " 'scrapy.extensions.logstats.LogStats']\n",
      "2019-12-31 08:31:15 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n",
      "2019-12-31 08:31:15 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2019-12-31 08:31:15 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "[]\n",
      "2019-12-31 08:31:15 [scrapy.core.engine] INFO: Spider opened\n",
      "2019-12-31 08:31:15 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:31:15 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6025\n",
      "2019-12-31 08:31:47 [scrapy.crawler] INFO: Overridden settings: {'DOWNLOAD_DELAY': 2}\n",
      "2019-12-31 08:31:47 [scrapy.extensions.telnet] INFO: Telnet Password: b6cf090cff6a709f\n",
      "2019-12-31 08:31:47 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.memusage.MemoryUsage',\n",
      " 'scrapy.extensions.logstats.LogStats']\n",
      "2019-12-31 08:31:47 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n",
      "2019-12-31 08:31:47 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2019-12-31 08:31:47 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "[]\n",
      "2019-12-31 08:31:47 [scrapy.core.engine] INFO: Spider opened\n",
      "2019-12-31 08:31:48 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:31:48 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6026\n",
      "2019-12-31 08:32:06 [scrapy.crawler] INFO: Overridden settings: {'DOWNLOAD_DELAY': 2}\n",
      "2019-12-31 08:32:06 [scrapy.extensions.telnet] INFO: Telnet Password: 8eb6582ae78dd07b\n",
      "2019-12-31 08:32:06 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.memusage.MemoryUsage',\n",
      " 'scrapy.extensions.logstats.LogStats']\n",
      "2019-12-31 08:32:06 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-31 08:32:06 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2019-12-31 08:32:06 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "[]\n",
      "2019-12-31 08:32:06 [scrapy.core.engine] INFO: Spider opened\n",
      "2019-12-31 08:32:06 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:32:06 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6027\n",
      "2019-12-31 08:32:06 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:32:07 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.telerama.fr/radio/a-radio-france,-la-greve-reprend-du-service,n6581605.php> (referer: None)\n",
      "2019-12-31 08:32:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.futura-sciences.com/planete/actualites/catastrophe-naturelle-rechauffement-climatique-cout-exorbitant-catastrophes-naturelles-2019-78985/#xtor%3DRSS-8> (referer: None)\n",
      "2019-12-31 08:32:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/vox/culture/reflets-du-temps-20191230> (referer: None)\n",
      "2019-12-31 08:32:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/economie/article/2019/12/30/affaire-carlos-ghosn-l-ex-pdg-du-groupe-renault-nissan-a-quitte-le-japon-pour-le-liban_6024424_3234.html> (referer: None)\n",
      "2019-12-31 08:32:08 [futurasciences] DEBUG: Saved file Futura Sciences #1.json\n",
      "2019-12-31 08:32:08 [lemonde] DEBUG: Saved file Le Monde #1.json\n",
      "2019-12-31 08:32:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/planete/2019/12/30/retraites-explosion-rue-de-trevise-frappes-americaines-en-somalie-et-en-irak-l-actu-de-ce-lundi_1771372?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://next.liberation.fr/culture/2019/12/30/mort-de-vaughan-oliver-graphiste-historique-du-label-4ad_1771364?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:08 [liberation] DEBUG: Saved file Liberation #1.json\n",
      "2019-12-31 08:32:08 [scrapy.extensions.logstats] INFO: Crawled 1 pages (at 1 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:32:09 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/economie/article/2019/12/30/le-chinois-casil-vend-l-aeroport-de-toulouse-a-eiffage-et-realise-une-grosse-plus-value_6024419_3234.html> (referer: None)\n",
      "2019-12-31 08:32:09 [lemonde] DEBUG: Saved file Le Monde #2.json\n",
      "2019-12-31 08:32:09 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/langue-francaise/francophonie/mathieu-bock-cote-une-novlangue-pese-sur-nous-20191230> (referer: None)\n",
      "2019-12-31 08:32:10 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.telerama.fr/monde/evelyne-debourg,-letoilee-de-la-cantine,n6559363.php> (referer: None)\n",
      "2019-12-31 08:32:10 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.futura-sciences.com/sciences/actualites/astronomie-etoile-betelgeuse-perd-son-eclat-va-t-elle-exploser-supernova-bientot-78987/#xtor%3DRSS-8> (referer: None)\n",
      "2019-12-31 08:32:10 [telerama] DEBUG: Saved file Telerama #1.json\n",
      "2019-12-31 08:32:10 [futurasciences] DEBUG: Saved file Futura Sciences #2.json\n",
      "2019-12-31 08:32:10 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/25/retraites-le-trafic-sncf-et-ratp-toujours-perturbe-mardi_1771022?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:10 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://next.liberation.fr/cinema/2019/12/30/fuchun-par-monts-et-par-maux_1771403?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:12 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/politique/article/2019/12/30/greve-contre-la-reforme-des-retraites-pour-la-saint-sylvestre-la-moitie-des-tgv-et-ter-circuleront-trafic-tres-perturbe-a-la-ratp_6024417_823448.html> (referer: None)\n",
      "2019-12-31 08:32:12 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/langue-francaise/actu-des-mots/2010-2020-ces-mots-nouveaux-qui-racontent-la-decennie-20191230> (referer: None)\n",
      "2019-12-31 08:32:12 [lemonde] DEBUG: Saved file Le Monde #3.json\n",
      "2019-12-31 08:32:12 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.telerama.fr/sortir/tolkien-a-la-bnf-une-exposition-monumentale,n6527441.php> (referer: None)\n",
      "2019-12-31 08:32:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.futura-sciences.com/planete/actualites/biodiversite-biodiversite-ces-especes-vous-ne-reverrez-plus-declarees-eteintes-78950/#xtor%3DRSS-8> (referer: None)\n",
      "2019-12-31 08:32:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/explosion-rue-de-trevise-la-ville-de-paris-mise-en-cause-par-les-experts_1771377?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:13 [futurasciences] DEBUG: Saved file Futura Sciences #3.json\n",
      "2019-12-31 08:32:13 [scrapy.core.engine] INFO: Closing spider (finished)\n",
      "2019-12-31 08:32:13 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 994,\n",
      " 'downloader/request_count': 3,\n",
      " 'downloader/request_method_count/GET': 3,\n",
      " 'downloader/response_bytes': 83818,\n",
      " 'downloader/response_count': 3,\n",
      " 'downloader/response_status_count/200': 3,\n",
      " 'elapsed_time_seconds': 6.449952,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2019, 12, 30, 23, 32, 13, 190922),\n",
      " 'log_count/DEBUG': 25,\n",
      " 'log_count/INFO': 12,\n",
      " 'memusage/max': 77438976,\n",
      " 'memusage/startup': 77438976,\n",
      " 'response_received_count': 3,\n",
      " 'scheduler/dequeued': 3,\n",
      " 'scheduler/dequeued/memory': 3,\n",
      " 'scheduler/enqueued': 3,\n",
      " 'scheduler/enqueued/memory': 3,\n",
      " 'start_time': datetime.datetime(2019, 12, 30, 23, 32, 6, 740970)}\n",
      "2019-12-31 08:32:13 [scrapy.core.engine] INFO: Spider closed (finished)\n",
      "2019-12-31 08:32:13 [liberation] DEBUG: Saved file Liberation #2.json\n",
      "2019-12-31 08:32:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://next.liberation.fr/cinema/2019/12/30/gu-xiaogang-la-chine-va-trop-vite-il-faut-donc-une-camera-patiente_1771401?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/actualite-france/la-chute-de-l-ecrivain-gabriel-matzneff-20191230> (referer: None)\n",
      "2019-12-31 08:32:14 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/police-justice/article/2019/12/30/explosion-rue-de-trevise-a-paris-des-experts-pointent-des-manquements-de-la-ville-et-d-une-entreprise-de-travaux-publics_6024408_1653578.html> (referer: None)\n",
      "2019-12-31 08:32:14 [lemonde] DEBUG: Saved file Le Monde #4.json\n",
      "2019-12-31 08:32:14 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.telerama.fr/radio/frederic-fromet,-limpertinent-chansonnier-en-concert-sur-france-inter,n6576396.php> (referer: None)\n",
      "2019-12-31 08:32:15 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/populisme-l-homme-fort-est-il-vraiment-un-homme-bete_1771370?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:15 [liberation] DEBUG: Saved file Liberation #3.json\n",
      "2019-12-31 08:32:15 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://next.liberation.fr/cinema/2019/12/30/les-filles-du-docteur-march-le-reve-est-hardi_1771384?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:15 [scrapy.extensions.logstats] INFO: Crawled 8 pages (at 8 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:32:16 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/malgre-les-incendies-sydney-maintient-son-feu-d-artifice-du-31-decembre-20191230> (referer: None)\n",
      "2019-12-31 08:32:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.telerama.fr/monde/precarite-et-endettement,-double-peine-pour-les-jeunes,n6579816.php> (referer: None)\n",
      "2019-12-31 08:32:17 [scrapy.core.engine] INFO: Closing spider (finished)\n",
      "2019-12-31 08:32:17 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 1414,\n",
      " 'downloader/request_count': 5,\n",
      " 'downloader/request_method_count/GET': 5,\n",
      " 'downloader/response_bytes': 131007,\n",
      " 'downloader/response_count': 5,\n",
      " 'downloader/response_status_count/200': 5,\n",
      " 'elapsed_time_seconds': 95.848411,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2019, 12, 30, 23, 32, 17, 278567),\n",
      " 'log_count/DEBUG': 36,\n",
      " 'log_count/INFO': 52,\n",
      " 'memusage/max': 77438976,\n",
      " 'memusage/startup': 75243520,\n",
      " 'response_received_count': 5,\n",
      " 'scheduler/dequeued': 5,\n",
      " 'scheduler/dequeued/memory': 5,\n",
      " 'scheduler/enqueued': 5,\n",
      " 'scheduler/enqueued/memory': 5,\n",
      " 'start_time': datetime.datetime(2019, 12, 30, 23, 30, 41, 430156)}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-31 08:32:17 [scrapy.core.engine] INFO: Spider closed (finished)\n",
      "2019-12-31 08:32:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/international/article/2019/12/30/un-attentat-de-l-etat-islamique-dejoue-en-russie-grace-a-des-renseignements-americains_6024420_3210.html> (referer: None)\n",
      "2019-12-31 08:32:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/audiences-tele-2019-arte-en-refuge-tf1-en-recul_1771375?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:17 [liberation] DEBUG: Saved file Liberation #4.json\n",
      "2019-12-31 08:32:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://next.liberation.fr/cinema/2019/12/30/ghost-tropic-attentions-a-la-marge_1771386?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/planete/2019/12/30/au-soudan-27-tortionnaires-de-l-ancien-regime-condamnes-a-mort_1771378?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:18 [liberation] DEBUG: Saved file Liberation #5.json\n",
      "2019-12-31 08:32:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/au-bresil-la-ville-de-sao-jose-a-vote-pour-le-changement-et-n-a-pas-de-regrets-20191230> (referer: None)\n",
      "2019-12-31 08:32:20 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/afrique/article/2019/12/30/au-soudan-27-membres-du-renseignement-condamnes-a-mort-pour-avoir-tue-un-manifestant_6024414_3212.html> (referer: None)\n",
      "2019-12-31 08:32:20 [lemonde] DEBUG: Saved file Le Monde #5.json\n",
      "2019-12-31 08:32:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://next.liberation.fr/cinema/2019/12/30/le-miracle-du-saint-inconnu-chacun-sa-part-de-desert_1771385?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/debats/2019/12/30/des-moyens-ou-la-fin-de-la-maison-des-ecrivains_1769589?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:21 [liberation] DEBUG: Saved file Liberation #6.json\n",
      "2019-12-31 08:32:22 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/vox/monde/renaud-girard-2010-2019-la-fin-du-leadership-occidental-20191230> (referer: None)\n",
      "2019-12-31 08:32:22 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/culture/article/2019/12/30/le-cinema-realise-sa-meilleure-annee-depuis-1966-avec-plus-de-213-millions-d-entrees_6024418_3246.html> (referer: None)\n",
      "2019-12-31 08:32:22 [lemonde] DEBUG: Saved file Le Monde #6.json\n",
      "2019-12-31 08:32:23 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://next.liberation.fr/cinema/2019/12/30/l-art-du-mensonge-le-senior-des-arnaques_1771381?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:23 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/sport/article/2019/12/30/football-l-assistance-a-l-arbitrage-video-doit-se-limiter-aux-erreurs-claires-et-evidentes-selon-l-ifab_6024422_3242.html> (referer: None)\n",
      "2019-12-31 08:32:24 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/2019-en-france-une-annee-enflammee_1771182?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:24 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/jair-bolsonaro-de-la-vulgarite-et-des-polemiques-20191230> (referer: None)\n",
      "2019-12-31 08:32:26 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/pixels/article/2019/12/30/scandale-cambridge-analytica-le-bresil-inflige-une-amende-de-1-5-million-d-euros-pour-facebook_6024423_4408996.html> (referer: None)\n",
      "2019-12-31 08:32:26 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/le-probleme-ce-n-est-pas-les-urgences-c-est-l-hopital_1771353?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:26 [liberation] DEBUG: Saved file Liberation #7.json\n",
      "2019-12-31 08:32:26 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/les-etats-unis-et-l-iran-s-affrontent-en-terrain-irakien-20191230> (referer: None)\n",
      "2019-12-31 08:32:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/afrique/article/2019/12/30/en-rdc-dix-huit-morts-dans-une-nouvelle-attaque-des-adf-a-beni_6024413_3212.html> (referer: None)\n",
      "2019-12-31 08:32:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/conjoncture/les-stations-de-ski-slaloment-entre-les-greves-20191230> (referer: None)\n",
      "2019-12-31 08:32:28 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/planete/2019/12/30/chine-le-pere-des-bebes-ogm-condamne-a-trois-ans-de-prison_1771360?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:29 [liberation] DEBUG: Saved file Liberation #8.json\n",
      "2019-12-31 08:32:30 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/societes/fans-de-foot-europeen-les-investisseurs-americains-partent-a-l-attaque-du-calcio-20191230> (referer: None)\n",
      "2019-12-31 08:32:30 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/idees/article/2019/12/30/maroc-un-journaliste-est-emprisonne-juste-pour-un-tweet-denoncant-l-injustice-de-la-justice_6024411_3232.html> (referer: None)\n",
      "2019-12-31 08:32:30 [lemonde] DEBUG: Saved file Le Monde #7.json\n",
      "2019-12-31 08:32:31 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/impot-sur-le-revenu-taxe-d-habitation-cotons-tiges-ce-qui-change-au-1er-janvier_1771359?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:31 [liberation] DEBUG: Saved file Liberation #9.json\n",
      "2019-12-31 08:32:33 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/culture/jean-pruvost-soyons-fiers-de-parler-francais-20191230> (referer: None)\n",
      "2019-12-31 08:32:33 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/culture/article/2019/12/30/57-rue-de-varenne-de-francois-perache-le-feuilleton-radio-d-un-ancien-de-matignon-et-de-l-elysee_6024421_3246.html> (referer: None)\n",
      "2019-12-31 08:32:33 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.liberation.fr/direct/element/lassociation-ni-putes-ni-soumises-expulsee-de-ses-locaux-a-montreuil_107128/> from <GET https://www.liberation.fr/france/2019/12/30/l-association-ni-putes-ni-soumises-expulsee-de-ses-locaux-a-montreuil_1771373?xtor=rss-450>\n",
      "2019-12-31 08:32:33 [lemonde] DEBUG: Saved file Le Monde #8.json\n",
      "2019-12-31 08:32:34 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://sante.lefigaro.fr/article/accidents-medicaux-les-autorites-ont-recense-820-cas-en-2018/> (referer: None)\n",
      "2019-12-31 08:32:34 [lefigaro] DEBUG: Saved file Le Figaro #1.json\n",
      "2019-12-31 08:32:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/debats/2019/12/30/rifaat-al-assad-la-repression-du-blanchiment-d-argent-au-coeur-d-un-proces-historique_1770393?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:35 [liberation] DEBUG: Saved file Liberation #10.json\n",
      "2019-12-31 08:32:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/conjoncture/les-tarifs-des-complementaires-sante-vont-flamber-20191230> (referer: None)\n",
      "2019-12-31 08:32:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://sante.lefigaro.fr/article/mois-sans-alcool-le-defi-de-janvier-suscite-la-polemique/> (referer: None)\n",
      "2019-12-31 08:32:36 [lefigaro] DEBUG: Saved file Le Figaro #2.json\n",
      "2019-12-31 08:32:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/sciences/article/2019/12/30/comment-les-idees-viennent-aux-scientifiques-mes-idees-les-plus-interessantes-ne-me-sont-pas-venues-assis-a-mon-bureau_6024415_1650684.html> (referer: None)\n",
      "2019-12-31 08:32:36 [lemonde] DEBUG: Saved file Le Monde #9.json\n",
      "2019-12-31 08:32:37 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/politiques/2019/12/30/creez-vos-voeux-presidentiels-pour-2020-avec-notre-generateur_1771356?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:38 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/le-ton-monte-entre-la-bolivie-l-espagne-et-le-mexique-apres-un-incident-diplomatique-20191230> (referer: None)\n",
      "2019-12-31 08:32:38 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lemonde.fr/sciences/article/2019/12/30/sciences-les-decouvertes-vertigineuses-de-2019_6024412_1650684.html> (referer: None)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-31 08:32:38 [scrapy.core.engine] INFO: Closing spider (finished)\n",
      "2019-12-31 08:32:38 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 4926,\n",
      " 'downloader/request_count': 14,\n",
      " 'downloader/request_method_count/GET': 14,\n",
      " 'downloader/response_bytes': 754995,\n",
      " 'downloader/response_count': 14,\n",
      " 'downloader/response_status_count/200': 14,\n",
      " 'elapsed_time_seconds': 90.312343,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2019, 12, 30, 23, 32, 38, 919931),\n",
      " 'log_count/DEBUG': 84,\n",
      " 'log_count/INFO': 46,\n",
      " 'memusage/max': 77438976,\n",
      " 'memusage/startup': 77438976,\n",
      " 'response_received_count': 14,\n",
      " 'scheduler/dequeued': 14,\n",
      " 'scheduler/dequeued/memory': 14,\n",
      " 'scheduler/enqueued': 14,\n",
      " 'scheduler/enqueued/memory': 14,\n",
      " 'start_time': datetime.datetime(2019, 12, 30, 23, 31, 8, 607588)}\n",
      "2019-12-31 08:32:38 [scrapy.core.engine] INFO: Spider closed (finished)\n",
      "2019-12-31 08:32:39 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/explosion-de-la-rue-de-trevise-manquements-de-la-ville-de-paris-et-d-une-entreprise-selon-des-expert_1771366?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:39 [liberation] DEBUG: Saved file Liberation #11.json\n",
      "2019-12-31 08:32:40 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/sciences/les-saignees-therapeutiques-en-danger-20191230> (referer: None)\n",
      "2019-12-31 08:32:42 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/medias/audiences-tv-annee-difficile-pour-les-chaines-historiques-20191230> (referer: None)\n",
      "2019-12-31 08:32:43 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/planete/2019/12/30/irak-derriere-les-frappes-americaines-l-ombre-du-conflit-avec-l-iran_1771420?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:43 [liberation] DEBUG: Saved file Liberation #12.json\n",
      "2019-12-31 08:32:45 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/planete/2019/12/30/irak-kataeb-hezbollah-la-milice-pro-iranienne-la-plus-puissante_1771419?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:45 [liberation] DEBUG: Saved file Liberation #13.json\n",
      "2019-12-31 08:32:45 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/musique/2020-l-annee-d-un-grand-mercato-dans-les-orchestres-20191230> (referer: None)\n",
      "2019-12-31 08:32:47 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/politique/mounir-mahjoubi-une-metamorphose-pour-rebondir-20191230> (referer: None)\n",
      "2019-12-31 08:32:47 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/planete/2019/12/30/bolsonaro-une-annee-de-sape-du-logement-social_1771414?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:48 [scrapy.extensions.logstats] INFO: Crawled 20 pages (at 20 pages/min), scraped 0 items (at 0 items/min)\n",
      "2019-12-31 08:32:48 [liberation] DEBUG: Saved file Liberation #14.json\n",
      "2019-12-31 08:32:49 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/vox/societe/eugenie-bastie-matzneff-quand-l-intelligentsia-defendait-la-pedophilie-20191230> (referer: None)\n",
      "2019-12-31 08:32:50 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/planete/2019/12/30/au-bresil-l-avenement-d-un-liberalisme-autoritaire_1771409?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:50 [liberation] DEBUG: Saved file Liberation #15.json\n",
      "2019-12-31 08:32:51 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/greve-a-la-ratp-chacun-cherche-sa-voie_1771424?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:51 [liberation] DEBUG: Saved file Liberation #16.json\n",
      "2019-12-31 08:32:51 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/conjoncture/audio-optique-dentaire-le-big-bang-de-la-reforme-du-100-sante-20191230> (referer: None)\n",
      "2019-12-31 08:32:52 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://bourse.lefigaro.fr/indices/les-taux-negatifs-chamboulent-l-epargne-20191230> from <GET http://bourse.lefigaro.fr/indices/les-taux-negatifs-chamboulent-l-epargne-20191230>\n",
      "2019-12-31 08:32:53 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/politique/perrineau-les-voeux-ne-suffisent-pas-a-retourner-l-opinion-20191230> (referer: None)\n",
      "2019-12-31 08:32:54 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/pigeons-bons-bisets-de-nos-villes_1771406?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:54 [liberation] DEBUG: Saved file Liberation #17.json\n",
      "2019-12-31 08:32:55 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/jair-bolsonaro-un-president-en-campagne-permanente-20191230> (referer: None)\n",
      "2019-12-31 08:32:55 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://bourse.lefigaro.fr/indices/les-taux-negatifs-chamboulent-l-epargne-20191230> (referer: None)\n",
      "2019-12-31 08:32:56 [lefigaro] DEBUG: Saved file Le Figaro #3.json\n",
      "2019-12-31 08:32:56 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/glyphosate-dans-les-urines-la-reponse-qu-on-cherche-encore_1771400?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:32:56 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://bourse.lefigaro.fr/indices/la-bourse-de-paris-boucle-sa-meilleure-annee-depuis-20-ans-20191230> from <GET http://bourse.lefigaro.fr/indices/la-bourse-de-paris-boucle-sa-meilleure-annee-depuis-20-ans-20191230>\n",
      "2019-12-31 08:32:57 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/heureux-comme-les-francais-expatries-au-quebec-20191230> (referer: None)\n",
      "2019-12-31 08:32:58 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://bourse.lefigaro.fr/indices/ces-entreprises-qui-affolent-les-compteurs-de-la-bourse-de-paris-20191230> from <GET http://bourse.lefigaro.fr/indices/ces-entreprises-qui-affolent-les-compteurs-de-la-bourse-de-paris-20191230>\n",
      "2019-12-31 08:32:59 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/vote-de-l-assemblee-sur-adp-la-reponse-la-plus-partagee_1771399?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:33:00 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/reveillon-sous-tension-pour-parly-dans-le-detroit-d-ormuz-20191230> (referer: None)\n",
      "2019-12-31 08:33:01 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://bourse.lefigaro.fr/indices/la-bourse-de-paris-boucle-sa-meilleure-annee-depuis-20-ans-20191230> (referer: None)\n",
      "2019-12-31 08:33:01 [lefigaro] DEBUG: Saved file Le Figaro #4.json\n",
      "2019-12-31 08:33:01 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/l-ecolo-humiliee-chez-praud-la-reponse-la-plus-lue_1771396?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:33:02 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://video.lefigaro.fr/figaro/video/reforme-des-retraites-yves-veyrier-appelle-le-gouvernement-a-revenir-a-la-sagesse/6118762290001/> (referer: None)\n",
      "2019-12-31 08:33:03 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://bourse.lefigaro.fr/indices/ces-entreprises-qui-affolent-les-compteurs-de-la-bourse-de-paris-20191230> (referer: None)\n",
      "2019-12-31 08:33:03 [lefigaro] DEBUG: Saved file Le Figaro #5.json\n",
      "2019-12-31 08:33:03 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://immobilier.lefigaro.fr/annonces/edito/acheter/j-achete/si-vous-achetez-un-logement-ancien-en-effectuant-des> (referer: None)\n",
      "2019-12-31 08:33:03 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/roman-souchtchenko-j-ai-tenu-grace-aux-livres-et-au-dessin-20191230> (referer: None)\n",
      "2019-12-31 08:33:03 [scrapy.core.scraper] ERROR: Spider error processing <GET https://immobilier.lefigaro.fr/annonces/edito/acheter/j-achete/si-vous-achetez-un-logement-ancien-en-effectuant-des> (referer: None)\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/processors.py\", line 59, in __call__\n",
      "    value = func(value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/newspaper_crawler/loaders/lefigaro_loader.py\", line 12, in clean_date\n",
      "    return re.search(\"[0-9]+/[0-9]+/[0-9]+\", rawdata).group(0)\n",
      "AttributeError: 'NoneType' object has no attribute 'group'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 159, in _process_input_value\n",
      "    return proc(value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/processors.py\", line 63, in __call__\n",
      "    (str(func), value, type(e).__name__, str(e)))\n",
      "ValueError: Error in Compose with <function LeFigaroLoader.clean_date at 0x1054fa048> value='Monday, December 30, 2019' error='AttributeError: 'NoneType' object has no attribute 'group''\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/twisted/internet/defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/newspaper_crawler/spiders/lefigaro_spider.py\", line 18, in parse\n",
      "    loader.add_xpath(\"date\", \"//time//text()\")\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 181, in add_xpath\n",
      "    self.add_value(field_name, values, *processors, **kw)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 78, in add_value\n",
      "    self._add_value(field_name, value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 92, in _add_value\n",
      "    processed_value = self._process_input_value(field_name, value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 164, in _process_input_value\n",
      "    value, type(e).__name__, str(e)))\n",
      "ValueError: Error with input processor Compose: field='date' value=['Monday, December 30, 2019'] error='ValueError: Error in Compose with <function LeFigaroLoader.clean_date at 0x1054fa048> value='Monday, December 30, 2019' error='AttributeError: 'NoneType' object has no attribute 'group'''\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-31 08:33:04 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/tchernobyl-la-reponse-la-plus-longue-qui-refait-l-histoire_1771394?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:33:04 [liberation] DEBUG: Saved file Liberation #18.json\n",
      "2019-12-31 08:33:04 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://immobilier.lefigaro.fr/annonces/edito/acheter/je-me-prepare/premier-achat-immobilier-conseils> (referer: None)\n",
      "2019-12-31 08:33:04 [scrapy.core.scraper] ERROR: Spider error processing <GET https://immobilier.lefigaro.fr/annonces/edito/acheter/je-me-prepare/premier-achat-immobilier-conseils> (referer: None)\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/processors.py\", line 59, in __call__\n",
      "    value = func(value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/newspaper_crawler/loaders/lefigaro_loader.py\", line 12, in clean_date\n",
      "    return re.search(\"[0-9]+/[0-9]+/[0-9]+\", rawdata).group(0)\n",
      "AttributeError: 'NoneType' object has no attribute 'group'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 159, in _process_input_value\n",
      "    return proc(value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/processors.py\", line 63, in __call__\n",
      "    (str(func), value, type(e).__name__, str(e)))\n",
      "ValueError: Error in Compose with <function LeFigaroLoader.clean_date at 0x1054fa048> value='Monday, December 30, 2019' error='AttributeError: 'NoneType' object has no attribute 'group''\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/twisted/internet/defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/newspaper_crawler/spiders/lefigaro_spider.py\", line 18, in parse\n",
      "    loader.add_xpath(\"date\", \"//time//text()\")\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 181, in add_xpath\n",
      "    self.add_value(field_name, values, *processors, **kw)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 78, in add_value\n",
      "    self._add_value(field_name, value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 92, in _add_value\n",
      "    processed_value = self._process_input_value(field_name, value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 164, in _process_input_value\n",
      "    value, type(e).__name__, str(e)))\n",
      "ValueError: Error with input processor Compose: field='date' value=['Monday, December 30, 2019'] error='ValueError: Error in Compose with <function LeFigaroLoader.clean_date at 0x1054fa048> value='Monday, December 30, 2019' error='AttributeError: 'NoneType' object has no attribute 'group'''\n",
      "2019-12-31 08:33:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/international/2019-nouvelle-annee-tres-meurtriere-aux-etats-unis-20191230> (referer: None)\n",
      "2019-12-31 08:33:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/quand-checknews-repond-aux-questions-sur-liberation_1771393?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:33:05 [liberation] DEBUG: Saved file Liberation #19.json\n",
      "2019-12-31 08:33:06 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://immobilier.lefigaro.fr/annonces/edito/louer/je-prepare-mon-dossier/etablir-etat-des-lieux> (referer: None)\n",
      "2019-12-31 08:33:06 [scrapy.core.scraper] ERROR: Spider error processing <GET https://immobilier.lefigaro.fr/annonces/edito/louer/je-prepare-mon-dossier/etablir-etat-des-lieux> (referer: None)\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/processors.py\", line 59, in __call__\n",
      "    value = func(value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/newspaper_crawler/loaders/lefigaro_loader.py\", line 12, in clean_date\n",
      "    return re.search(\"[0-9]+/[0-9]+/[0-9]+\", rawdata).group(0)\n",
      "AttributeError: 'NoneType' object has no attribute 'group'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 159, in _process_input_value\n",
      "    return proc(value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/processors.py\", line 63, in __call__\n",
      "    (str(func), value, type(e).__name__, str(e)))\n",
      "ValueError: Error in Compose with <function LeFigaroLoader.clean_date at 0x1054fa048> value='Monday, December 30, 2019' error='AttributeError: 'NoneType' object has no attribute 'group''\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/twisted/internet/defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/newspaper_crawler/spiders/lefigaro_spider.py\", line 18, in parse\n",
      "    loader.add_xpath(\"date\", \"//time//text()\")\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 181, in add_xpath\n",
      "    self.add_value(field_name, values, *processors, **kw)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 78, in add_value\n",
      "    self._add_value(field_name, value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 92, in _add_value\n",
      "    processed_value = self._process_input_value(field_name, value)\n",
      "  File \"/anaconda3/lib/python3.7/site-packages/scrapy/loader/__init__.py\", line 164, in _process_input_value\n",
      "    value, type(e).__name__, str(e)))\n",
      "ValueError: Error with input processor Compose: field='date' value=['Monday, December 30, 2019'] error='ValueError: Error in Compose with <function LeFigaroLoader.clean_date at 0x1054fa048> value='Monday, December 30, 2019' error='AttributeError: 'NoneType' object has no attribute 'group'''\n",
      "2019-12-31 08:33:07 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/sciences/l-hiver-polaire-une-epreuve-d-isolement-extreme-qui-affecte-le-mental-20191230> (referer: None)\n",
      "2019-12-31 08:33:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/france/2019/12/30/femmes-voilees-pas-vues-a-la-tele-la-reponse-ecrite-a-40-mains_1771390?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:33:08 [liberation] DEBUG: Saved file Liberation #20.json\n",
      "2019-12-31 08:33:09 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/economie/carlos-ghosn-aurait-quitte-le-japon-pour-beyrouth-20191230> (referer: None)\n",
      "2019-12-31 08:33:10 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/direct/element/lassociation-ni-putes-ni-soumises-expulsee-de-ses-locaux-a-montreuil_107128/> (referer: None)\n",
      "2019-12-31 08:33:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/societes/le-japon-reste-sourd-aux-critiques-de-la-defense-de-carlos-ghosn-20191119> (referer: None)\n",
      "2019-12-31 08:33:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liberation.fr/direct/element/lassociation-ni-putes-ni-soumises-expulsee-de-ses-locaux-a-montreuil_107128/?xtor=rss-450> (referer: None)\n",
      "2019-12-31 08:33:13 [scrapy.core.engine] INFO: Closing spider (finished)\n",
      "2019-12-31 08:33:13 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 11809,\n",
      " 'downloader/request_count': 36,\n",
      " 'downloader/request_method_count/GET': 36,\n",
      " 'downloader/response_bytes': 1389605,\n",
      " 'downloader/response_count': 36,\n",
      " 'downloader/response_status_count/200': 35,\n",
      " 'downloader/response_status_count/301': 1,\n",
      " 'elapsed_time_seconds': 117.977945,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2019, 12, 30, 23, 33, 13, 609161),\n",
      " 'log_count/DEBUG': 137,\n",
      " 'log_count/ERROR': 3,\n",
      " 'log_count/INFO': 41,\n",
      " 'memusage/max': 77438976,\n",
      " 'memusage/startup': 77438976,\n",
      " 'response_received_count': 35,\n",
      " 'scheduler/dequeued': 36,\n",
      " 'scheduler/dequeued/memory': 36,\n",
      " 'scheduler/enqueued': 36,\n",
      " 'scheduler/enqueued/memory': 36,\n",
      " 'start_time': datetime.datetime(2019, 12, 30, 23, 31, 15, 631216)}\n",
      "2019-12-31 08:33:13 [scrapy.core.engine] INFO: Spider closed (finished)\n",
      "2019-12-31 08:33:15 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/vox/economie/bertille-bayart-et-emmanuel-egloff-la-chute-de-carlos-ghosn-a-ete-orchestree-20191107> (referer: None)\n",
      "2019-12-31 08:33:17 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://amp.lefigaro.fr/story/carlos-ghosn-du-sommet-a-la-disgrace-12111> from <GET https://www.lefigaro.fr/story/carlos-ghosn-du-sommet-a-la-disgrace-12111>\n",
      "2019-12-31 08:33:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/social/sncf-pas-d-amelioration-du-trafic-pour-la-saint-sylvestre-20191230> (referer: None)\n",
      "2019-12-31 08:33:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/conso/bouches-du-rhone-environ-150-courriers-decouverts-dans-une-poubelle-20191230> (referer: None)\n",
      "2019-12-31 08:33:23 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/social/de-l-hopital-au-cimetiere-l-argentine-ce-pays-ou-il-fait-cher-mourir-20191230> (referer: None)\n",
      "2019-12-31 08:33:26 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/conjoncture/pourquoi-bruno-le-maire-n-est-pas-plus-present-dans-les-medias-pour-defendre-la-reforme-des-retraites-20191230> (referer: None)\n",
      "2019-12-31 08:33:28 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/conjoncture/macron-presentera-fin-janvier-debut-fevrier-le-contenu-du-pacte-productif-20191230> (referer: None)\n",
      "2019-12-31 08:33:31 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/culture/retraites-les-danseurs-de-l-opera-de-paris-rejettent-la-clause-du-grand-pere-20191230> (referer: None)\n",
      "2019-12-31 08:33:34 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/culture/apres-20-jours-de-greve-les-negociations-sur-les-retraites-s-ouvrent-a-l-opera-de-paris-20191224> (referer: None)\n",
      "2019-12-31 08:33:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/cinema/greta-gerwig-sans-jo-march-je-ne-serais-ni-scenariste-ni-realisatrice-20191230> (referer: None)\n",
      "2019-12-31 08:33:38 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/cinema/les-filles-du-docteur-march-en-place-pour-le-quadrille-20191230> (referer: None)\n",
      "2019-12-31 08:33:41 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/cinema/sejour-dans-les-monts-fuchun-gu-xiaogang-cineaste-de-l-intranquillite-20191230> (referer: None)\n",
      "2019-12-31 08:33:43 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.lefigaro.fr/livres/gerard-manset-un-artiste-en-solitaire-20191230> (referer: None)\n",
      "2019-12-31 08:33:46 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://amp.lefigaro.fr/story/carlos-ghosn-du-sommet-a-la-disgrace-12111> (referer: None)\n",
      "2019-12-31 08:33:46 [scrapy.core.engine] INFO: Closing spider (finished)\n",
      "2019-12-31 08:33:46 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 16065,\n",
      " 'downloader/request_count': 55,\n",
      " 'downloader/request_method_count/GET': 55,\n",
      " 'downloader/response_bytes': 2417558,\n",
      " 'downloader/response_count': 55,\n",
      " 'downloader/response_status_count/200': 51,\n",
      " 'downloader/response_status_count/301': 4,\n",
      " 'elapsed_time_seconds': 118.892406,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2019, 12, 30, 23, 33, 46, 895396),\n",
      " 'log_count/DEBUG': 151,\n",
      " 'log_count/ERROR': 3,\n",
      " 'log_count/INFO': 35,\n",
      " 'memusage/max': 77438976,\n",
      " 'memusage/startup': 77438976,\n",
      " 'response_received_count': 51,\n",
      " 'scheduler/dequeued': 55,\n",
      " 'scheduler/dequeued/memory': 55,\n",
      " 'scheduler/enqueued': 55,\n",
      " 'scheduler/enqueued/memory': 55,\n",
      " 'spider_exceptions/ValueError': 3,\n",
      " 'start_time': datetime.datetime(2019, 12, 30, 23, 31, 48, 2990)}\n",
      "2019-12-31 08:33:46 [scrapy.core.engine] INFO: Spider closed (finished)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from newspaper_crawler import NewspaperCrawler\n",
    "\n",
    "sources_to_crawl = [\n",
    "    \"futurasciences\",\n",
    "    \"lefigaro\",\n",
    "    \"lemonde\",\n",
    "    \"liberation\",\n",
    "    \"nouvelobs\",\n",
    "    \"telerama\",\n",
    "]\n",
    "# current folder\n",
    "my_current_path = os.getcwd()\n",
    "print(\"Current Folder: \", my_current_path)\n",
    "# change to save folder\n",
    "os.chdir(PATH_FOLDER_SAVE)\n",
    "print(\"Scraping Results Folder: \", PATH_FOLDER_SAVE)\n",
    "# scraping\n",
    "crawler = NewspaperCrawler(sources_to_crawl, has_database=True)\n",
    "crawler.start_crawling()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Return Folder:  /Users/gregory/Documents/CloudStationSinchon/Applications/python/OC_IML_projet_7/code/FakeNewsDetectionFr\n"
     ]
    }
   ],
   "source": [
    "os.chdir(my_current_path)\n",
    "print(\"Return Folder: \", os.getcwd())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save json into data folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../../data'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PATH_FOLDER_SAVE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Directory '../../data/pressarticles_20191231_08_55_03' created\n"
     ]
    }
   ],
   "source": [
    "# move results to directory with date\n",
    "save_results_newpaper_crawler(PATH_FOLDER_SAVE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read DataBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-31 08:55:05 [matplotlib] DEBUG: $HOME=/Users/gregory\n",
      "2019-12-31 08:55:05 [matplotlib] DEBUG: CONFIGDIR=/Users/gregory/.matplotlib\n",
      "2019-12-31 08:55:05 [matplotlib] DEBUG: matplotlib data path: /anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data\n",
      "2019-12-31 08:55:05 [matplotlib] DEBUG: loaded rc file /anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/matplotlibrc\n",
      "2019-12-31 08:55:05 [matplotlib] DEBUG: matplotlib version 3.0.2\n",
      "2019-12-31 08:55:05 [matplotlib] DEBUG: interactive is False\n",
      "2019-12-31 08:55:05 [matplotlib] DEBUG: platform is darwin\n",
      "2019-12-31 08:55:05 [matplotlib] DEBUG: loaded modules: ['sys', 'builtins', '_frozen_importlib', '_imp', '_thread', '_warnings', '_weakref', 'zipimport', '_frozen_importlib_external', '_io', 'marshal', 'posix', 'encodings', 'codecs', '_codecs', 'encodings.aliases', 'encodings.utf_8', '_signal', '__main__', 'encodings.latin_1', 'io', 'abc', '_abc', '_bootlocale', '_locale', 'site', 'os', 'stat', '_stat', 'posixpath', 'genericpath', 'os.path', '_collections_abc', '_sitebuiltins', 'types', 'importlib', 'importlib._bootstrap', 'importlib._bootstrap_external', 'warnings', 'importlib.util', 'importlib.abc', 'importlib.machinery', 'contextlib', 'collections', 'operator', '_operator', 'keyword', 'heapq', '_heapq', 'itertools', 'reprlib', '_collections', 'functools', '_functools', 'mpl_toolkits', 'sphinxcontrib', 'zope', 'runpy', 'pkgutil', 'weakref', '_weakrefset', 'ipykernel', 'ipykernel._version', 'ipykernel.connect', '__future__', 'json', 'json.decoder', 're', 'enum', 'sre_compile', '_sre', 'sre_parse', 'sre_constants', 'copyreg', 'json.scanner', '_json', 'json.encoder', 'subprocess', 'time', 'signal', 'errno', '_posixsubprocess', 'select', 'selectors', 'collections.abc', 'math', 'threading', 'traceback', 'linecache', 'tokenize', 'token', 'IPython', 'IPython.core', 'IPython.core.getipython', 'IPython.core.release', 'IPython.core.application', 'atexit', 'copy', 'glob', 'fnmatch', 'logging', 'string', '_string', 'shutil', 'zlib', 'bz2', '_compression', '_bz2', 'lzma', '_lzma', 'pwd', 'grp', 'traitlets', 'traitlets.traitlets', 'inspect', 'dis', 'opcode', '_opcode', 'six', 'struct', '_struct', 'traitlets.utils', 'traitlets.utils.getargspec', 'traitlets.utils.importstring', 'ipython_genutils', 'ipython_genutils._version', 'ipython_genutils.py3compat', 'ipython_genutils.encoding', 'locale', 'platform', 'traitlets.utils.sentinel', 'traitlets.utils.bunch', 'traitlets._version', 'traitlets.config', 'traitlets.config.application', 'decorator', 'traitlets.config.configurable', 'traitlets.config.loader', 'argparse', 'gettext', 'ast', '_ast', 'ipython_genutils.path', 'random', 'hashlib', '_hashlib', '_blake2', '_sha3', 'bisect', '_bisect', '_random', 'ipython_genutils.text', 'textwrap', 'ipython_genutils.importstring', 'IPython.core.crashhandler', 'pprint', 'IPython.core.ultratb', 'pydoc', 'urllib', 'urllib.parse', 'IPython.core.debugger', 'bdb', 'IPython.utils', 'IPython.utils.PyColorize', 'IPython.utils.coloransi', 'IPython.utils.ipstruct', 'IPython.utils.colorable', 'pygments', 'pygments.util', 'IPython.utils.py3compat', 'IPython.utils.encoding', 'IPython.core.excolors', 'IPython.testing', 'IPython.testing.skipdoctest', 'pdb', 'cmd', 'code', 'codeop', 'IPython.core.display_trap', 'IPython.utils.path', 'IPython.utils.process', 'IPython.utils._process_posix', 'pexpect', 'pexpect.exceptions', 'pexpect.utils', 'pexpect.expect', 'pexpect.pty_spawn', 'pty', 'tty', 'termios', 'ptyprocess', 'ptyprocess.ptyprocess', 'fcntl', 'resource', 'ptyprocess.util', 'pexpect.spawnbase', 'pexpect.run', 'IPython.utils._process_common', 'shlex', 'IPython.utils.decorators', 'IPython.utils.data', 'IPython.utils.terminal', 'IPython.utils.sysinfo', 'IPython.utils._sysinfo', 'IPython.core.profiledir', 'IPython.paths', 'tempfile', 'IPython.utils.importstring', 'IPython.terminal', 'IPython.terminal.embed', 'IPython.core.compilerop', 'IPython.core.magic_arguments', 'IPython.core.error', 'IPython.utils.text', 'pathlib', 'ntpath', 'IPython.core.magic', 'getopt', 'IPython.core.oinspect', 'IPython.core.page', 'IPython.core.display', 'binascii', 'mimetypes', 'IPython.lib', 'IPython.lib.security', 'getpass', 'IPython.lib.pretty', 'datetime', '_datetime', 'IPython.utils.openpy', 'IPython.utils.dir2', 'IPython.utils.wildcard', 'pygments.lexers', 'pygments.lexers._mapping', 'pygments.modeline', 'pygments.plugin', 'pygments.lexers.python', 'pygments.lexer', 'pygments.filter', 'pygments.filters', 'pygments.token', 'pygments.regexopt', 'pygments.unistring', 'pygments.formatters', 'pygments.formatters._mapping', 'pygments.formatters.html', 'pygments.formatter', 'pygments.styles', 'IPython.core.inputtransformer2', 'typing', 'typing.io', 'typing.re', 'IPython.core.interactiveshell', 'asyncio', 'asyncio.base_events', 'concurrent', 'concurrent.futures', 'concurrent.futures._base', 'socket', '_socket', 'ssl', '_ssl', 'base64', 'asyncio.constants', 'asyncio.coroutines', 'asyncio.base_futures', 'asyncio.format_helpers', 'asyncio.log', 'asyncio.events', 'contextvars', '_contextvars', 'asyncio.base_tasks', '_asyncio', 'asyncio.futures', 'asyncio.protocols', 'asyncio.sslproto', 'asyncio.transports', 'asyncio.tasks', 'asyncio.locks', 'asyncio.runners', 'asyncio.queues', 'asyncio.streams', 'asyncio.subprocess', 'asyncio.unix_events', 'asyncio.base_subprocess', 'asyncio.selector_events', 'pickleshare', 'pickle', '_compat_pickle', '_pickle', 'IPython.core.prefilter', 'IPython.core.autocall', 'IPython.core.macro', 'IPython.core.splitinput', 'IPython.core.alias', 'IPython.core.builtin_trap', 'IPython.core.events', 'backcall', 'backcall.backcall', 'IPython.core.displayhook', 'IPython.core.displaypub', 'IPython.core.extensions', 'IPython.core.formatters', 'IPython.utils.sentinel', 'IPython.core.history', 'sqlite3', 'sqlite3.dbapi2', '_sqlite3', 'IPython.core.logger', 'IPython.core.payload', 'IPython.core.usage', 'IPython.display', 'IPython.lib.display', 'html', 'html.entities', 'IPython.utils.io', 'IPython.utils.capture', 'IPython.utils.strdispatch', 'IPython.core.hooks', 'IPython.utils.syspathcontext', 'IPython.utils.tempdir', 'IPython.utils.contexts', 'IPython.core.async_helpers', 'IPython.terminal.interactiveshell', 'prompt_toolkit', 'prompt_toolkit.application', 'prompt_toolkit.application.application', 'prompt_toolkit.buffer', 'prompt_toolkit.application.current', 'prompt_toolkit.eventloop', 'prompt_toolkit.eventloop.base', 'prompt_toolkit.log', 'prompt_toolkit.eventloop.coroutine', 'prompt_toolkit.eventloop.defaults', 'prompt_toolkit.utils', 'six.moves', 'wcwidth', 'wcwidth.wcwidth', 'wcwidth.table_wide', 'wcwidth.table_zero', 'prompt_toolkit.cache', 'prompt_toolkit.eventloop.future', 'prompt_toolkit.eventloop.context', 'prompt_toolkit.eventloop.async_generator', 'queue', '_queue', 'six.moves.queue', 'prompt_toolkit.eventloop.event', 'prompt_toolkit.application.run_in_terminal', 'prompt_toolkit.auto_suggest', 'prompt_toolkit.filters', 'prompt_toolkit.filters.base', 'prompt_toolkit.filters.app', 'prompt_toolkit.enums', 'prompt_toolkit.filters.utils', 'prompt_toolkit.filters.cli', 'prompt_toolkit.clipboard', 'prompt_toolkit.clipboard.base', 'prompt_toolkit.selection', 'prompt_toolkit.clipboard.in_memory', 'prompt_toolkit.completion', 'prompt_toolkit.completion.base', 'prompt_toolkit.completion.filesystem', 'prompt_toolkit.completion.word_completer', 'prompt_toolkit.document', 'prompt_toolkit.history', 'prompt_toolkit.search', 'prompt_toolkit.key_binding', 'prompt_toolkit.key_binding.key_bindings', 'prompt_toolkit.keys', 'prompt_toolkit.key_binding.vi_state', 'prompt_toolkit.validation', 'prompt_toolkit.input', 'prompt_toolkit.input.base', 'prompt_toolkit.input.defaults', 'prompt_toolkit.input.typeahead', 'prompt_toolkit.key_binding.bindings', 'prompt_toolkit.key_binding.bindings.page_navigation', 'prompt_toolkit.key_binding.bindings.scroll', 'prompt_toolkit.key_binding.defaults', 'prompt_toolkit.key_binding.bindings.basic', 'prompt_toolkit.key_binding.key_processor', 'prompt_toolkit.key_binding.bindings.named_commands', 'prompt_toolkit.key_binding.bindings.completion', 'prompt_toolkit.key_binding.bindings.emacs', 'prompt_toolkit.key_binding.bindings.vi', 'prompt_toolkit.input.vt100_parser', 'prompt_toolkit.input.ansi_escape_sequences', 'prompt_toolkit.key_binding.digraphs', 'prompt_toolkit.key_binding.bindings.mouse', 'prompt_toolkit.layout', 'prompt_toolkit.layout.containers', 'prompt_toolkit.layout.controls', 'prompt_toolkit.formatted_text', 'prompt_toolkit.formatted_text.base', 'prompt_toolkit.formatted_text.html', 'xml', 'xml.dom', 'xml.dom.domreg', 'xml.dom.minidom', 'xml.dom.minicompat', 'xml.dom.xmlbuilder', 'xml.dom.NodeFilter', 'prompt_toolkit.formatted_text.ansi', 'prompt_toolkit.output', 'prompt_toolkit.output.base', 'prompt_toolkit.layout.screen', 'prompt_toolkit.output.defaults', 'prompt_toolkit.output.color_depth', 'prompt_toolkit.output.vt100', 'prompt_toolkit.styles', 'prompt_toolkit.styles.base', 'prompt_toolkit.styles.defaults', 'prompt_toolkit.styles.style', 'prompt_toolkit.styles.named_colors', 'prompt_toolkit.styles.pygments', 'prompt_toolkit.styles.style_transformation', 'colorsys', 'array', 'prompt_toolkit.formatted_text.pygments', 'prompt_toolkit.formatted_text.utils', 'prompt_toolkit.lexers', 'prompt_toolkit.lexers.base', 'prompt_toolkit.lexers.pygments', 'prompt_toolkit.mouse_events', 'prompt_toolkit.layout.processors', 'prompt_toolkit.layout.utils', 'prompt_toolkit.layout.dimension', 'prompt_toolkit.layout.margins', 'prompt_toolkit.layout.layout', 'prompt_toolkit.layout.menus', 'prompt_toolkit.renderer', 'prompt_toolkit.layout.mouse_handlers', 'prompt_toolkit.key_binding.bindings.cpr', 'prompt_toolkit.key_binding.emacs_state', 'prompt_toolkit.layout.dummy', 'prompt_toolkit.application.dummy', 'prompt_toolkit.shortcuts', 'prompt_toolkit.shortcuts.dialogs', 'prompt_toolkit.key_binding.bindings.focus', 'prompt_toolkit.widgets', 'prompt_toolkit.widgets.base', 'prompt_toolkit.widgets.toolbars', 'prompt_toolkit.widgets.dialogs', 'prompt_toolkit.widgets.menus', 'prompt_toolkit.shortcuts.prompt', 'prompt_toolkit.key_binding.bindings.auto_suggest', 'prompt_toolkit.key_binding.bindings.open_in_editor', 'prompt_toolkit.shortcuts.utils', 'prompt_toolkit.shortcuts.progress_bar', 'prompt_toolkit.shortcuts.progress_bar.base', 'prompt_toolkit.shortcuts.progress_bar.formatters', 'prompt_toolkit.patch_stdout', 'pygments.style', 'IPython.terminal.debugger', 'IPython.core.completer', 'unicodedata', 'IPython.core.latex_symbols', 'IPython.utils.generics', 'jedi', 'jedi.api', 'parso', 'parso.parser', 'parso.tree', 'parso._compatibility', 'parso.pgen2', 'parso.pgen2.generator', 'parso.pgen2.grammar_parser', 'parso.python', 'parso.python.tokenize', 'parso.python.token', 'parso.utils', 'parso.grammar', 'parso.python.diff', 'difflib', 'parso.python.parser', 'parso.python.tree', 'parso.python.prefix', 'parso.cache', 'gc', 'parso.python.errors', 'parso.normalizer', 'parso.python.pep8', 'jedi._compatibility', 'jedi.parser_utils', 'jedi.debug', 'colorama', 'colorama.initialise', 'colorama.ansitowin32', 'colorama.ansi', 'colorama.winterm', 'colorama.win32', 'ctypes', '_ctypes', 'ctypes._endian', 'jedi.settings', 'jedi.cache', 'jedi.api.classes', 'jedi.evaluate', 'jedi.evaluate.utils', 'jedi.evaluate.imports', 'jedi.evaluate.sys_path', 'jedi.evaluate.cache', 'jedi.evaluate.base_context', 'jedi.common', 'jedi.common.context', 'jedi.evaluate.helpers', 'jedi.common.utils', 'jedi.evaluate.compiled', 'jedi.evaluate.compiled.context', 'jedi.evaluate.filters', 'jedi.evaluate.flow_analysis', 'jedi.evaluate.recursion', 'jedi.evaluate.lazy_context', 'jedi.evaluate.compiled.access', 'jedi.evaluate.compiled.getattr_static', 'jedi.evaluate.compiled.fake', 'jedi.evaluate.analysis', 'jedi.evaluate.context', 'jedi.evaluate.context.module', 'jedi.evaluate.context.klass', 'jedi.evaluate.context.function', 'jedi.evaluate.docstrings', 'jedi.evaluate.pep0484', 'jedi.evaluate.arguments', 'jedi.evaluate.context.iterable', 'jedi.evaluate.param', 'jedi.evaluate.context.asynchronous', 'jedi.evaluate.parser_cache', 'jedi.evaluate.context.instance', 'jedi.evaluate.syntax_tree', 'jedi.evaluate.finder', 'jedi.api.keywords', 'pydoc_data', 'pydoc_data.topics', 'jedi.api.interpreter', 'jedi.evaluate.compiled.mixed', 'jedi.api.helpers', 'jedi.api.completion', 'jedi.api.environment', 'filecmp', 'jedi.evaluate.compiled.subprocess', 'jedi.evaluate.compiled.subprocess.functions', 'jedi.api.exceptions', 'jedi.api.project', 'jedi.evaluate.usages', 'IPython.terminal.ptutils', 'IPython.terminal.shortcuts', 'IPython.terminal.magics', 'IPython.lib.clipboard', 'IPython.terminal.pt_inputhooks', 'IPython.terminal.prompts', 'IPython.terminal.ipapp', 'IPython.core.magics', 'IPython.core.magics.auto', 'IPython.core.magics.basic', 'IPython.core.magics.code', 'urllib.request', 'email', 'http', 'http.client', 'email.parser', 'email.feedparser', 'email.errors', 'email._policybase', 'email.header', 'email.quoprimime', 'email.base64mime', 'email.charset', 'email.encoders', 'quopri', 'email.utils', 'email._parseaddr', 'calendar', 'email.message', 'uu', 'email._encoded_words', 'email.iterators', 'urllib.error', 'urllib.response', '_scproxy', 'IPython.core.magics.config', 'IPython.core.magics.display', 'IPython.core.magics.execution', 'timeit', 'cProfile', '_lsprof', 'profile', 'pstats', 'IPython.utils.module_paths', 'IPython.utils.timing', 'IPython.core.magics.extension', 'IPython.core.magics.history', 'IPython.core.magics.logging', 'IPython.core.magics.namespace', 'IPython.core.magics.osm', 'IPython.core.magics.pylab', 'IPython.core.pylabtools', 'IPython.core.magics.script', 'IPython.lib.backgroundjobs', 'IPython.core.shellapp', 'IPython.extensions', 'IPython.extensions.storemagic', 'IPython.utils.frame', 'jupyter_client', 'jupyter_client._version', 'jupyter_client.connect', 'zmq', 'zmq.backend', 'zmq.backend.select', 'zmq.backend.cython', 'cython_runtime', 'zmq.backend.cython.constants', '_cython_0_28_5', 'zmq.backend.cython.error', 'zmq.backend.cython.message', 'zmq.error', 'zmq.backend.cython.context', 'zmq.backend.cython.socket', 'zmq.backend.cython.utils', 'zmq.backend.cython._poll', 'zmq.backend.cython._version', 'zmq.backend.cython._device', 'zmq.sugar', 'zmq.sugar.constants', 'zmq.utils', 'zmq.utils.constant_names', 'zmq.sugar.context', 'zmq.sugar.attrsettr', 'zmq.sugar.socket', 'zmq.sugar.poll', 'zmq.utils.jsonapi', 'zmq.utils.strtypes', 'zmq.sugar.frame', 'zmq.sugar.tracker', 'zmq.sugar.version', 'zmq.sugar.stopwatch', 'jupyter_client.localinterfaces', 'jupyter_core', 'jupyter_core.version', 'jupyter_core.paths', 'jupyter_client.launcher', 'traitlets.log', 'jupyter_client.client', 'jupyter_client.channels', 'jupyter_client.channelsabc', 'jupyter_client.clientabc', 'jupyter_client.manager', 'jupyter_client.kernelspec', 'jupyter_client.managerabc', 'jupyter_client.blocking', 'jupyter_client.blocking.client', 'jupyter_client.blocking.channels', 'jupyter_client.multikernelmanager', 'uuid', '_uuid', 'ipykernel.kernelapp', 'tornado', 'tornado.ioloop', 'numbers', 'tornado.concurrent', 'tornado.log', 'logging.handlers', 'tornado.escape', 'tornado.util', 'tornado.speedups', 'curses', '_curses', 'tornado.stack_context', 'tornado.platform', 'tornado.platform.auto', 'tornado.platform.posix', 'tornado.platform.common', 'tornado.platform.interface', 'concurrent.futures.thread', 'zmq.eventloop', 'zmq.eventloop.ioloop', 'tornado.platform.asyncio', 'tornado.gen', 'zmq.eventloop.zmqstream', 'ipykernel.iostream', 'imp', 'jupyter_client.session', 'hmac', 'jupyter_client.jsonutil', 'dateutil', 'dateutil._version', 'dateutil.parser', 'dateutil.parser._parser', 'decimal', '_decimal', 'dateutil.relativedelta', 'dateutil._common', 'dateutil.tz', 'dateutil.tz.tz', 'dateutil.tz._common', 'dateutil.tz._factories', 'dateutil.parser.isoparser', '_strptime', 'jupyter_client.adapter', 'ipykernel.heartbeat', 'ipykernel.ipkernel', 'IPython.utils.tokenutil', 'ipykernel.comm', 'ipykernel.comm.manager', 'ipykernel.comm.comm', 'ipykernel.kernelbase', 'tornado.queues', 'tornado.locks', 'ipykernel.jsonutil', 'ipykernel.zmqshell', 'IPython.core.payloadpage', 'ipykernel.displayhook', 'ipykernel.parentpoller', 'faulthandler', 'ipykernel.datapub', 'ipykernel.serialize', 'ipykernel.pickleutil', 'ipykernel.codeutil', 'IPython.core.completerlib', 'storemagic', 'ipywidgets', 'ipywidgets._version', 'ipywidgets.widgets', 'ipywidgets.widgets.widget', 'ipywidgets.widgets.domwidget', 'ipywidgets.widgets.trait_types', 'ipywidgets.widgets.widget_layout', 'ipywidgets.widgets.widget_style', 'ipywidgets.widgets.valuewidget', 'ipywidgets.widgets.widget_core', 'ipywidgets.widgets.widget_bool', 'ipywidgets.widgets.widget_description', 'ipywidgets.widgets.widget_button', 'ipywidgets.widgets.widget_box', 'ipywidgets.widgets.docutils', 'ipywidgets.widgets.widget_float', 'ipywidgets.widgets.widget_int', 'ipywidgets.widgets.widget_color', 'ipywidgets.widgets.widget_date', 'ipywidgets.widgets.widget_output', 'ipywidgets.widgets.widget_selection', 'ipywidgets.widgets.widget_selectioncontainer', 'ipywidgets.widgets.widget_string', 'ipywidgets.widgets.widget_controller', 'ipywidgets.widgets.interaction', 'ipywidgets.widgets.widget_link', 'ipywidgets.widgets.widget_media', 'autopep8', 'pycodestyle', 'optparse', 'configparser', 'newspaper_crawler', 'newspaper_crawler.crawler', 'newspaper_crawler.crawling_jobs', 'newspaper_crawler.crawling_jobs.futurasciences_crawling_job', 'newspaper_crawler.crawling_jobs.base_crawling_job', 'feedparser', 'cgi', 'gzip', 'xml.sax', 'xml.sax.xmlreader', 'xml.sax.handler', 'xml.sax._exceptions', 'xml.sax.saxutils', 'xml.sax.expatreader', 'xml.parsers', 'xml.parsers.expat', 'pyexpat.errors', 'pyexpat.model', 'pyexpat', 'xml.parsers.expat.model', 'xml.parsers.expat.errors', 'chardet', 'chardet.compat', 'chardet.universaldetector', 'chardet.charsetgroupprober', 'chardet.enums', 'chardet.charsetprober', 'chardet.escprober', 'chardet.codingstatemachine', 'chardet.escsm', 'chardet.latin1prober', 'chardet.mbcsgroupprober', 'chardet.utf8prober', 'chardet.mbcssm', 'chardet.sjisprober', 'chardet.mbcharsetprober', 'chardet.chardistribution', 'chardet.euctwfreq', 'chardet.euckrfreq', 'chardet.gb2312freq', 'chardet.big5freq', 'chardet.jisfreq', 'chardet.jpcntx', 'chardet.eucjpprober', 'chardet.gb2312prober', 'chardet.euckrprober', 'chardet.cp949prober', 'chardet.big5prober', 'chardet.euctwprober', 'chardet.sbcsgroupprober', 'chardet.sbcharsetprober', 'chardet.langcyrillicmodel', 'chardet.langgreekmodel', 'chardet.langbulgarianmodel', 'chardet.langthaimodel', 'chardet.langhebrewmodel', 'chardet.hebrewprober', 'chardet.langturkishmodel', 'chardet.version', 'scrapy', 'scrapy._monkeypatches', 'twisted', 'twisted._version', 'incremental', 'incremental._version', 'twisted.persisted', 'twisted.persisted.styles', 'twisted.python', 'twisted.python.compat', 'http.cookiejar', 'twisted.python.versions', 'twisted.python.deprecate', 'twisted.python.log', 'zope.interface', 'zope.interface.interface', 'zope.interface.exceptions', 'zope.interface.ro', 'zope.interface._zope_interface_coptimizations', 'zope.interface.declarations', 'zope.interface.advice', 'zope.interface._compat', 'zope.interface.interfaces', 'twisted.python.context', 'twisted.python._oldstyle', 'twisted.python.reflect', 'twisted.python.util', 'twisted.python.failure', 'twisted.python.threadable', 'twisted.logger', 'twisted.logger._levels', 'constantly', 'constantly._constants', 'constantly._version', 'twisted.logger._flatten', 'twisted.logger._format', 'twisted.python._tzhelper', 'twisted.logger._logger', 'twisted.logger._global', 'twisted.logger._buffer', 'twisted.logger._observer', 'twisted.logger._filter', 'twisted.logger._io', 'twisted.logger._file', 'twisted.logger._stdlib', 'twisted.logger._legacy', 'twisted.logger._json', 'twisted.logger._capture', 'scrapy.spiders', 'scrapy.signals', 'scrapy.http', 'scrapy.http.headers', 'w3lib', 'w3lib.http', 'scrapy.utils', 'scrapy.utils.datatypes', 'scrapy.exceptions', 'scrapy.utils.python', 'scrapy.utils.decorators', 'twisted.internet', 'twisted.internet.defer', 'attr', 'attr.converters', 'attr._make', 'attr._config', 'attr._compat', 'attr.exceptions', 'attr.filters', 'attr.validators', 'attr._funcs', 'twisted.python.lockfile', 'twisted.python.runtime', 'twisted.internet.threads', 'scrapy.http.request', 'w3lib.url', 'six.moves.urllib', 'six.moves.urllib.parse', 'six.moves.urllib.request', 'w3lib.util', 'scrapy.utils.trackref', 'scrapy.utils.url', 'scrapy.http.common', 'scrapy.utils.curl', 'http.cookies', 'six.moves.http_cookies', 'scrapy.http.request.form', 'lxml', 'lxml.html', 'lxml.etree', 'lxml._elementpath', 'lxml.html.defs', 'lxml.html._setmixin', 'parsel', 'parsel.selector', 'parsel.utils', 'w3lib.html', 'parsel.csstranslator', 'cssselect', 'cssselect.parser', 'cssselect.xpath', 'parsel.xpathfuncs', 'scrapy.utils.response', 'webbrowser', 'twisted.web', 'twisted.web.http', 'twisted.python.components', 'zope.interface.adapter', 'twisted.internet.interfaces', 'twisted.internet.protocol', 'twisted.internet.error', 'twisted.internet.address', 'twisted.python.filepath', 'twisted.python.win32', 'twisted.internet._producer_helpers', 'twisted.internet.task', 'twisted.internet.base', 'twisted.internet.fdesc', 'twisted.internet.main', 'twisted.internet.abstract', 'twisted.internet._resolver', 'twisted.internet._idna', 'twisted.protocols', 'twisted.protocols.policies', 'twisted.protocols.basic', 'twisted.web.iweb', 'twisted.cred', 'twisted.cred.credentials', 'twisted.python.randbytes', 'twisted.cred._digest', 'twisted.cred.error', 'twisted.web.http_headers', 'twisted.web._responses', 'scrapy.http.request.rpc', 'xmlrpc', 'xmlrpc.client', 'scrapy.http.request.json_request', 'scrapy.utils.deprecate', 'scrapy.http.response', 'scrapy.link', 'scrapy.http.response.html', 'scrapy.http.response.text', 'w3lib.encoding', 'scrapy.http.response.xml', 'scrapy.spiders.crawl', 'scrapy.linkextractors', 'scrapy.utils.misc', 'scrapy.item', 'scrapy.linkextractors.lxmlhtml', 'scrapy.utils.spider', 'scrapy.spiders.feed', 'scrapy.utils.iterators', 'csv', '_csv', 'scrapy.selector', 'scrapy.selector.unified', 'scrapy.spiders.sitemap', 'scrapy.utils.sitemap', 'scrapy.utils.gz', 'scrapy.crawler', 'twisted.internet.default', 'twisted.internet.selectreactor', 'twisted.internet.posixbase', 'twisted.internet.udp', 'twisted.internet.tcp', 'twisted.internet._newtls', 'twisted.protocols.tls', 'OpenSSL', 'OpenSSL.crypto', 'cryptography', 'cryptography.__about__', 'cryptography.x509', 'cryptography.x509.certificate_transparency', 'cryptography.x509.base', 'cryptography.utils', 'cryptography.hazmat', 'cryptography.hazmat.primitives', 'cryptography.hazmat.primitives.asymmetric', 'cryptography.hazmat.primitives.asymmetric.dsa', 'cryptography.hazmat.primitives.asymmetric.ec', 'cryptography.hazmat._oid', 'cryptography.hazmat.primitives.asymmetric.rsa', 'cryptography.exceptions', 'cryptography.hazmat.backends', 'cryptography.hazmat.backends.interfaces', 'cryptography.x509.extensions', 'ipaddress', 'asn1crypto', 'asn1crypto.version', 'asn1crypto.keys', 'asn1crypto._elliptic_curve', 'asn1crypto._int', 'asn1crypto.util', 'asn1crypto._errors', 'asn1crypto._iri', 'encodings.idna', 'stringprep', 'asn1crypto._types', 'asn1crypto._ordereddict', 'asn1crypto._ffi', 'asn1crypto._perf', 'asn1crypto._perf._big_num_ctypes', 'ctypes.util', 'ctypes.macholib', 'ctypes.macholib.dyld', 'ctypes.macholib.framework', 'ctypes.macholib.dylib', 'asn1crypto.algos', 'asn1crypto.core', 'asn1crypto._teletex_codec', 'asn1crypto.parser', 'cryptography.hazmat.primitives.constant_time', 'cryptography.hazmat.bindings', '_cffi_backend', '_constant_time.lib', '_constant_time', 'cryptography.hazmat.bindings._constant_time', 'cryptography.hazmat.primitives.serialization', 'cryptography.x509.general_name', 'idna', 'idna.package_data', 'idna.core', 'idna.idnadata', 'idna.intranges', 'cryptography.x509.name', 'cryptography.x509.oid', 'cryptography.hazmat.primitives.hashes', 'OpenSSL._util', 'cryptography.hazmat.bindings.openssl', 'cryptography.hazmat.bindings.openssl.binding', '_openssl.lib', '_openssl', 'cryptography.hazmat.bindings._openssl', 'cryptography.hazmat.bindings.openssl._conditional', 'OpenSSL.SSL', 'OpenSSL.version', 'twisted.internet._sslverify', 'service_identity', 'service_identity.cryptography', 'pyasn1', 'pyasn1.codec', 'pyasn1.codec.der', 'pyasn1.codec.der.decoder', 'pyasn1.codec.cer', 'pyasn1.codec.cer.decoder', 'pyasn1.error', 'pyasn1.codec.ber', 'pyasn1.codec.ber.decoder', 'pyasn1.debug', 'pyasn1.compat', 'pyasn1.compat.octets', 'pyasn1.codec.ber.eoo', 'pyasn1.type', 'pyasn1.type.base', 'pyasn1.compat.calling', 'pyasn1.type.constraint', 'pyasn1.type.error', 'pyasn1.type.tag', 'pyasn1.type.tagmap', 'pyasn1.compat.integer', 'pyasn1.type.char', 'pyasn1.type.univ', 'pyasn1.compat.binary', 'pyasn1.type.namedtype', 'pyasn1.type.namedval', 'pyasn1.type.useful', 'pyasn1.compat.dateandtime', 'pyasn1.compat.string', 'service_identity._common', 'service_identity._compat', 'service_identity.exceptions', 'service_identity.pyopenssl', 'pyasn1_modules', 'pyasn1_modules.rfc2459', 'pyasn1.type.opentype', 'twisted.internet.unix', 'twisted.python.sendmsg', 'twisted.internet.process', 'twisted.internet._baseprocess', 'twisted.internet._signals', 'twisted.internet.reactor', 'zope.interface.verify', 'scrapy.core', 'scrapy.core.engine', 'scrapy.core.scraper', 'scrapy.utils.defer', 'scrapy.utils.log', 'logging.config', 'socketserver', 'scrapy.settings', 'scrapy.settings.default_settings', 'scrapy.utils.versions', 'scrapy.utils.ssl', 'scrapy.core.spidermw', 'scrapy.middleware', 'scrapy.utils.conf', 'scrapy.utils.request', 'scrapy.utils.httpobj', 'scrapy.utils.reactor', 'scrapy.resolver', 'scrapy.interfaces', 'scrapy.extension', 'scrapy.signalmanager', 'pydispatch', 'pydispatch.dispatcher', 'pydispatch.saferef', 'pydispatch.robustapply', 'pydispatch.errors', 'scrapy.utils.signal', 'scrapy.utils.ossignal', 'newspaper_crawler.database', 'newspaper_crawler.spiders', 'newspaper_crawler.spiders.futurasciences_spider', 'newspaper_crawler.loaders', 'newspaper_crawler.loaders.futurasciences_loader', 'scrapy.loader', 'scrapy.loader.common', 'scrapy.loader.processors', 'newspaper_crawler.loaders.base_loader', 'newspaper_crawler.loaders.liberation_loader', 'newspaper_crawler.loaders.nouvelobs_loader', 'newspaper_crawler.loaders.telerama_loader', 'newspaper_crawler.loaders.lefigaro_loader', 'newspaper_crawler.loaders.lemonde_loader', 'newspaper_crawler.spiders.base_spider', 'newspaper_crawler.items', 'newspaper_crawler.spiders.liberation_spider', 'newspaper_crawler.spiders.nouvelobs_spider', 'newspaper_crawler.spiders.telerama_spider', 'newspaper_crawler.spiders.lefigaro_spider', 'newspaper_crawler.spiders.lemonde_spider', 'newspaper_crawler.crawling_jobs.liberation_crawling_job', 'newspaper_crawler.crawling_jobs.nouvelobs_crawling_job', 'newspaper_crawler.crawling_jobs.telerama_crawling_job', 'newspaper_crawler.crawling_jobs.lefigaro_crawling_job', 'newspaper_crawler.crawling_jobs.lemonde_crawling_job', 'scrapy.spiderloader', 'scrapy.statscollectors', 'scrapy.logformatter', 'scrapy.extensions', 'scrapy.extensions.corestats', 'scrapy.extensions.telnet', 'twisted.conch', 'twisted.conch.manhole', 'twisted.conch.recvline', 'twisted.conch.insults', 'twisted.conch.insults.insults', 'twisted.conch.insults.helper', 'twisted.python._textattributes', 'twisted.python.htmlizer', 'twisted.conch.telnet', 'scrapy.utils.engine', 'scrapy.extensions.memusage', 'scrapy.mail', 'email.mime', 'email.mime.multipart', 'email.mime.base', 'email.policy', 'email.headerregistry', 'email._header_value_parser', 'email.contentmanager', 'six.moves.email_mime_multipart', 'email.mime.text', 'email.mime.nonmultipart', 'six.moves.email_mime_text', 'six.moves.email_mime_base', 'twisted.internet.ssl', 'scrapy.extensions.memdebug', 'scrapy.extensions.closespider', 'scrapy.extensions.feedexport', 'ftplib', 'scrapy.utils.ftp', 'scrapy.utils.boto', 'scrapy.extensions.logstats', 'scrapy.extensions.spiderstate', 'scrapy.utils.job', 'scrapy.extensions.throttle', 'scrapy.core.scheduler', 'queuelib', 'queuelib.queue', 'queuelib.pqueue', 'queuelib.rrqueue', 'scrapy.core.downloader', 'scrapy.core.downloader.middleware', 'scrapy.core.downloader.handlers', 'scrapy.core.downloader.handlers.datauri', 'scrapy.responsetypes', 'scrapy.core.downloader.handlers.file', 'scrapy.core.downloader.handlers.http', 'scrapy.core.downloader.handlers.http10', 'scrapy.core.downloader.handlers.http11', 'twisted.web.client', 'twisted.internet.endpoints', 'twisted.internet.stdio', 'twisted.internet._posixstdio', 'twisted.plugin', 'twisted.python.modules', 'twisted.python.zippath', 'zipfile', 'twisted.python.systemd', 'twisted.web.error', 'twisted.web._newclient', 'scrapy.core.downloader.webclient', 'scrapy.core.downloader.tls', 'scrapy.core.downloader.contextfactory', 'scrapy.core.downloader.handlers.s3', 'scrapy.core.downloader.handlers.ftp', 'twisted.protocols.ftp', 'twisted.copyright', 'twisted.cred.portal', 'twisted.cred.checkers', 'scrapy.downloadermiddlewares', 'scrapy.downloadermiddlewares.robotstxt', 'scrapy.downloadermiddlewares.httpauth', 'scrapy.downloadermiddlewares.downloadtimeout', 'scrapy.downloadermiddlewares.defaultheaders', 'scrapy.downloadermiddlewares.useragent', 'scrapy.downloadermiddlewares.retry', 'scrapy.downloadermiddlewares.ajaxcrawl', 'scrapy.downloadermiddlewares.redirect', 'scrapy.downloadermiddlewares.httpcompression', 'scrapy.downloadermiddlewares.cookies', 'scrapy.http.cookies', 'six.moves.http_cookiejar', 'scrapy.downloadermiddlewares.httpproxy', 'scrapy.downloadermiddlewares.stats', 'scrapy.downloadermiddlewares.httpcache', 'scrapy.spidermiddlewares', 'scrapy.spidermiddlewares.httperror', 'scrapy.spidermiddlewares.offsite', 'scrapy.spidermiddlewares.referer', 'scrapy.spidermiddlewares.urllength', 'scrapy.spidermiddlewares.depth', 'scrapy.pipelines', 'scrapy.dupefilters', 'scrapy.pqueues', 'scrapy.utils.reqser', 'scrapy.squeues', 'encodings.charmap', 'twisted.python.threadpool', 'twisted._threads', 'twisted._threads._threadworker', 'twisted._threads._ithreads', 'twisted._threads._convenience', 'twisted._threads._team', 'twisted._threads._memory', 'twisted._threads._pool', 'encodings.cp1252', 'pandas', 'numpy', 'numpy._globals', 'numpy.__config__', 'numpy.version', 'numpy._import_tools', 'numpy.add_newdocs', 'numpy.lib', 'numpy.lib.info', 'numpy.lib.type_check', 'numpy.core', 'numpy.core.info', 'numpy.core.multiarray', 'numpy.core.umath', 'numpy.core._internal', 'numpy.compat', 'numpy.compat._inspect', 'numpy.compat.py3k', 'numpy.core.numerictypes', 'numpy.core.numeric', 'numpy.core.fromnumeric', 'numpy.core._methods', 'numpy.core.arrayprint', 'numpy.core.defchararray', 'numpy.core.records', 'numpy.core.memmap', 'numpy.core.function_base', 'numpy.core.machar', 'numpy.core.getlimits', 'numpy.core.shape_base', 'numpy.core.einsumfunc', 'numpy.testing', 'unittest', 'unittest.result', 'unittest.util', 'unittest.case', 'unittest.suite', 'unittest.loader', 'unittest.main', 'unittest.runner', 'unittest.signals', 'numpy.testing._private', 'numpy.testing._private.utils', 'numpy.lib.utils', 'numpy.testing._private.decorators', 'numpy.testing._private.nosetester', 'numpy.testing._private.pytesttester', 'numpy.lib.ufunclike', 'numpy.lib.index_tricks', 'numpy.lib.function_base', 'numpy.lib.twodim_base', 'numpy.lib.histograms', 'numpy.matrixlib', 'numpy.matrixlib.defmatrix', 'numpy.linalg', 'numpy.linalg.info', 'numpy.linalg.linalg', 'numpy.linalg.lapack_lite', 'numpy.linalg._umath_linalg', 'numpy.lib.stride_tricks', 'numpy.lib.mixins', 'numpy.lib.nanfunctions', 'numpy.lib.shape_base', 'numpy.lib.scimath', 'numpy.lib.polynomial', 'numpy.lib.arraysetops', 'numpy.lib.npyio', 'numpy.lib.format', 'numpy.lib._datasource', 'numpy.lib._iotools', 'numpy.lib.financial', 'numpy.lib.arrayterator', 'numpy.lib.arraypad', 'numpy.lib._version', 'numpy.core._multiarray_tests', 'numpy._distributor_init', 'numpy._mklinit', 'numpy.fft', 'numpy.fft.info', 'numpy.fft.fftpack', 'numpy.fft.fftpack_lite', 'numpy.fft.helper', 'mkl_fft', 'mkl_fft._pydfti', '_cython_0_29', 'mkl_fft._version', 'mkl_fft._numpy_fft', 'numpy.polynomial', 'numpy.polynomial.polynomial', 'numpy.polynomial.polyutils', 'numpy.polynomial._polybase', 'numpy.polynomial.chebyshev', 'numpy.polynomial.legendre', 'numpy.polynomial.hermite', 'numpy.polynomial.hermite_e', 'numpy.polynomial.laguerre', 'numpy.random', 'numpy.random.info', 'numpy.random.mtrand', 'mtrand', 'numpy.ctypeslib', 'numpy.ma', 'numpy.ma.core', 'numpy.ma.extras', 'pytz', 'pytz.exceptions', 'pytz.lazy', 'pytz.tzinfo', 'pytz.tzfile', 'pandas.compat', 'distutils', 'distutils.version', 'pandas.compat.chainmap', 'pandas.compat.numpy', 'pandas._libs', 'pandas._libs.tslib', 'pandas._libs.tslibs', 'pandas._libs.tslibs.conversion', 'pandas._libs.tslibs.np_datetime', '_cython_0_28_4', 'pandas._libs.tslibs.nattype', 'pandas._libs.tslibs.timedeltas', 'pandas._libs.tslibs.timezones', 'pandas._libs.tslibs.parsing', 'pandas._libs.tslibs.ccalendar', 'pandas._libs.tslibs.strptime', 'pandas._libs.tslibs.timestamps', 'pandas._libs.tslibs.fields', 'pandas._libs.hashtable', 'pandas._libs.missing', 'pandas._libs.lib', 'pandas.core', 'pandas.core.config_init', 'pandas.core.config', 'pandas.io', 'pandas.io.formats', 'pandas.io.formats.printing', 'pandas.core.dtypes', 'pandas.core.dtypes.inference', 'pandas.io.formats.console', 'pandas.io.formats.terminal', 'pandas.core.api', 'pandas.core.algorithms', 'pandas.core.dtypes.cast', 'pandas.core.dtypes.common', 'pandas._libs.algos', 'pandas.core.dtypes.dtypes', 'pandas.core.dtypes.generic', 'pandas.core.dtypes.base', 'pandas.errors', 'pandas.core.dtypes.missing', 'pandas.core.common', 'pandas.util', 'pandas.util._decorators', 'pandas._libs.properties', 'pandas.core.util', 'pandas.core.util.hashing', 'pandas._libs.hashing', 'pandas.core.arrays', 'pandas.core.arrays.base', 'pandas.compat.numpy.function', 'pandas.util._validators', 'pandas.core.arrays.categorical', 'pandas.core.accessor', 'pandas.core.base', 'pandas.core.nanops', 'bottleneck', 'bottleneck.reduce', 'bottleneck.nonreduce', 'bottleneck.nonreduce_axis', 'bottleneck.move', 'bottleneck.slow', 'bottleneck.slow.reduce', 'bottleneck.slow.nonreduce', 'bottleneck.slow.nonreduce_axis', 'bottleneck.slow.move', 'bottleneck.version', 'bottleneck.benchmark', 'bottleneck.benchmark.bench', 'bottleneck.benchmark.autotimeit', 'bottleneck.benchmark.bench_detailed', 'bottleneck.tests', 'bottleneck.tests.util', 'pandas.core.missing', 'pandas.core.groupby', 'pandas.core.groupby.groupby', 'pandas.core.index', 'pandas.core.indexes', 'pandas.core.indexes.api', 'pandas.core.indexes.base', 'pandas._libs.index', 'pandas._libs.tslibs.period', 'pandas._libs.tslibs.frequencies', 'pandas._libs.tslibs.resolution', 'pandas.tseries', 'pandas.tseries.offsets', 'pandas.core.tools', 'pandas.core.tools.datetimes', 'dateutil.easter', 'pandas._libs.tslibs.offsets', 'pandas.tseries.frequencies', 'pandas._libs.join', 'pandas.core.ops', 'pandas._libs.ops', 'pandas.core.indexes.frozen', 'pandas.core.dtypes.concat', 'pandas.core.sorting', 'pandas.core.strings', 'pandas.core.indexes.category', 'pandas.core.indexes.multi', 'pandas.core.indexes.interval', 'pandas._libs.interval', 'pandas.core.indexes.datetimes', 'pandas.core.indexes.numeric', 'pandas.core.indexes.datetimelike', 'pandas.core.tools.timedeltas', 'pandas.core.indexes.timedeltas', 'pandas.core.indexes.range', 'pandas.core.indexes.period', 'pandas.core.frame', 'pandas.core.generic', 'pandas.core.indexing', 'pandas._libs.indexing', 'pandas.core.internals', 'pandas._libs.internals', 'pandas.core.sparse', 'pandas.core.sparse.array', 'pandas._libs.sparse', 'pandas.io.formats.format', 'pandas.io.common', 'mmap', 'pandas.core.series', 'pandas.core.indexes.accessors', 'pandas.plotting', 'pandas.plotting._misc', 'pandas.plotting._style', 'pandas.plotting._compat', 'pandas.plotting._tools', 'pandas.plotting._core', 'pandas.plotting._converter', 'matplotlib', 'matplotlib.cbook', 'matplotlib.cbook.deprecation', 'matplotlib.rcsetup', 'matplotlib.fontconfig_pattern', 'pyparsing', 'matplotlib.colors', 'matplotlib._color_data', 'cycler', 'matplotlib._version']\n"
     ]
    }
   ],
   "source": [
    "# newspaper_db\n",
    "import pandas as pd\n",
    "import sqlite3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Read sqlite query results into a pandas DataFrame\n",
    "con = sqlite3.connect(PATH_FOLDER_SAVE + \"/newspaper_db\")\n",
    "df = pd.read_sql_query(\"SELECT * from articles\", con)\n",
    "\n",
    "con.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_art</th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>theme</th>\n",
       "      <th>description</th>\n",
       "      <th>date_published</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>905</th>\n",
       "      <td>906</td>\n",
       "      <td>https://bourse.lefigaro.fr/indices/la-bourse-d...</td>\n",
       "      <td>Le Figaro</td>\n",
       "      <td>Hervé Rousseau</td>\n",
       "      <td>La Bourse de Paris boucle sa meilleure année d...</td>\n",
       "      <td>indices</td>\n",
       "      <td>Le CAC 40 a gagné près de 27 % cette année. Le...</td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>Un millésime à marquer d’une pierre blanche. L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>906</th>\n",
       "      <td>907</td>\n",
       "      <td>https://bourse.lefigaro.fr/indices/ces-entrepr...</td>\n",
       "      <td>Le Figaro</td>\n",
       "      <td>Hervé Rousseau</td>\n",
       "      <td>Ces entreprises qui affolent les compteurs de ...</td>\n",
       "      <td>indices</td>\n",
       "      <td>LVMH, le numéro un mondial du luxe est de très...</td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>Cette année, le CAC 40 doit sa formidable perf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>907</th>\n",
       "      <td>908</td>\n",
       "      <td>https://www.liberation.fr/france/2019/12/30/tc...</td>\n",
       "      <td>Liberation</td>\n",
       "      <td>Cédric Mathiot</td>\n",
       "      <td>Tchernobyl : la réponse la plus longue (qui re...</td>\n",
       "      <td>france</td>\n",
       "      <td></td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>On dit souvent du fact-checking qu’il est né a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>908</th>\n",
       "      <td>909</td>\n",
       "      <td>https://www.liberation.fr/france/2019/12/30/qu...</td>\n",
       "      <td>Liberation</td>\n",
       "      <td>Pauline Moullot, Robin Andraca</td>\n",
       "      <td>Quand «CheckNews» répond aux questions sur «Li...</td>\n",
       "      <td>france</td>\n",
       "      <td>Le service est fréquemment interrogé sur son p...</td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>La tendance se confirme. Depuis le lancement d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>909</th>\n",
       "      <td>910</td>\n",
       "      <td>https://www.liberation.fr/france/2019/12/30/fe...</td>\n",
       "      <td>Liberation</td>\n",
       "      <td>Robin Andraca</td>\n",
       "      <td>Femmes voilées (pas) vues à la télé : la répon...</td>\n",
       "      <td>france</td>\n",
       "      <td></td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>Il nous arrive, lorsque les circonstances le r...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id_art                                                url      source  \\\n",
       "905     906  https://bourse.lefigaro.fr/indices/la-bourse-d...   Le Figaro   \n",
       "906     907  https://bourse.lefigaro.fr/indices/ces-entrepr...   Le Figaro   \n",
       "907     908  https://www.liberation.fr/france/2019/12/30/tc...  Liberation   \n",
       "908     909  https://www.liberation.fr/france/2019/12/30/qu...  Liberation   \n",
       "909     910  https://www.liberation.fr/france/2019/12/30/fe...  Liberation   \n",
       "\n",
       "                             author  \\\n",
       "905                 Hervé Rousseau    \n",
       "906                 Hervé Rousseau    \n",
       "907                  Cédric Mathiot   \n",
       "908  Pauline Moullot, Robin Andraca   \n",
       "909                   Robin Andraca   \n",
       "\n",
       "                                                 title    theme  \\\n",
       "905  La Bourse de Paris boucle sa meilleure année d...  indices   \n",
       "906  Ces entreprises qui affolent les compteurs de ...  indices   \n",
       "907  Tchernobyl : la réponse la plus longue (qui re...   france   \n",
       "908  Quand «CheckNews» répond aux questions sur «Li...   france   \n",
       "909  Femmes voilées (pas) vues à la télé : la répon...   france   \n",
       "\n",
       "                                           description date_published  \\\n",
       "905  Le CAC 40 a gagné près de 27 % cette année. Le...     30/12/2019   \n",
       "906  LVMH, le numéro un mondial du luxe est de très...     30/12/2019   \n",
       "907                                                        30/12/2019   \n",
       "908  Le service est fréquemment interrogé sur son p...     30/12/2019   \n",
       "909                                                        30/12/2019   \n",
       "\n",
       "                                                  body  \n",
       "905  Un millésime à marquer d’une pierre blanche. L...  \n",
       "906  Cette année, le CAC 40 doit sa formidable perf...  \n",
       "907  On dit souvent du fact-checking qu’il est né a...  \n",
       "908  La tendance se confirme. Depuis le lancement d...  \n",
       "909  Il nous arrive, lorsque les circonstances le r...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Verify that result of SQL query is stored in the dataframe\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(910, 9)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df[\"body\"].isna()].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Futura Sciences    341\n",
       "Liberation         336\n",
       "Telerama           103\n",
       "Le Monde            66\n",
       "Le Figaro           64\n",
       "Name: source, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"source\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_art</th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>theme</th>\n",
       "      <th>description</th>\n",
       "      <th>date_published</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>790</th>\n",
       "      <td>791</td>\n",
       "      <td>https://www.lemonde.fr/culture/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Nos films coup de cœur de 2019</td>\n",
       "      <td>culture</td>\n",
       "      <td>« Portrait de la jeune fille en feu », « Par...</td>\n",
       "      <td>2019-12-27</td>\n",
       "      <td>La sélection de Clarisse Fabre 1770-2034 : à c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>791</th>\n",
       "      <td>792</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Chercheurs détenus en Iran : Téhéran dénonce «...</td>\n",
       "      <td>international</td>\n",
       "      <td>La France avait réagi à la détention de deux...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Les autorités iraniennes ont dénoncé dimanche ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793</th>\n",
       "      <td>794</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Benoît Vitkine,</td>\n",
       "      <td>Nouvel échange de prisonniers entre Kiev et le...</td>\n",
       "      <td>international</td>\n",
       "      <td>Des policiers impliqués dans le meurtre de m...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Il s’agit de la première avancée concrète depu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>796</td>\n",
       "      <td>https://www.lemonde.fr/pixels/article/2019/12/...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Damien Leloup,</td>\n",
       "      <td>A Leipzig, l’avenir des messageries sécurisées...</td>\n",
       "      <td>pixels</td>\n",
       "      <td>Les progrès restant à faire pour sécuriser e...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Comment s’assurer que des oreilles indiscrètes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>797</td>\n",
       "      <td>https://www.lemonde.fr/m-perso/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Sexe et alcool, pourvu qu’on ait l’ivresse !</td>\n",
       "      <td>m perso</td>\n",
       "      <td></td>\n",
       "      <td>2019-12-28</td>\n",
       "      <td>Aurez-vous encore envie de boire de l’alcool d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>798</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Les Etats-Unis ont connu un nombre record de t...</td>\n",
       "      <td>international</td>\n",
       "      <td>Les médias américains ont recensé 41 tueries...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Les Etats-Unis ont connu un nombre record de t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>801</th>\n",
       "      <td>802</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Pour le Nouvel An, Sydney maintient son feu d’...</td>\n",
       "      <td>international</td>\n",
       "      <td>Une pétition réunissant 270 000 signatures d...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>La tradition sera respectée, en dépit du conte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>803</th>\n",
       "      <td>804</td>\n",
       "      <td>https://www.lemonde.fr/afrique/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Youenn Gourlay,</td>\n",
       "      <td>En Côte d’Ivoire, enseigner à hauteur d’élève ...</td>\n",
       "      <td>afrique</td>\n",
       "      <td>Une meilleure école pour moins de pauvreté e...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Dans cette salle de classe du sud-ouest ivoiri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>805</td>\n",
       "      <td>https://www.lemonde.fr/afrique/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Mathieu Pellerin</td>\n",
       "      <td>« Le terrorisme sahélien entre global et local »</td>\n",
       "      <td>afrique</td>\n",
       "      <td>RENDEZ-VOUS DES IDÉES. Mathieu Pellerin, che...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Document.  L’adjectif « glocal » est souvent u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>805</th>\n",
       "      <td>806</td>\n",
       "      <td>https://www.lemonde.fr/sciences/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Vahé Ter Minassian,</td>\n",
       "      <td>Série Cosmos : un trou noir stellaire atypique</td>\n",
       "      <td>sciences</td>\n",
       "      <td>A l’occasion de la parution du deuxième ouvr...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Rien ne va plus dans le petit monde des chasse...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>807</td>\n",
       "      <td>https://www.lemonde.fr/sciences/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Comment les idées viennent aux scientifiques :...</td>\n",
       "      <td>sciences</td>\n",
       "      <td>Stéphanie Lacour, 44 ans, directrice du cent...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Comment les idées nouvelles naissent-elles ? A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>807</th>\n",
       "      <td>808</td>\n",
       "      <td>https://www.lemonde.fr/planete/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Des orques vues « pour la première fois » dans...</td>\n",
       "      <td>planete</td>\n",
       "      <td>Trois orques se trouvaient vendredi entre la...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>C’est une première dans ce bras de mer entre l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>809</th>\n",
       "      <td>810</td>\n",
       "      <td>https://www.lemonde.fr/politique/article/2019/...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Réforme des retraites : les vœux très attendus...</td>\n",
       "      <td>politique</td>\n",
       "      <td>Au 26e jour d’un conflit sur les retraites e...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Mardi 31 décembre à 20 heures, Emmanuel Macron...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811</th>\n",
       "      <td>812</td>\n",
       "      <td>https://www.lemonde.fr/planete/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec Reuters</td>\n",
       "      <td>Australie : face aux incendies, 100 000 person...</td>\n",
       "      <td>planete</td>\n",
       "      <td>Un pompier volontaire est mort et deux autre...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Environ 100 000 personnes ont été contraintes,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>814</th>\n",
       "      <td>815</td>\n",
       "      <td>https://www.lemonde.fr/economie/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Joël Morio,</td>\n",
       "      <td>Une année 2019 exceptionnelle pour les marchés...</td>\n",
       "      <td>economie</td>\n",
       "      <td>Les craintes de récession et les tensions co...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>La visibilité retrouvée en cette fin d’année, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>816</th>\n",
       "      <td>817</td>\n",
       "      <td>https://www.lemonde.fr/afrique/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Jean-Philippe Rémy,</td>\n",
       "      <td>En Somalie, un nouveau carnage aux portes de l...</td>\n",
       "      <td>afrique</td>\n",
       "      <td>Parmi les victimes de l’attentat de samedi, ...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Dans la foule bloquée, aux heures de pointe du...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>820</td>\n",
       "      <td>https://www.lemonde.fr/economie/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Hélène Jouan,</td>\n",
       "      <td>Le cannabis canadien, de l’euphorie à la décep...</td>\n",
       "      <td>economie</td>\n",
       "      <td>La légalisation de la marijuana avait suscit...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Le 17 octobre 2018, lorsque le gouvernement de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>821</th>\n",
       "      <td>822</td>\n",
       "      <td>https://www.lemonde.fr/societe/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Mattea Battaglia,   et   , Camille Strombo...</td>\n",
       "      <td>« Tout sauf mon collège de secteur » : les str...</td>\n",
       "      <td>societe</td>\n",
       "      <td>Parents d’élèves, ils ont choisi d’éviter po...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Ils ne sont jamais à court d’arguments pour ex...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>823</th>\n",
       "      <td>824</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Irak : des raids meurtriers des Etats-Unis con...</td>\n",
       "      <td>international</td>\n",
       "      <td>Selon le Pentagone, les cibles visées, diman...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>L’armée américaine a annoncé, dimanche 29 déce...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>826</th>\n",
       "      <td>827</td>\n",
       "      <td>https://www.lemonde.fr/politique/article/2019/...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Cédric Pietralunga,</td>\n",
       "      <td>« C’est le livre macronien par excellence, il ...</td>\n",
       "      <td>politique</td>\n",
       "      <td>Des macronistes se prennent de passion pour ...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Les macronistes sont parfois étonnants. Alors ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>828</th>\n",
       "      <td>829</td>\n",
       "      <td>https://www.lemonde.fr/planete/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Alors que la Russie connaît l’année la plus ch...</td>\n",
       "      <td>planete</td>\n",
       "      <td>Dans la capitale, le 18 décembre, le mercure...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>C’est l’un des jours les plus importants pour ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>832</th>\n",
       "      <td>833</td>\n",
       "      <td>https://www.lemonde.fr/football/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Anthony Hernandez,</td>\n",
       "      <td>Ligue 1 : pour oublier Jardim, Monaco mise sur...</td>\n",
       "      <td>football</td>\n",
       "      <td>L’éphémère sélectionneur par intérim de l’éq...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>A Monaco, l’ère incarnée par l’ex-vice-préside...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833</th>\n",
       "      <td>834</td>\n",
       "      <td>https://www.lemonde.fr/societe/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Luc Leroux,   et   , Zineb Dryef,</td>\n",
       "      <td>Féminicides : le Téléphone grave danger, bilan...</td>\n",
       "      <td>societe</td>\n",
       "      <td>Destiné à protéger les femmes des violences ...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Ce vendredi de novembre, Camille a inspiré un ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>837</td>\n",
       "      <td>https://www.lemonde.fr/culture/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Pédophilie : l’écrivain Gabriel Matzneff dénon...</td>\n",
       "      <td>culture</td>\n",
       "      <td>L’écrivaine Vanessa Springora raconte dans u...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>L’écrivain Gabriel Matzneff, au cœur d’une pol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839</th>\n",
       "      <td>840</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Bruno Meyerfeld,</td>\n",
       "      <td>Brésil : Jair Bolsonaro est parti pour durer</td>\n",
       "      <td>idees</td>\n",
       "      <td>On aurait tort de ne voir le président brési...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Analyse.  Qu’on se le dise : après un an de ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>840</th>\n",
       "      <td>841</td>\n",
       "      <td>https://www.lemonde.fr/afrique/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>La Guinée-Bissau a voté pour élire son nouveau...</td>\n",
       "      <td>afrique</td>\n",
       "      <td>La population et la communauté international...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Les Bissau-Guinéens ont voté dimanche 29 décem...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>841</th>\n",
       "      <td>842</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Intérim,</td>\n",
       "      <td>Transparence du Conseil de l’Union européenne ...</td>\n",
       "      <td>international</td>\n",
       "      <td>Malgré quelques progrès, l’institution est t...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>La Finlande, qui arrive au terme de six mois d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>842</th>\n",
       "      <td>843</td>\n",
       "      <td>https://www.lemonde.fr/afrique/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Au Maroc, des centaines de manifestants contre...</td>\n",
       "      <td>afrique</td>\n",
       "      <td>Omar Radi a été placé en détention provisoir...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Plusieurs centaines de personnes ont manifesté...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>843</th>\n",
       "      <td>844</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Hélène Sallon,</td>\n",
       "      <td>Risque d’escalade entre les Etats-Unis et les ...</td>\n",
       "      <td>international</td>\n",
       "      <td>Des frappes américaines ont visé, dimanche, ...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Les tensions entre Washington et Téhéran monte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>844</th>\n",
       "      <td>845</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>En Libye, un échec européen</td>\n",
       "      <td>idees</td>\n",
       "      <td></td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Editorial du « Monde ».  Un désastre géopoliti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>851</th>\n",
       "      <td>852</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>John Lewis, icône des droits civiques aux Etat...</td>\n",
       "      <td>international</td>\n",
       "      <td>L’élu démocrate de Géorgie à la Chambre des ...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>John Lewis, icône du mouvement américain des d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>852</th>\n",
       "      <td>853</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>« Sous couvert de lutte contre le terrorisme, ...</td>\n",
       "      <td>idees</td>\n",
       "      <td></td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Chronique.  De la froideur déshumanisante d’un...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>853</th>\n",
       "      <td>854</td>\n",
       "      <td>https://www.lemonde.fr/international/article/2...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Jean-Pierre Stroobants,</td>\n",
       "      <td>Le fléau du travail au noir en Belgique, « ce ...</td>\n",
       "      <td>international</td>\n",
       "      <td>Un magazine belge a révélé que des dizaines ...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Grande pourvoyeuse de règles, vigoureuse avoca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>854</th>\n",
       "      <td>855</td>\n",
       "      <td>https://www.lemonde.fr/politique/article/2019/...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Mikaël Pichard,</td>\n",
       "      <td>Dans un village de Mayenne, maire de père en f...</td>\n",
       "      <td>politique</td>\n",
       "      <td>Dans le village mayennais de Saint-Laurent-l...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Montre en main, il ne faut pas une minute pour...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>855</th>\n",
       "      <td>856</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Réforme des retraites : « Un minimum garanti p...</td>\n",
       "      <td>idees</td>\n",
       "      <td></td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Tribune.  L’ex-haut-commissaire à la réforme d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>856</th>\n",
       "      <td>857</td>\n",
       "      <td>https://www.lemonde.fr/culture/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Rosita Boisseau,</td>\n",
       "      <td>Retraites : le ballet de la grève des danseurs...</td>\n",
       "      <td>culture</td>\n",
       "      <td>Depuis le 5 décembre, les danseurs de l’inst...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Les images de la performance se sont répandues...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>858</td>\n",
       "      <td>https://www.lemonde.fr/societe/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Sofia Fischer,</td>\n",
       "      <td>« Olivia », prostituée albanaise tuée la nuit ...</td>\n",
       "      <td>societe</td>\n",
       "      <td>Un homme de 28 ans a été mis en examen pour ...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>C’est une joggeuse qui a donné l’alerte, mercr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858</th>\n",
       "      <td>859</td>\n",
       "      <td>https://www.lemonde.fr/societe/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Luc Leroux,</td>\n",
       "      <td>Féminicides : « Tant que j’ai mon Téléphone gr...</td>\n",
       "      <td>societe</td>\n",
       "      <td>L’année 2018 a été marquée par la mort de de...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>C’était son bouclier, son armure. Elle avait u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>859</th>\n",
       "      <td>860</td>\n",
       "      <td>https://www.lemonde.fr/societe/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Madjid Zerrouky,</td>\n",
       "      <td>Le retour à la vie du monastère de Tibhirine</td>\n",
       "      <td>societe</td>\n",
       "      <td>Vingt-trois ans après l’assassinat de sept m...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Ils dorment au milieu d’une clairière, à l’omb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>861</td>\n",
       "      <td>https://www.lemonde.fr/societe/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Gilles Triolier,</td>\n",
       "      <td>Le groupe Korian de nouveau visé par une enquê...</td>\n",
       "      <td>societe</td>\n",
       "      <td>Trois ans après la mort d’une résidente de 9...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Comment Jeanine Dezaille, 94 ans, résidente d’...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>861</th>\n",
       "      <td>862</td>\n",
       "      <td>https://www.lemonde.fr/m-le-mag/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Nathalie Stey,</td>\n",
       "      <td>« Le rebond », un podcast pour une reconversio...</td>\n",
       "      <td>m le mag</td>\n",
       "      <td>Alors que les carrières hachées sont au cœur...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Prendre sa vie en main, changer de boulot,   l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>862</th>\n",
       "      <td>863</td>\n",
       "      <td>https://www.lemonde.fr/afrique/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Sophie Douce,</td>\n",
       "      <td>Au Burkina Faso, la scolarisation des filles, ...</td>\n",
       "      <td>afrique</td>\n",
       "      <td>Une meilleure école pour moins de pauvreté e...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Lorsque Fadiratou évoque ses rêves, ses yeux n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>863</th>\n",
       "      <td>864</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Véronique Tadjo : « Crises démocratique et env...</td>\n",
       "      <td>idees</td>\n",
       "      <td></td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Nous avons demandé à six écrivains de choisir ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>864</th>\n",
       "      <td>865</td>\n",
       "      <td>https://www.lemonde.fr/planete/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Matteo Maillard,</td>\n",
       "      <td>Haïdar El Ali, « l’homme qui enrichit la terre...</td>\n",
       "      <td>planete</td>\n",
       "      <td>Autodidacte, l’homme qui vient d’être nommé ...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>« Sembe ! Sembe ! » , le cri se perd dans le s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>865</th>\n",
       "      <td>866</td>\n",
       "      <td>https://www.lemonde.fr/economie/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Joël Morio,</td>\n",
       "      <td>Les entreprises ne se bousculent pas au portil...</td>\n",
       "      <td>economie</td>\n",
       "      <td>Echaudés par plusieurs retournements de marc...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>C’est pour le moins un paradoxe. Les marchés f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>866</th>\n",
       "      <td>867</td>\n",
       "      <td>https://www.lemonde.fr/culture/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Francis Marmande,</td>\n",
       "      <td>Hal Singer, le jazz « du fond du cœur »</td>\n",
       "      <td>culture</td>\n",
       "      <td>Le compositeur, saxophoniste ténor américain...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Le 8 octobre, Hal Singer, illustre compositeur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>867</th>\n",
       "      <td>868</td>\n",
       "      <td>https://www.lemonde.fr/culture/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Jacques Mandelbaum,</td>\n",
       "      <td>« Planetarium » : cinéma et séance de spiritis...</td>\n",
       "      <td>culture</td>\n",
       "      <td>Rebecca Zlotowski évoque les années 1930 en ...</td>\n",
       "      <td>2019-12-29</td>\n",
       "      <td>Dans le paysage relativement cérébral et dista...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868</th>\n",
       "      <td>869</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Pierre Lepidi,</td>\n",
       "      <td>« Les Apparences dépouillées » : noir dans le ...</td>\n",
       "      <td>idees</td>\n",
       "      <td>Laurent Laviolette a rencontré dix personnal...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Le livre.  « Quand avez-vous découvert que vou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869</th>\n",
       "      <td>870</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Antoine Reverchon,</td>\n",
       "      <td>Deirdre McCloskey, économiste libertarienne d’...</td>\n",
       "      <td>idees</td>\n",
       "      <td>L’économiste Américaine transgenre, spéciali...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Portrait.  Chacun s’empresse autour d’elle tan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>870</th>\n",
       "      <td>871</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Propos recueillis par    , Solenn de Royer,</td>\n",
       "      <td>« Nous devons trouver un moyen de civiliser à ...</td>\n",
       "      <td>idees</td>\n",
       "      <td>Le politiste David Djaïz explique que la cri...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Normalien et énarque, David Djaïz est haut fon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>872</td>\n",
       "      <td>https://www.lemonde.fr/sport/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Ces débats, polémiques et affaires qui ont agi...</td>\n",
       "      <td>sport</td>\n",
       "      <td>Racisme dans les stades, compétitions dans d...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>De la dénonciation de la faible implication de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>873</th>\n",
       "      <td>874</td>\n",
       "      <td>https://www.lemonde.fr/economie/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Carlos Ghosn, ex-PDG de Renault-Nissan, a quit...</td>\n",
       "      <td>economie</td>\n",
       "      <td>L’annonce de ce départ est une surprise, alo...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Carlos Ghosn ne passera pas le Nouvel An au Ja...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875</th>\n",
       "      <td>876</td>\n",
       "      <td>https://www.lemonde.fr/economie/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Le chinois Casil vend l’aéroport de Toulouse à...</td>\n",
       "      <td>economie</td>\n",
       "      <td>A l’automne, les actionnaires locaux avaient...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Près de 200 millions d’euros : c’est la plus-v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>878</th>\n",
       "      <td>879</td>\n",
       "      <td>https://www.lemonde.fr/politique/article/2019/...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Grève contre la réforme des retraites : le 31 ...</td>\n",
       "      <td>politique</td>\n",
       "      <td>A Paris, seules les deux lignes de métro aut...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Les festivités du Nouvel An n’échapperont pas ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881</th>\n",
       "      <td>882</td>\n",
       "      <td>https://www.lemonde.fr/police-justice/article/...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Yann Bouchez,</td>\n",
       "      <td>Explosion rue de Trévise à Paris : des experts...</td>\n",
       "      <td>police justice</td>\n",
       "      <td>Un an après le drame qui a tué quatre person...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>L’accident meurtrier aurait pu être évité, san...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>885</th>\n",
       "      <td>886</td>\n",
       "      <td>https://www.lemonde.fr/afrique/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Au Soudan, 27 membres du renseignement condamn...</td>\n",
       "      <td>afrique</td>\n",
       "      <td>Ahmed Al-Khair, un enseignant de 36 ans, ava...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>C’est la première condamnation à mort de milit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>887</th>\n",
       "      <td>888</td>\n",
       "      <td>https://www.lemonde.fr/culture/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>Le Monde avec AFP</td>\n",
       "      <td>Le cinéma réalise sa meilleure année depuis 19...</td>\n",
       "      <td>culture</td>\n",
       "      <td>Qu’est-ce qui a dopé la fréquentation des ci...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Les salles obscures françaises ont rarement ét...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>891</td>\n",
       "      <td>https://www.lemonde.fr/idees/article/2019/12/3...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td></td>\n",
       "      <td>Maroc : « Un journaliste est emprisonné juste ...</td>\n",
       "      <td>idees</td>\n",
       "      <td></td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Tribune.  Omar Radi, un journaliste marocain d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>892</th>\n",
       "      <td>893</td>\n",
       "      <td>https://www.lemonde.fr/culture/article/2019/12...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Emilie Grangeray,</td>\n",
       "      <td>« 57, rue de Varenne » de François Pérache : l...</td>\n",
       "      <td>culture</td>\n",
       "      <td>L’excellente fiction politique de France Cul...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Il se dit que « 57, rue de Varenne », l’excell...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>896</th>\n",
       "      <td>897</td>\n",
       "      <td>https://www.lemonde.fr/sciences/article/2019/1...</td>\n",
       "      <td>Le Monde</td>\n",
       "      <td>, Nathaniel Herzberg,</td>\n",
       "      <td>Comment les idées viennent aux scientifiques :...</td>\n",
       "      <td>sciences</td>\n",
       "      <td>David Elbaz, 53 ans, est astrophysicien, dir...</td>\n",
       "      <td>2019-12-30</td>\n",
       "      <td>Comment les idées nouvelles naissent-elles ? A...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>66 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     id_art                                                url    source  \\\n",
       "790     791  https://www.lemonde.fr/culture/article/2019/12...  Le Monde   \n",
       "791     792  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "793     794  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "795     796  https://www.lemonde.fr/pixels/article/2019/12/...  Le Monde   \n",
       "796     797  https://www.lemonde.fr/m-perso/article/2019/12...  Le Monde   \n",
       "797     798  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "801     802  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "803     804  https://www.lemonde.fr/afrique/article/2019/12...  Le Monde   \n",
       "804     805  https://www.lemonde.fr/afrique/article/2019/12...  Le Monde   \n",
       "805     806  https://www.lemonde.fr/sciences/article/2019/1...  Le Monde   \n",
       "806     807  https://www.lemonde.fr/sciences/article/2019/1...  Le Monde   \n",
       "807     808  https://www.lemonde.fr/planete/article/2019/12...  Le Monde   \n",
       "809     810  https://www.lemonde.fr/politique/article/2019/...  Le Monde   \n",
       "811     812  https://www.lemonde.fr/planete/article/2019/12...  Le Monde   \n",
       "814     815  https://www.lemonde.fr/economie/article/2019/1...  Le Monde   \n",
       "816     817  https://www.lemonde.fr/afrique/article/2019/12...  Le Monde   \n",
       "819     820  https://www.lemonde.fr/economie/article/2019/1...  Le Monde   \n",
       "821     822  https://www.lemonde.fr/societe/article/2019/12...  Le Monde   \n",
       "823     824  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "826     827  https://www.lemonde.fr/politique/article/2019/...  Le Monde   \n",
       "828     829  https://www.lemonde.fr/planete/article/2019/12...  Le Monde   \n",
       "832     833  https://www.lemonde.fr/football/article/2019/1...  Le Monde   \n",
       "833     834  https://www.lemonde.fr/societe/article/2019/12...  Le Monde   \n",
       "836     837  https://www.lemonde.fr/culture/article/2019/12...  Le Monde   \n",
       "839     840  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "840     841  https://www.lemonde.fr/afrique/article/2019/12...  Le Monde   \n",
       "841     842  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "842     843  https://www.lemonde.fr/afrique/article/2019/12...  Le Monde   \n",
       "843     844  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "844     845  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "..      ...                                                ...       ...   \n",
       "851     852  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "852     853  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "853     854  https://www.lemonde.fr/international/article/2...  Le Monde   \n",
       "854     855  https://www.lemonde.fr/politique/article/2019/...  Le Monde   \n",
       "855     856  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "856     857  https://www.lemonde.fr/culture/article/2019/12...  Le Monde   \n",
       "857     858  https://www.lemonde.fr/societe/article/2019/12...  Le Monde   \n",
       "858     859  https://www.lemonde.fr/societe/article/2019/12...  Le Monde   \n",
       "859     860  https://www.lemonde.fr/societe/article/2019/12...  Le Monde   \n",
       "860     861  https://www.lemonde.fr/societe/article/2019/12...  Le Monde   \n",
       "861     862  https://www.lemonde.fr/m-le-mag/article/2019/1...  Le Monde   \n",
       "862     863  https://www.lemonde.fr/afrique/article/2019/12...  Le Monde   \n",
       "863     864  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "864     865  https://www.lemonde.fr/planete/article/2019/12...  Le Monde   \n",
       "865     866  https://www.lemonde.fr/economie/article/2019/1...  Le Monde   \n",
       "866     867  https://www.lemonde.fr/culture/article/2019/12...  Le Monde   \n",
       "867     868  https://www.lemonde.fr/culture/article/2019/12...  Le Monde   \n",
       "868     869  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "869     870  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "870     871  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "871     872  https://www.lemonde.fr/sport/article/2019/12/3...  Le Monde   \n",
       "873     874  https://www.lemonde.fr/economie/article/2019/1...  Le Monde   \n",
       "875     876  https://www.lemonde.fr/economie/article/2019/1...  Le Monde   \n",
       "878     879  https://www.lemonde.fr/politique/article/2019/...  Le Monde   \n",
       "881     882  https://www.lemonde.fr/police-justice/article/...  Le Monde   \n",
       "885     886  https://www.lemonde.fr/afrique/article/2019/12...  Le Monde   \n",
       "887     888  https://www.lemonde.fr/culture/article/2019/12...  Le Monde   \n",
       "890     891  https://www.lemonde.fr/idees/article/2019/12/3...  Le Monde   \n",
       "892     893  https://www.lemonde.fr/culture/article/2019/12...  Le Monde   \n",
       "896     897  https://www.lemonde.fr/sciences/article/2019/1...  Le Monde   \n",
       "\n",
       "                                                author  \\\n",
       "790                                                      \n",
       "791                                  Le Monde avec AFP   \n",
       "793                                , Benoît Vitkine,     \n",
       "795                                 , Damien Leloup,     \n",
       "796                                                      \n",
       "797                                                      \n",
       "801                                  Le Monde avec AFP   \n",
       "803                                , Youenn Gourlay,     \n",
       "804                                  Mathieu Pellerin    \n",
       "805                            , Vahé Ter Minassian,     \n",
       "806                                                      \n",
       "807                                  Le Monde avec AFP   \n",
       "809                                  Le Monde avec AFP   \n",
       "811                              Le Monde avec Reuters   \n",
       "814                                    , Joël Morio,     \n",
       "816                            , Jean-Philippe Rémy,     \n",
       "819                                  , Hélène Jouan,     \n",
       "821    , Mattea Battaglia,   et   , Camille Strombo...   \n",
       "823                                  Le Monde avec AFP   \n",
       "826                            , Cédric Pietralunga,     \n",
       "828                                                      \n",
       "832                             , Anthony Hernandez,     \n",
       "833              , Luc Leroux,   et   , Zineb Dryef,     \n",
       "836                                  Le Monde avec AFP   \n",
       "839                               , Bruno Meyerfeld,     \n",
       "840                                  Le Monde avec AFP   \n",
       "841                                       , Intérim,     \n",
       "842                                  Le Monde avec AFP   \n",
       "843                                 , Hélène Sallon,     \n",
       "844                                                      \n",
       "..                                                 ...   \n",
       "851                                  Le Monde avec AFP   \n",
       "852                                                      \n",
       "853                        , Jean-Pierre Stroobants,     \n",
       "854                                , Mikaël Pichard,     \n",
       "855                                                      \n",
       "856                               , Rosita Boisseau,     \n",
       "857                                 , Sofia Fischer,     \n",
       "858                                    , Luc Leroux,     \n",
       "859                               , Madjid Zerrouky,     \n",
       "860                               , Gilles Triolier,     \n",
       "861                                 , Nathalie Stey,     \n",
       "862                                  , Sophie Douce,     \n",
       "863                                                      \n",
       "864                               , Matteo Maillard,     \n",
       "865                                    , Joël Morio,     \n",
       "866                              , Francis Marmande,     \n",
       "867                            , Jacques Mandelbaum,     \n",
       "868                                 , Pierre Lepidi,     \n",
       "869                             , Antoine Reverchon,     \n",
       "870      Propos recueillis par    , Solenn de Royer,     \n",
       "871                                                      \n",
       "873                                                      \n",
       "875                                                      \n",
       "878                                  Le Monde avec AFP   \n",
       "881                                  , Yann Bouchez,     \n",
       "885                                                      \n",
       "887                                  Le Monde avec AFP   \n",
       "890                                                      \n",
       "892                              , Emilie Grangeray,     \n",
       "896                            , Nathaniel Herzberg,     \n",
       "\n",
       "                                                 title           theme  \\\n",
       "790                     Nos films coup de cœur de 2019         culture   \n",
       "791  Chercheurs détenus en Iran : Téhéran dénonce «...   international   \n",
       "793  Nouvel échange de prisonniers entre Kiev et le...   international   \n",
       "795  A Leipzig, l’avenir des messageries sécurisées...          pixels   \n",
       "796       Sexe et alcool, pourvu qu’on ait l’ivresse !         m perso   \n",
       "797  Les Etats-Unis ont connu un nombre record de t...   international   \n",
       "801  Pour le Nouvel An, Sydney maintient son feu d’...   international   \n",
       "803  En Côte d’Ivoire, enseigner à hauteur d’élève ...         afrique   \n",
       "804   « Le terrorisme sahélien entre global et local »         afrique   \n",
       "805     Série Cosmos : un trou noir stellaire atypique        sciences   \n",
       "806  Comment les idées viennent aux scientifiques :...        sciences   \n",
       "807  Des orques vues « pour la première fois » dans...         planete   \n",
       "809  Réforme des retraites : les vœux très attendus...       politique   \n",
       "811  Australie : face aux incendies, 100 000 person...         planete   \n",
       "814  Une année 2019 exceptionnelle pour les marchés...        economie   \n",
       "816  En Somalie, un nouveau carnage aux portes de l...         afrique   \n",
       "819  Le cannabis canadien, de l’euphorie à la décep...        economie   \n",
       "821  « Tout sauf mon collège de secteur » : les str...         societe   \n",
       "823  Irak : des raids meurtriers des Etats-Unis con...   international   \n",
       "826  « C’est le livre macronien par excellence, il ...       politique   \n",
       "828  Alors que la Russie connaît l’année la plus ch...         planete   \n",
       "832  Ligue 1 : pour oublier Jardim, Monaco mise sur...        football   \n",
       "833  Féminicides : le Téléphone grave danger, bilan...         societe   \n",
       "836  Pédophilie : l’écrivain Gabriel Matzneff dénon...         culture   \n",
       "839       Brésil : Jair Bolsonaro est parti pour durer           idees   \n",
       "840  La Guinée-Bissau a voté pour élire son nouveau...         afrique   \n",
       "841  Transparence du Conseil de l’Union européenne ...   international   \n",
       "842  Au Maroc, des centaines de manifestants contre...         afrique   \n",
       "843  Risque d’escalade entre les Etats-Unis et les ...   international   \n",
       "844                        En Libye, un échec européen           idees   \n",
       "..                                                 ...             ...   \n",
       "851  John Lewis, icône des droits civiques aux Etat...   international   \n",
       "852  « Sous couvert de lutte contre le terrorisme, ...           idees   \n",
       "853  Le fléau du travail au noir en Belgique, « ce ...   international   \n",
       "854  Dans un village de Mayenne, maire de père en f...       politique   \n",
       "855  Réforme des retraites : « Un minimum garanti p...           idees   \n",
       "856  Retraites : le ballet de la grève des danseurs...         culture   \n",
       "857  « Olivia », prostituée albanaise tuée la nuit ...         societe   \n",
       "858  Féminicides : « Tant que j’ai mon Téléphone gr...         societe   \n",
       "859       Le retour à la vie du monastère de Tibhirine         societe   \n",
       "860  Le groupe Korian de nouveau visé par une enquê...         societe   \n",
       "861  « Le rebond », un podcast pour une reconversio...        m le mag   \n",
       "862  Au Burkina Faso, la scolarisation des filles, ...         afrique   \n",
       "863  Véronique Tadjo : « Crises démocratique et env...           idees   \n",
       "864  Haïdar El Ali, « l’homme qui enrichit la terre...         planete   \n",
       "865  Les entreprises ne se bousculent pas au portil...        economie   \n",
       "866            Hal Singer, le jazz « du fond du cœur »         culture   \n",
       "867  « Planetarium » : cinéma et séance de spiritis...         culture   \n",
       "868  « Les Apparences dépouillées » : noir dans le ...           idees   \n",
       "869  Deirdre McCloskey, économiste libertarienne d’...           idees   \n",
       "870  « Nous devons trouver un moyen de civiliser à ...           idees   \n",
       "871  Ces débats, polémiques et affaires qui ont agi...           sport   \n",
       "873  Carlos Ghosn, ex-PDG de Renault-Nissan, a quit...        economie   \n",
       "875  Le chinois Casil vend l’aéroport de Toulouse à...        economie   \n",
       "878  Grève contre la réforme des retraites : le 31 ...       politique   \n",
       "881  Explosion rue de Trévise à Paris : des experts...  police justice   \n",
       "885  Au Soudan, 27 membres du renseignement condamn...         afrique   \n",
       "887  Le cinéma réalise sa meilleure année depuis 19...         culture   \n",
       "890  Maroc : « Un journaliste est emprisonné juste ...           idees   \n",
       "892  « 57, rue de Varenne » de François Pérache : l...         culture   \n",
       "896  Comment les idées viennent aux scientifiques :...        sciences   \n",
       "\n",
       "                                           description date_published  \\\n",
       "790    « Portrait de la jeune fille en feu », « Par...     2019-12-27   \n",
       "791    La France avait réagi à la détention de deux...     2019-12-29   \n",
       "793    Des policiers impliqués dans le meurtre de m...     2019-12-29   \n",
       "795    Les progrès restant à faire pour sécuriser e...     2019-12-29   \n",
       "796                                                        2019-12-28   \n",
       "797    Les médias américains ont recensé 41 tueries...     2019-12-29   \n",
       "801    Une pétition réunissant 270 000 signatures d...     2019-12-29   \n",
       "803    Une meilleure école pour moins de pauvreté e...     2019-12-29   \n",
       "804    RENDEZ-VOUS DES IDÉES. Mathieu Pellerin, che...     2019-12-29   \n",
       "805    A l’occasion de la parution du deuxième ouvr...     2019-12-29   \n",
       "806    Stéphanie Lacour, 44 ans, directrice du cent...     2019-12-29   \n",
       "807    Trois orques se trouvaient vendredi entre la...     2019-12-29   \n",
       "809    Au 26e jour d’un conflit sur les retraites e...     2019-12-30   \n",
       "811    Un pompier volontaire est mort et deux autre...     2019-12-30   \n",
       "814    Les craintes de récession et les tensions co...     2019-12-30   \n",
       "816    Parmi les victimes de l’attentat de samedi, ...     2019-12-30   \n",
       "819    La légalisation de la marijuana avait suscit...     2019-12-30   \n",
       "821    Parents d’élèves, ils ont choisi d’éviter po...     2019-12-30   \n",
       "823    Selon le Pentagone, les cibles visées, diman...     2019-12-29   \n",
       "826    Des macronistes se prennent de passion pour ...     2019-12-29   \n",
       "828    Dans la capitale, le 18 décembre, le mercure...     2019-12-30   \n",
       "832    L’éphémère sélectionneur par intérim de l’éq...     2019-12-30   \n",
       "833    Destiné à protéger les femmes des violences ...     2019-12-30   \n",
       "836    L’écrivaine Vanessa Springora raconte dans u...     2019-12-29   \n",
       "839    On aurait tort de ne voir le président brési...     2019-12-30   \n",
       "840    La population et la communauté international...     2019-12-30   \n",
       "841    Malgré quelques progrès, l’institution est t...     2019-12-30   \n",
       "842    Omar Radi a été placé en détention provisoir...     2019-12-30   \n",
       "843    Des frappes américaines ont visé, dimanche, ...     2019-12-30   \n",
       "844                                                        2019-12-30   \n",
       "..                                                 ...            ...   \n",
       "851    L’élu démocrate de Géorgie à la Chambre des ...     2019-12-30   \n",
       "852                                                        2019-12-30   \n",
       "853    Un magazine belge a révélé que des dizaines ...     2019-12-30   \n",
       "854    Dans le village mayennais de Saint-Laurent-l...     2019-12-30   \n",
       "855                                                        2019-12-30   \n",
       "856    Depuis le 5 décembre, les danseurs de l’inst...     2019-12-30   \n",
       "857    Un homme de 28 ans a été mis en examen pour ...     2019-12-30   \n",
       "858    L’année 2018 a été marquée par la mort de de...     2019-12-30   \n",
       "859    Vingt-trois ans après l’assassinat de sept m...     2019-12-30   \n",
       "860    Trois ans après la mort d’une résidente de 9...     2019-12-30   \n",
       "861    Alors que les carrières hachées sont au cœur...     2019-12-30   \n",
       "862    Une meilleure école pour moins de pauvreté e...     2019-12-30   \n",
       "863                                                        2019-12-30   \n",
       "864    Autodidacte, l’homme qui vient d’être nommé ...     2019-12-29   \n",
       "865    Echaudés par plusieurs retournements de marc...     2019-12-30   \n",
       "866    Le compositeur, saxophoniste ténor américain...     2019-12-30   \n",
       "867    Rebecca Zlotowski évoque les années 1930 en ...     2019-12-29   \n",
       "868    Laurent Laviolette a rencontré dix personnal...     2019-12-30   \n",
       "869    L’économiste Américaine transgenre, spéciali...     2019-12-30   \n",
       "870    Le politiste David Djaïz explique que la cri...     2019-12-30   \n",
       "871    Racisme dans les stades, compétitions dans d...     2019-12-30   \n",
       "873    L’annonce de ce départ est une surprise, alo...     2019-12-30   \n",
       "875    A l’automne, les actionnaires locaux avaient...     2019-12-30   \n",
       "878    A Paris, seules les deux lignes de métro aut...     2019-12-30   \n",
       "881    Un an après le drame qui a tué quatre person...     2019-12-30   \n",
       "885    Ahmed Al-Khair, un enseignant de 36 ans, ava...     2019-12-30   \n",
       "887    Qu’est-ce qui a dopé la fréquentation des ci...     2019-12-30   \n",
       "890                                                        2019-12-30   \n",
       "892    L’excellente fiction politique de France Cul...     2019-12-30   \n",
       "896    David Elbaz, 53 ans, est astrophysicien, dir...     2019-12-30   \n",
       "\n",
       "                                                  body  \n",
       "790  La sélection de Clarisse Fabre 1770-2034 : à c...  \n",
       "791  Les autorités iraniennes ont dénoncé dimanche ...  \n",
       "793  Il s’agit de la première avancée concrète depu...  \n",
       "795  Comment s’assurer que des oreilles indiscrètes...  \n",
       "796  Aurez-vous encore envie de boire de l’alcool d...  \n",
       "797  Les Etats-Unis ont connu un nombre record de t...  \n",
       "801  La tradition sera respectée, en dépit du conte...  \n",
       "803  Dans cette salle de classe du sud-ouest ivoiri...  \n",
       "804  Document.  L’adjectif « glocal » est souvent u...  \n",
       "805  Rien ne va plus dans le petit monde des chasse...  \n",
       "806  Comment les idées nouvelles naissent-elles ? A...  \n",
       "807  C’est une première dans ce bras de mer entre l...  \n",
       "809  Mardi 31 décembre à 20 heures, Emmanuel Macron...  \n",
       "811  Environ 100 000 personnes ont été contraintes,...  \n",
       "814  La visibilité retrouvée en cette fin d’année, ...  \n",
       "816  Dans la foule bloquée, aux heures de pointe du...  \n",
       "819  Le 17 octobre 2018, lorsque le gouvernement de...  \n",
       "821  Ils ne sont jamais à court d’arguments pour ex...  \n",
       "823  L’armée américaine a annoncé, dimanche 29 déce...  \n",
       "826  Les macronistes sont parfois étonnants. Alors ...  \n",
       "828  C’est l’un des jours les plus importants pour ...  \n",
       "832  A Monaco, l’ère incarnée par l’ex-vice-préside...  \n",
       "833  Ce vendredi de novembre, Camille a inspiré un ...  \n",
       "836  L’écrivain Gabriel Matzneff, au cœur d’une pol...  \n",
       "839  Analyse.  Qu’on se le dise : après un an de ma...  \n",
       "840  Les Bissau-Guinéens ont voté dimanche 29 décem...  \n",
       "841  La Finlande, qui arrive au terme de six mois d...  \n",
       "842  Plusieurs centaines de personnes ont manifesté...  \n",
       "843  Les tensions entre Washington et Téhéran monte...  \n",
       "844  Editorial du « Monde ».  Un désastre géopoliti...  \n",
       "..                                                 ...  \n",
       "851  John Lewis, icône du mouvement américain des d...  \n",
       "852  Chronique.  De la froideur déshumanisante d’un...  \n",
       "853  Grande pourvoyeuse de règles, vigoureuse avoca...  \n",
       "854  Montre en main, il ne faut pas une minute pour...  \n",
       "855  Tribune.  L’ex-haut-commissaire à la réforme d...  \n",
       "856  Les images de la performance se sont répandues...  \n",
       "857  C’est une joggeuse qui a donné l’alerte, mercr...  \n",
       "858  C’était son bouclier, son armure. Elle avait u...  \n",
       "859  Ils dorment au milieu d’une clairière, à l’omb...  \n",
       "860  Comment Jeanine Dezaille, 94 ans, résidente d’...  \n",
       "861  Prendre sa vie en main, changer de boulot,   l...  \n",
       "862  Lorsque Fadiratou évoque ses rêves, ses yeux n...  \n",
       "863  Nous avons demandé à six écrivains de choisir ...  \n",
       "864  « Sembe ! Sembe ! » , le cri se perd dans le s...  \n",
       "865  C’est pour le moins un paradoxe. Les marchés f...  \n",
       "866  Le 8 octobre, Hal Singer, illustre compositeur...  \n",
       "867  Dans le paysage relativement cérébral et dista...  \n",
       "868  Le livre.  « Quand avez-vous découvert que vou...  \n",
       "869  Portrait.  Chacun s’empresse autour d’elle tan...  \n",
       "870  Normalien et énarque, David Djaïz est haut fon...  \n",
       "871  De la dénonciation de la faible implication de...  \n",
       "873  Carlos Ghosn ne passera pas le Nouvel An au Ja...  \n",
       "875  Près de 200 millions d’euros : c’est la plus-v...  \n",
       "878  Les festivités du Nouvel An n’échapperont pas ...  \n",
       "881  L’accident meurtrier aurait pu être évité, san...  \n",
       "885  C’est la première condamnation à mort de milit...  \n",
       "887  Les salles obscures françaises ont rarement ét...  \n",
       "890  Tribune.  Omar Radi, un journaliste marocain d...  \n",
       "892  Il se dit que « 57, rue de Varenne », l’excell...  \n",
       "896  Comment les idées nouvelles naissent-elles ? A...  \n",
       "\n",
       "[66 rows x 9 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df[\"source\"] == \"Le Monde\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check body data not empty?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df[\"body\"].isna()].shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### delete useless special text patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Avec 2920 nouveaux cas en 2018, le cancer du col de l’utérus est le douzième cancer en matière de fréquence chez la femme en France. Depuis 35 ans, son incidence a été divisée par deux. Cette diminution, très forte dans les années 1980, grâce au dépistage des lésions précancéreuses par la réalisation régulière de frottis, marque nettement le pas. Il s’agit du seul cancer dont le pronostic se dégrade dans notre pays, avec un taux de survie à cinq ans passé de 68\\xa0% dans les années 1990 à 62\\xa0% aujourd’hui. Causes, symptômes, traitements...  Le Figaro  fait le point sur ce cancer. 1. Quelles causes? Dans près de 99\\xa0% des cas, ce cancer est lié à une infection persistante par le papillomavirus humain ou HPV. Ce virus, très contagieux, se propage par simple contact sexuel.  «À la différence des autres infections sexuellement transmissibles, il se transmet dès les premiers attouchements, même sans pénétration»,  indique le D r\\xa0 Delphine Hudry, chirurgien gynécologue au centre de lutte contre le cancer Oscar-Lambret (Lille). Le préservatif n’en protège pas, car le virus, très petit, peut passer à travers ses pores.  On estime qu’environ 80\\xa0% des hommes et des femmes seront infectés par un papillomavirus au moins une fois dans leur vie. Chez la femme, la majorité des infections au niveau du col guérissent spontanément. Mais dans 10\\xa0% des cas, le virus s’installe durablement et provoquera des lésions précancéreuses ou cancéreuses. Si le papillomavirus est le facteur de risque principal, d’autres cofacteurs importants interviennent également, en particulier le tabagisme.  «En induisant une inflammation chronique au niveau du col, en plus de celle liée au virus, le tabac l’empêche de guérir de son infection»,  précise le D r  Hudry. Des infections du col (chlamydia, virus de l’immunodéficience humaine, gonocoque, virus herpès simplex 2), évitables, elles, grâce à l’utilisation du préservatif sont aussi mises en cause.  La prise prolongée de contraceptifs oraux ou les accouchements multiples (à partir de 6 à 8, selon le D r  Hudry) peuvent jouer un rôle, mais dans une moindre mesure. Quant au nombre élevé de partenaires sexuels, parfois évoqué,  «c’est un facteur de risque beaucoup moins important, voire discutable»,  souligne la gynécologue. » LIRE AUSSI -  Cancer du col de l’utérus: dépistage et vaccination doivent être améliorés     2. Quelle prévention? Seule la vaccination contre le HPV permet de prévenir l’apparition de lésions précancéreuses et, donc, à terme, des cancers du col de l’utérus, à condition d’intervenir avant les premiers rapports sexuels. Elle permet aussi d’éviter d’autres cancers liés au HPV, dont certains peuvent toucher les hommes: certains cancers ORL (oto-rhino-laryngologiques), de la verge, de l’anus, de la vulve et du vagin, ainsi que les condylomes (ou verrues génitales), des lésions bénignes liées à l’infection par le papillomavirus.  En France, la vaccination est recommandée chez les jeunes filles de 11 à 14 ans et jusqu’à l’âge de 19 ans en rattrapage\\xa0; chez les hommes ayant des rapports sexuels avec d’autres hommes jusqu’à l’âge de 26 ans\\xa0; et chez les personnes immunodéprimées. Son extension à l’ensemble des garçons, en cours de discussion à la Haute Autorité de santé (HAS), est expérimentée depuis fin 2018 dans les régions Grand Est et Auvergne-Rhône-Alpes. Cervarix, Gardasil et Gardasil 9: trois vaccins sont aujourd’hui disponibles, protégeant contre les principales souches du HPV - dont les génotypes 16 et 18, responsables de près de 70\\xa0% des cancers du col.  «Malheureusement, la vaccination est sous-utilisée, puisque seulement 15\\xa0% des enfants sont vaccinés»,  regrette le D r  Delphine Hudry. En France, le taux de couverture vaccinale des jeunes filles contre les\\xa0principales souches du papillomavirus humain est l’un des plus bas d’Europe: moins de 20\\xa0% (contre 87\\xa0% au Portugal). 3. Quels symptômes? Dans la majorité des cas, il n’existe pas de symptômes aux stades précoces de la maladie, c’est-à-dire avant l’apparition des lésions cancéreuses. Lorsque ces lésions sont présentes, des saignements (métrorragies) peuvent se produire lors des rapports sexuels, entre les périodes de règles ou après la ménopause. Il peut aussi y avoir des pertes vaginales anormales et/ou persistantes liées à un déséquilibre de la flore vaginale provoqué par le cancer.  «En cas de modification des pertes, de saignements ou de tout symptôme gynécologique anormal, il est donc impératif de consulter»,  souligne le D r  Hudry. » LIRE AUSSI -  Cancer de l’utérus: réagir aux premiers saignements     4. Quel diagnostic? Les lésions précancéreuses évoluant de façon silencieuse, le dépistage par frottis cervico-utérin, qui consiste à prélever des cellules superficielles au niveau du col avec une petite brosse, reste le meilleur moyen de les détecter. Et ce, que les femmes soient ou non vaccinées, car la vaccination ne protège pas contre la totalité des virus HPV à haut risque oncogène.  En France, la Haute Autorité de santé (HAS) recommande que ce frottis soit effectué tous les trois ans, après 2 frottis annuels normaux, chez toutes les femmes asymptomatiques âgées de 25 à 65 ans. Depuis cette année, le cancer du col de l’utérus fait l’objet d’un dépistage organisé au niveau national, comme le cancer du sein et le cancer colorectal avant lui.  Chaque année, les femmes de 25 à 65 ans qui n’ont pas réalisé de frottis depuis au moins trois ans seront invitées, par courrier, à se rendre chez un médecin généraliste, un gynécologue ou une sage-femme pour effectuer ce test de dépistage pris en charge à 100\\xa0% par l’Assurance-maladie. «Lorsque le col est normal, on réalise un frottis pour dépister une éventuelle anomalie non visible à l’œil nu»,  explique le D r  Hudry. Si le résultat de ce frottis est anormal, un nouveau frottis - ou une biopsie - est alors recommandé.  «Lorsque le col présente des anomalies, une biopsie est effectuée d’emblée»,  précise la gynécologue.  Seule la biopsie, réalisée grâce à un microscope grossissant appelé «colposcope», permet de déterminer la nature des lésions et de poser un diagnostic.  «Lorsqu’elle est anormale, on réalise soit une biopsie chirurgicale, appelée conisation, qui\\xa0consiste à enlever une partie plus importante du col pour avoir une analyse plus fine,  poursuit-elle,  soit une imagerie par résonance magnétique (IRM) pelvienne pour déterminer l’étendue des lésions.»   5. Quels traitements? Les lésions précancéreuses de bas grade sont soit simplement surveillées, car elles peuvent régresser spontanément, soit détruites (par laser ou cryothérapie).  «Pour les lésions précancéreuses de haut grade ou lorsque la tumeur mesure moins de 4\\xa0cm, le traitement sera uniquement chirurgical»,  indique le D r \\xa0Hudry. Il consistera à retirer une partie du col de l’utérus (conisation) au laser ou à l’anse diathermique (instrument tranchant fonctionnant à l’électricité). Une ablation totale de l’utérus (hystérectomie) est pratiquée si la tumeur mesure plus de 1\\xa0cm.  «Lorsque la tumeur reste localisée au niveau du col mais qu’elle mesure plus de 4\\xa0cm, il faudra faire une radio-chimiothérapie concomitante»,  poursuit la gynécologue.  La radiothérapie sera externe et couplée à une séance de chimiothérapie toutes les semaines pour aider les rayons à fonctionner. Le traitement se clôturera par une curiethérapie, sorte de radiothérapie interne utilisant un applicateur disposé au niveau du vagin, qui délivre de hautes doses de rayons à l’endroit où était localisée la tumeur.  «En\\xa0cas de métastases, une chimiothérapie seule traitera l’ensemble de l’organisme. Si elle fonctionne bien et que la maladie ne progresse pas, le cancer initial peut parfois\\xa0être traité avec une radiothérapie externe et une curiethérapie»,  précise la gynécologue. 6. Quel suivi? Pour détecter les rechutes,  «la surveillance clinique prime» , indique le D r \\xa0Hudry. Le suivi est le même pour toutes les patientes: un examen clinique tous les quatre mois pendant trois ans, puis tous les six mois jusqu’à cinq ans et enfin tous les ans, à vie.',\n",
       " 'Le créateur d’Astérix, René Goscinny, a payé de sa vie la rupture d’une plaque d’athérome, à l’origine de 80\\xa0% des morts subites et conséquence dramatique de l’athérosclérose. Cette maladie de l’artère débute dès l’enfance par de simples stries lipidiques ou amas graisseux à la surface de l’intima, l’une des trois couches des artères.  Leurs parois sont tapissées par l’endothélium, un revêtement que l’on a cru, pendant longtemps, suffisamment lisse pour empêcher toute adhésion. Mais par un phénomène inflammatoire associant une fibrose à l’accumulation de mauvais cholestérol (LDL cholestérol), l’athérosclérose peut se développer. D’abord asymptomatique, elle peut ensuite provoquer un rétrécissement ou sténose, gênant le passage du sang et son apport en oxygène à l’organe irrigué. L’athérosclérose touche surtout  «les zones de turbulence, carrefours ou bifurcations des moyennes et grosses artères»,  explique le Pr\\xa0Claire Mounier-Véhier, chef de service à l’Institut cœur poumon au CHRU de Lille et présidente honoraire de la Fédération française de cardiologie. Des symptômes tels qu’essoufflement à l’effort ou à la marche, vertiges et douleur thoracique doivent alerter. La rupture d’une plaque devenue trop instable peut provoquer, selon sa localisation, un  accident vasculaire cérébral , un syndrome coronaire aigu, une rupture d’anévrisme aortique athéromateux ou une ischémie aiguë de membre inférieur. » LIRE AUSSI -  AVC, infarctus: pourquoi le tabac est-il à l’origine de maladies cardiovasculaires?     Évitable dans 80% des cas Dans 8 cas sur 10, la maladie est évitable.  «L’athérosclérose est favorisée par le tabagisme, la sédentarité, le surpoids, une alimentation trop sucrée, trop grasse, trop salée. L’âge et l’hérédité entrent également en cause, mais ne sont pas évitables»,  précise Claire Le Hello, professeur de médecine vasculaire au CHU Nord de Saint-Etienne. Si les femmes sont peu exposées avant 50\\xa0ans, leur risque vasculaire rejoint celui des hommes après la ménopause, souligne le professeur.  Plusieurs marqueurs de risque que sont l’apnée du sommeil, l’isolement ou la précarité sociale et la pollution atmosphérique sont également à prendre en compte. Tous se catalysent les uns les autres avec un risque de complication majoré avec l’âge. Un bilan des facteurs de risque devrait être systématiquement réalisé après 40\\xa0ans chez l’homme et 50\\xa0ans chez la femme. \\n     \\n         «L’athérosclérose est favorisée par le tabagisme, la sédentarité, le surpoids, une alimentation trop sucrée, trop grasse, trop salée.» \\n                     Claire Le Hello, professeur de médecine vasculaire au CHU Nord de Saint-Etienne \\n             \\n Un diagnostic complet Utilisé depuis quelques années pour le dépistage de l’athérosclérose,  «le score calcique est un outil parmi les plus efficaces pour prévenir le risque d’infarctus» , précise le Dr\\xa0Patrick Assyag, cardiologue à Paris et membre de la FFC. Il permet une évaluation chiffrée de l’étendue des dépôts des artères coronaires. S’ensuit un examen clinique exhaustif, qui comprend une auscultation du cœur, des artères de l’abdomen, des jambes, des artères du cou et des artères sous-clavières.  «Il ne doit pas omettre l’examen cutané des pieds pour l’artérite, la palpation des pouls, la mesure de la tension artérielle prise aux deux bras et aux deux jambes, sans oublier la mesure du périmètre abdominal» , recommande Claire MounierVéhier.  Un bilan biologique standard complète l’ensemble (étude de la glycémie à jeun, bilan lipidique complet, fonction rénale) et si possible, une auto mesure de la tension pour dépister l’hypertension artérielle, très fréquente chez le patient vasculaire.  En cas de symptômes d’alerte évocateurs de maladie coronaire  (douleur constrictive au thorax irradiant dans le bras, la mâchoire ou le creux de l’estomac), l’électrocardiogramme de repos sera complété par une épreuve d’effort. Mieux vaut ne pas attendre l’accident pour agir sur ses facteurs de risque grâce à une bonne hygiène de vie et se faire dépister.  À voir aussi -  C’est quoi un infarctus? \\n         \\n         \\n         \\n         \\n     \\n \\n ']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df[\"source\"] == \"Le Figaro\"][\"body\"].tolist()[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete body special text patterns\n",
    "df[\"body\"] = df[\"body\"].str.replace(u'\\xa0', u' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Title\n",
    "df[\"title\"] = df[\"title\"].str.replace(u'\\xa0', u' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# description\n",
    "df[\"description\"] = df[\"description\"].str.replace(u'\\xa0', u' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_art</th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>theme</th>\n",
       "      <th>description</th>\n",
       "      <th>date_published</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>https://www.telerama.fr/sortir/war-horse-a-la-...</td>\n",
       "      <td>Telerama</td>\n",
       "      <td></td>\n",
       "      <td>“War Horse” à la Seine musicale : un spectacle...</td>\n",
       "      <td>sortir</td>\n",
       "      <td>Admirablement scénographiée, la saga équine ad...</td>\n",
       "      <td>16/12/2019</td>\n",
       "      <td>Étrange qu’il ait fallu attendre douze ans pou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>https://www.liberation.fr/france/2019/12/15/le...</td>\n",
       "      <td>Liberation</td>\n",
       "      <td>Nathalie Raulin</td>\n",
       "      <td>Les chefs de service hospitaliers en appellent...</td>\n",
       "      <td>france</td>\n",
       "      <td>Jugeant très insuffisante la réponse du gouver...</td>\n",
       "      <td>15/12/2019</td>\n",
       "      <td>C’est une menace d’une démission collective qu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>https://www.futura-sciences.com/sante/actualit...</td>\n",
       "      <td>Futura Sciences</td>\n",
       "      <td>Futura avec Relaxnews</td>\n",
       "      <td>L'origine des comportements alimentaires ident...</td>\n",
       "      <td>sante</td>\n",
       "      <td>D'après une étude publiée dans la revue</td>\n",
       "      <td>16/12/2019</td>\n",
       "      <td>D'après une étude publiée dans la revue  Natur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>https://www.futura-sciences.com/tech/actualite...</td>\n",
       "      <td>Futura Sciences</td>\n",
       "      <td>Louis Neveu</td>\n",
       "      <td>Microsoft alerte sur de nouvelles techniques d...</td>\n",
       "      <td>tech</td>\n",
       "      <td>Le géant de l'informatique Microsoft vient de ...</td>\n",
       "      <td>16/12/2019</td>\n",
       "      <td>Le géant de l'informatique Microsoft vient de ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>https://www.futura-sciences.com/sciences/actua...</td>\n",
       "      <td>Futura Sciences</td>\n",
       "      <td>Rémy Decourt</td>\n",
       "      <td>L'Europe se dote d'une infrastructure de téléc...</td>\n",
       "      <td>sciences</td>\n",
       "      <td>Pour garantir la sécurité de la libre circulat...</td>\n",
       "      <td>16/12/2019</td>\n",
       "      <td>Pour garantir la sécurité de la libre circulat...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_art                                                url           source  \\\n",
       "0       1  https://www.telerama.fr/sortir/war-horse-a-la-...         Telerama   \n",
       "1       2  https://www.liberation.fr/france/2019/12/15/le...       Liberation   \n",
       "2       3  https://www.futura-sciences.com/sante/actualit...  Futura Sciences   \n",
       "3       4  https://www.futura-sciences.com/tech/actualite...  Futura Sciences   \n",
       "4       5  https://www.futura-sciences.com/sciences/actua...  Futura Sciences   \n",
       "\n",
       "                  author                                              title  \\\n",
       "0                         “War Horse” à la Seine musicale : un spectacle...   \n",
       "1        Nathalie Raulin  Les chefs de service hospitaliers en appellent...   \n",
       "2  Futura avec Relaxnews  L'origine des comportements alimentaires ident...   \n",
       "3            Louis Neveu  Microsoft alerte sur de nouvelles techniques d...   \n",
       "4           Rémy Decourt  L'Europe se dote d'une infrastructure de téléc...   \n",
       "\n",
       "      theme                                        description date_published  \\\n",
       "0    sortir  Admirablement scénographiée, la saga équine ad...     16/12/2019   \n",
       "1    france  Jugeant très insuffisante la réponse du gouver...     15/12/2019   \n",
       "2     sante           D'après une étude publiée dans la revue      16/12/2019   \n",
       "3      tech  Le géant de l'informatique Microsoft vient de ...     16/12/2019   \n",
       "4  sciences  Pour garantir la sécurité de la libre circulat...     16/12/2019   \n",
       "\n",
       "                                                body  \n",
       "0  Étrange qu’il ait fallu attendre douze ans pou...  \n",
       "1  C’est une menace d’une démission collective qu...  \n",
       "2  D'après une étude publiée dans la revue  Natur...  \n",
       "3  Le géant de l'informatique Microsoft vient de ...  \n",
       "4  Pour garantir la sécurité de la libre circulat...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_art</th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>theme</th>\n",
       "      <th>description</th>\n",
       "      <th>date_published</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>905</th>\n",
       "      <td>906</td>\n",
       "      <td>https://bourse.lefigaro.fr/indices/la-bourse-d...</td>\n",
       "      <td>Le Figaro</td>\n",
       "      <td>Hervé Rousseau</td>\n",
       "      <td>La Bourse de Paris boucle sa meilleure année d...</td>\n",
       "      <td>indices</td>\n",
       "      <td>Le CAC 40 a gagné près de 27 % cette année. Le...</td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>Un millésime à marquer d’une pierre blanche. L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>906</th>\n",
       "      <td>907</td>\n",
       "      <td>https://bourse.lefigaro.fr/indices/ces-entrepr...</td>\n",
       "      <td>Le Figaro</td>\n",
       "      <td>Hervé Rousseau</td>\n",
       "      <td>Ces entreprises qui affolent les compteurs de ...</td>\n",
       "      <td>indices</td>\n",
       "      <td>LVMH, le numéro un mondial du luxe est de très...</td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>Cette année, le CAC 40 doit sa formidable perf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>907</th>\n",
       "      <td>908</td>\n",
       "      <td>https://www.liberation.fr/france/2019/12/30/tc...</td>\n",
       "      <td>Liberation</td>\n",
       "      <td>Cédric Mathiot</td>\n",
       "      <td>Tchernobyl : la réponse la plus longue (qui re...</td>\n",
       "      <td>france</td>\n",
       "      <td></td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>On dit souvent du fact-checking qu’il est né a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>908</th>\n",
       "      <td>909</td>\n",
       "      <td>https://www.liberation.fr/france/2019/12/30/qu...</td>\n",
       "      <td>Liberation</td>\n",
       "      <td>Pauline Moullot, Robin Andraca</td>\n",
       "      <td>Quand «CheckNews» répond aux questions sur «Li...</td>\n",
       "      <td>france</td>\n",
       "      <td>Le service est fréquemment interrogé sur son p...</td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>La tendance se confirme. Depuis le lancement d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>909</th>\n",
       "      <td>910</td>\n",
       "      <td>https://www.liberation.fr/france/2019/12/30/fe...</td>\n",
       "      <td>Liberation</td>\n",
       "      <td>Robin Andraca</td>\n",
       "      <td>Femmes voilées (pas) vues à la télé : la répon...</td>\n",
       "      <td>france</td>\n",
       "      <td></td>\n",
       "      <td>30/12/2019</td>\n",
       "      <td>Il nous arrive, lorsque les circonstances le r...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id_art                                                url      source  \\\n",
       "905     906  https://bourse.lefigaro.fr/indices/la-bourse-d...   Le Figaro   \n",
       "906     907  https://bourse.lefigaro.fr/indices/ces-entrepr...   Le Figaro   \n",
       "907     908  https://www.liberation.fr/france/2019/12/30/tc...  Liberation   \n",
       "908     909  https://www.liberation.fr/france/2019/12/30/qu...  Liberation   \n",
       "909     910  https://www.liberation.fr/france/2019/12/30/fe...  Liberation   \n",
       "\n",
       "                             author  \\\n",
       "905                 Hervé Rousseau    \n",
       "906                 Hervé Rousseau    \n",
       "907                  Cédric Mathiot   \n",
       "908  Pauline Moullot, Robin Andraca   \n",
       "909                   Robin Andraca   \n",
       "\n",
       "                                                 title    theme  \\\n",
       "905  La Bourse de Paris boucle sa meilleure année d...  indices   \n",
       "906  Ces entreprises qui affolent les compteurs de ...  indices   \n",
       "907  Tchernobyl : la réponse la plus longue (qui re...   france   \n",
       "908  Quand «CheckNews» répond aux questions sur «Li...   france   \n",
       "909  Femmes voilées (pas) vues à la télé : la répon...   france   \n",
       "\n",
       "                                           description date_published  \\\n",
       "905  Le CAC 40 a gagné près de 27 % cette année. Le...     30/12/2019   \n",
       "906  LVMH, le numéro un mondial du luxe est de très...     30/12/2019   \n",
       "907                                                        30/12/2019   \n",
       "908  Le service est fréquemment interrogé sur son p...     30/12/2019   \n",
       "909                                                        30/12/2019   \n",
       "\n",
       "                                                  body  \n",
       "905  Un millésime à marquer d’une pierre blanche. L...  \n",
       "906  Cette année, le CAC 40 doit sa formidable perf...  \n",
       "907  On dit souvent du fact-checking qu’il est né a...  \n",
       "908  La tendance se confirme. Depuis le lancement d...  \n",
       "909  Il nous arrive, lorsque les circonstances le r...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1    910\n",
       "Name: body, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check\n",
    "df[\"body\"].str.find(u'\\xa0').value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1    910\n",
       "Name: title, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check\n",
    "df[\"title\"].str.find(u'\\xa0').value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop duplicates for body"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "877"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"body\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop_duplicates(subset=\"body\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "877"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"body\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_art</th>\n",
       "      <th>url</th>\n",
       "      <th>source</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>theme</th>\n",
       "      <th>description</th>\n",
       "      <th>date_published</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>https://www.telerama.fr/sortir/war-horse-a-la-...</td>\n",
       "      <td>Telerama</td>\n",
       "      <td></td>\n",
       "      <td>“War Horse” à la Seine musicale : un spectacle...</td>\n",
       "      <td>sortir</td>\n",
       "      <td>Admirablement scénographiée, la saga équine ad...</td>\n",
       "      <td>16/12/2019</td>\n",
       "      <td>Étrange qu’il ait fallu attendre douze ans pou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>https://www.liberation.fr/france/2019/12/15/le...</td>\n",
       "      <td>Liberation</td>\n",
       "      <td>Nathalie Raulin</td>\n",
       "      <td>Les chefs de service hospitaliers en appellent...</td>\n",
       "      <td>france</td>\n",
       "      <td>Jugeant très insuffisante la réponse du gouver...</td>\n",
       "      <td>15/12/2019</td>\n",
       "      <td>C’est une menace d’une démission collective qu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>https://www.futura-sciences.com/sante/actualit...</td>\n",
       "      <td>Futura Sciences</td>\n",
       "      <td>Futura avec Relaxnews</td>\n",
       "      <td>L'origine des comportements alimentaires ident...</td>\n",
       "      <td>sante</td>\n",
       "      <td>D'après une étude publiée dans la revue</td>\n",
       "      <td>16/12/2019</td>\n",
       "      <td>D'après une étude publiée dans la revue  Natur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>https://www.futura-sciences.com/tech/actualite...</td>\n",
       "      <td>Futura Sciences</td>\n",
       "      <td>Louis Neveu</td>\n",
       "      <td>Microsoft alerte sur de nouvelles techniques d...</td>\n",
       "      <td>tech</td>\n",
       "      <td>Le géant de l'informatique Microsoft vient de ...</td>\n",
       "      <td>16/12/2019</td>\n",
       "      <td>Le géant de l'informatique Microsoft vient de ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>https://www.futura-sciences.com/sciences/actua...</td>\n",
       "      <td>Futura Sciences</td>\n",
       "      <td>Rémy Decourt</td>\n",
       "      <td>L'Europe se dote d'une infrastructure de téléc...</td>\n",
       "      <td>sciences</td>\n",
       "      <td>Pour garantir la sécurité de la libre circulat...</td>\n",
       "      <td>16/12/2019</td>\n",
       "      <td>Pour garantir la sécurité de la libre circulat...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_art                                                url           source  \\\n",
       "0       1  https://www.telerama.fr/sortir/war-horse-a-la-...         Telerama   \n",
       "1       2  https://www.liberation.fr/france/2019/12/15/le...       Liberation   \n",
       "2       3  https://www.futura-sciences.com/sante/actualit...  Futura Sciences   \n",
       "3       4  https://www.futura-sciences.com/tech/actualite...  Futura Sciences   \n",
       "4       5  https://www.futura-sciences.com/sciences/actua...  Futura Sciences   \n",
       "\n",
       "                  author                                              title  \\\n",
       "0                         “War Horse” à la Seine musicale : un spectacle...   \n",
       "1        Nathalie Raulin  Les chefs de service hospitaliers en appellent...   \n",
       "2  Futura avec Relaxnews  L'origine des comportements alimentaires ident...   \n",
       "3            Louis Neveu  Microsoft alerte sur de nouvelles techniques d...   \n",
       "4           Rémy Decourt  L'Europe se dote d'une infrastructure de téléc...   \n",
       "\n",
       "      theme                                        description date_published  \\\n",
       "0    sortir  Admirablement scénographiée, la saga équine ad...     16/12/2019   \n",
       "1    france  Jugeant très insuffisante la réponse du gouver...     15/12/2019   \n",
       "2     sante           D'après une étude publiée dans la revue      16/12/2019   \n",
       "3      tech  Le géant de l'informatique Microsoft vient de ...     16/12/2019   \n",
       "4  sciences  Pour garantir la sécurité de la libre circulat...     16/12/2019   \n",
       "\n",
       "                                                body  \n",
       "0  Étrange qu’il ait fallu attendre douze ans pou...  \n",
       "1  C’est une menace d’une démission collective qu...  \n",
       "2  D'après une étude publiée dans la revue  Natur...  \n",
       "3  Le géant de l'informatique Microsoft vient de ...  \n",
       "4  Pour garantir la sécurité de la libre circulat...  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../../data/df_true_news.pkl'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PATH_DF_TRUE_NEWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "from sklearn.externals import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " df Saved here:\n",
      "../../data/df_true_news.pkl\n"
     ]
    }
   ],
   "source": [
    "# save results\n",
    "joblib.dump(df, PATH_DF_TRUE_NEWS, compress=True)\n",
    "print(\"df Saved here:\\n\" + PATH_DF_TRUE_NEWS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "225px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}