ryan-xpcs-crawl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pickle\n",
    "import time\n",
    "import json\n",
    "import mdf_toolbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Headers: {'Authorization': 'Bearer AgrqGv4oypjr5xvQDpD0dEmQoDXVbOqjd0k2xjK8lrPvrq4nB1FXCEv0KGlbw01BOY7KYngbKmkNblTlBgMVrtvo2d', 'Search': 'Ag4087QMzvjBqapQpX5OPEVNQwj45vzqYlpBm0bl3BaqWo01NQt7CPDajPdOjNkV5voPB8yaBQBVx9hjgeNl5S5K22ToNW4', 'Openid': 'AgyBMokw8J144VlWmarD6joeg8EgWlpwkp3aVB8y0eX7rxrQQguVC1n9l2jY0KV3zwDDy0X1b9weK1SOGvrlPCp6Vn'}\n"
     ]
    }
   ],
   "source": [
    "# eb_url = \"http://xtractcrawler5-env.eba-akbhvznm.us-east-1.elasticbeanstalk.com/\"\n",
    "eb_url = \"http://127.0.0.1:5000/\"\n",
    "grouper = \"file_is_group\"\n",
    "\n",
    "auths = mdf_toolbox.login(\n",
    "    services=[\n",
    "        \"openid\",\n",
    "        \"data_mdf\",\n",
    "        \"search\",\n",
    "        \"petrel\",\n",
    "        \"transfer\",\n",
    "        \"dlhub\",\n",
    "        \"https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all\",\n",
    "    ],\n",
    "    app_name=\"Foundry\",\n",
    "    make_clients=True,\n",
    "    no_browser=False,\n",
    "    no_local_server=False,\n",
    "    # force=True\n",
    ")\n",
    "\n",
    "# print(auths['search'])\n",
    "# print(dir(auths['search']))\n",
    "\n",
    "# print(auths)\n",
    "fx_scope = 'https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all'\n",
    "headers = {'Authorization': f\"Bearer {auths['petrel']}\", 'Transfer': auths['transfer'], 'FuncX': auths[fx_scope], 'Petrel': auths['petrel']}\n",
    "fx_headers = {'Authorization': f\"Bearer {auths[fx_scope].access_token}\", \n",
    "             'Search': auths['search'].authorizer.access_token, \n",
    "             'Openid': auths['openid'].access_token}\n",
    "print(f\"Headers: {str(fx_headers)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Crawl URL is : http://127.0.0.1:5000//crawl\n",
      "{'Transfer': 'AgP9wb9XnjYlJOxVoO5l48We7lMxMk2BrOQMrdj1zoqOOr2GlaTmCb74djPYKBQdJK0O5BYg5KgbBMcqzVvmNH8kND', 'Authorization': 'Bearer Ag06qlEVbj09jeGd7DpnKqex2j0X84wXgjymgVGyN4qgaGPev7s7CBp1vYojgEWWzPx979oog56BywswreDkQU8dn0', 'FuncX': 'AgrqGv4oypjr5xvQDpD0dEmQoDXVbOqjd0k2xjK8lrPvrq4nB1FXCEv0KGlbw01BOY7KYngbKmkNblTlBgMVrtvo2d'}\n",
      "b'{\"crawl_id\":\"065048b7-2d25-48b0-bb6d-ff3866ff5cf3\",\"status\":\"200 (OK)\"}\\n'\n",
      "Crawl ID: 065048b7-2d25-48b0-bb6d-ff3866ff5cf3\n"
     ]
    }
   ],
   "source": [
    "# Now do the actual crawl \n",
    "cycle = \"2018-3\"\n",
    "globus_eid = \"e55b4eab-6d04-11e5-ba46-22000b92c6ec\"\n",
    "pub8_path = f\"/XPCSDATA/{cycle}\"\n",
    "funcx_eid = \"ff24c520-a419-45ba-a96c-5e6091878df0\"\n",
    "\n",
    "skip_list = [\".imm\", \".IMM\", \".batchinfo\"]\n",
    "\n",
    "crawl_url = f'{eb_url}/crawl'\n",
    "print(f\"Crawl URL is : {crawl_url}\")\n",
    "\n",
    "first_ep_dict = {\n",
    "    'repo_type': 'GLOBUS',\n",
    "    'eid': globus_eid,\n",
    "    'fx_eid': funcx_eid,\n",
    "    'dir_paths': [pub8_path],# , source_ep_path_2],\n",
    "    'grouper': grouper, \n",
    "    # 'prefetch_list':\n",
    "    #     [{'dest_ep_id': dest_ep_id, 'path': '/project2/chard/skluzacek/data_to_process'}]\n",
    "}\n",
    "\n",
    "tokens = {'Transfer': auths['transfer'].authorizer.access_token, \n",
    "          'Authorization': f\"Bearer {auths['petrel'].access_token}\", \n",
    "          'FuncX': auths[fx_scope].access_token}\n",
    "print(tokens)\n",
    "\n",
    "crawl_req = requests.post(f'{eb_url}/crawl', json={'endpoints': [first_ep_dict], \n",
    "                                                   'skip_list': skip_list,\n",
    "                                                   'tokens': tokens})\n",
    "print(crawl_req.content)\n",
    "crawl_id = json.loads(crawl_req.content)['crawl_id']\n",
    "print(f\"Crawl ID: {crawl_id}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 115168811132, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 132, 'groups_crawled': 130}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 136434491665, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 447, 'groups_crawled': 447}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 156517471978, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 768, 'groups_crawled': 768}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 170352996220, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 993, 'groups_crawled': 993}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 190453157103, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 1332, 'groups_crawled': 1329}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 227865849289, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 1683, 'groups_crawled': 1683}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 266943959685, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 2037, 'groups_crawled': 2037}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 295083771360, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 2397, 'groups_crawled': 2397}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 323993985542, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 2769, 'groups_crawled': 2769}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 348044166946, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 3138, 'groups_crawled': 3138}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 367337248079, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 3447, 'groups_crawled': 3447}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 388170079105, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 3800, 'groups_crawled': 3800}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 406752974432, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 4121, 'groups_crawled': 4121}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 432290006805, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 4523, 'groups_crawled': 4523}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 456683250940, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 4934, 'groups_crawled': 4934}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 482047352351, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 5342, 'groups_crawled': 5342}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 508902889932, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 5762, 'groups_crawled': 5762}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 536287610476, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 6176, 'groups_crawled': 6176}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 563323015023, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 6587, 'groups_crawled': 6587}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 589012983800, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 6992, 'groups_crawled': 6992}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 608527718744, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 7328, 'groups_crawled': 7328}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 850103865973, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 9169, 'groups_crawled': 9169}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 860332848505, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 9439, 'groups_crawled': 9439}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 870057334465, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 9703, 'groups_crawled': 9703}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 925453075639, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 9963, 'groups_crawled': 9963}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 991249881249, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 10360, 'groups_crawled': 10360}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 1143792496113, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 10926, 'groups_crawled': 10926}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 1250469516938, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 11325, 'groups_crawled': 11325}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 1371967479014, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 11700, 'groups_crawled': 11700}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 1488402604231, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 12093, 'groups_crawled': 12093}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 1609754749035, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 12498, 'groups_crawled': 12498}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 1719448041745, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 12917, 'groups_crawled': 12917}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 1827047702155, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 13337, 'groups_crawled': 13337}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 2827451388247, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 16202, 'groups_crawled': 16202}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3149354556217, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 16613, 'groups_crawled': 16613}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3264045056412, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 18106, 'groups_crawled': 18106}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3272191540278, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 18498, 'groups_crawled': 18498}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3279141659082, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 18840, 'groups_crawled': 18840}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3287123111469, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 19251, 'groups_crawled': 19251}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3295371790306, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 19647, 'groups_crawled': 19647}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3301538862717, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 20034, 'groups_crawled': 20034}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3307538895567, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 20439, 'groups_crawled': 20439}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 3312642396937, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 20853, 'groups_crawled': 20853}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4291593053559, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 22179, 'groups_crawled': 22179}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4309574221035, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 22465, 'groups_crawled': 22465}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4327453701939, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 22749, 'groups_crawled': 22749}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4343717400155, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 23009, 'groups_crawled': 23009}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4359783660239, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 23267, 'groups_crawled': 23267}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4376678983435, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 23539, 'groups_crawled': 23539}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4394094710443, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 23819, 'groups_crawled': 23819}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4410973133411, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 24093, 'groups_crawled': 24093}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4428069372807, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 24373, 'groups_crawled': 24373}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4445026609751, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 24651, 'groups_crawled': 24651}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4460775197503, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 24909, 'groups_crawled': 24909}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4477238174883, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 25179, 'groups_crawled': 25179}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4493323999951, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 25447, 'groups_crawled': 25447}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4509558993019, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 25719, 'groups_crawled': 25719}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4525463933991, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 25985, 'groups_crawled': 25985}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4542138169635, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 26263, 'groups_crawled': 26263}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4556707581075, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 26507, 'groups_crawled': 26507}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4572977553231, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 26785, 'groups_crawled': 26785}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4590991749083, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 27063, 'groups_crawled': 27063}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4605286791831, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 27287, 'groups_crawled': 27287}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4623347376359, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 27571, 'groups_crawled': 27571}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4640841450795, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 27845, 'groups_crawled': 27845}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 4648380638115, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 28105, 'groups_crawled': 28105}\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 5165506675739, 'crawl_id': '065048b7-2d25-48b0-bb6d-ff3866ff5cf3', 'crawl_status': 'crawling', 'files_crawled': 34726, 'groups_crawled': 33837}\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-80-1e951d74353c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Now monitor status until crawl is complete\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mcrawl_status\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'{eb_url}/get_crawl_status'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'crawl_id'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcrawl_id\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcrawl_status\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Now monitor status until crawl is complete\n",
    "while True: \n",
    "    time.sleep(5)\n",
    "    crawl_status = requests.get(f'{eb_url}/get_crawl_status', json={'crawl_id': crawl_id})\n",
    "    print(crawl_status)\n",
    "    crawl_content = json.loads(crawl_status.content)\n",
    "    print(f\"Crawl Status: {crawl_content}\")\n",
    "    if crawl_content['crawl_status'] =='complete':\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38144\n",
      "8950\n",
      "7390\n",
      "38144\n",
      "0\n",
      "2319\n",
      "10102630400\n",
      "2638\n",
      "1306\n",
      "4764\n",
      "3486\n",
      "1711\n",
      "1541413268\n",
      "38144\n",
      "281051144\n",
      "38144\n",
      "38144\n",
      "38144\n",
      "38144\n",
      "1914\n",
      "1675336\n",
      "7385\n",
      "2390\n",
      "1711\n",
      "135350668\n",
      "132644048\n",
      "132296160\n",
      "3826\n",
      "4930\n",
      "2868\n",
      "1792\n",
      "17090095\n",
      "288227990\n",
      "38144\n",
      "38144\n",
      "38144\n",
      "1877\n",
      "292262780\n",
      "294317570\n",
      "308072222\n",
      "0\n",
      "2300\n",
      "1692\n",
      "89496094720\n",
      "132902644\n",
      "132706392\n",
      "132711648\n",
      "132650684\n",
      "38144\n",
      "1879\n",
      "1914\n",
      "1912\n",
      "305619488\n",
      "800\n",
      "38144\n",
      "208444016\n",
      "1879\n",
      "38144\n",
      "201583340\n",
      "38144\n",
      "6148\n",
      "4096\n",
      "2188864180\n",
      "292497\n",
      "7687\n",
      "37872\n",
      "1237039328\n",
      "1821\n",
      "38144\n",
      "288125576\n",
      "199260548\n",
      "1879\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-81-e0d563c9c222>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0mcsv_writer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwriterow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrawl_timestamp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;31m# Sleep so I don't crash my service.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m         \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Fetch all of the metadata from SQS\n",
    "import time\n",
    "import csv\n",
    "#print(dir(auths['transfer'].authorizer))\n",
    "\n",
    "with open(f\"/Users/tylerskluzacek/Desktop/xpcs_crawl_info_{cycle}.csv\", 'w') as f:\n",
    "    field_names = ['petrel_path', 'crawl_timestamp', 'size']\n",
    "    csv_writer = csv.writer(f)\n",
    "    \n",
    "    csv_writer.writerow(field_names)\n",
    "    file_count = 0\n",
    "    while True: \n",
    "        poller = requests.post(f'{eb_url}/fetch_crawl_mdata', json={'crawl_id': crawl_id, \n",
    "                                                                    'Transfer': auths['transfer'].authorizer.access_token, \n",
    "                                                                    'n': 1000})\n",
    "        file_batch = json.loads(poller.content)['file_ls']\n",
    "        for file in file_batch:\n",
    "            file_count += 1\n",
    "            path = file['path']\n",
    "            size = file['size']\n",
    "            print(size)\n",
    "\n",
    "            crawl_timestamp = file['crawl_timestamp']\n",
    "            csv_writer.writerow([path, crawl_timestamp, size])\n",
    "        # Sleep so I don't crash my service.\n",
    "        time.sleep(0.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (plz)",
   "language": "python",
   "name": "plz"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}