git2hash.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Git2Hash\n",
    "For a local git repo, extract the hashes and a synthetic VirusTotal link for all files in the commit history of the master branch.\n",
    "\n",
    "The purpose of this is to extract information about files that have been deleted from the current master branch on GitHub.\n",
    "\n",
    "This requires these modules in addition to pandas:\n",
    "\n",
    "- gitpython\n",
    "- filemagic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "from git import Repo\n",
    "import magic\n",
    "import hashlib\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/tmp/sample'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linkify_vt(val):\n",
    "    return '<a target=\"_blank\" href=\"{}\">virustotal</a>'.format(val)\n",
    "\n",
    "def git2hash(path):\n",
    "    r = Repo(path)\n",
    "    file_objects = {}\n",
    "    for t in r.iter_trees():\n",
    "        for blob in t.blobs:\n",
    "            obj = blob.data_stream.read()\n",
    "            with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m:\n",
    "                mime_type = m.id_buffer(obj[:4194304])\n",
    "            sha256 = hashlib.sha256(obj).hexdigest()\n",
    "            sha1 = hashlib.sha1(obj).hexdigest()\n",
    "            md5 = hashlib.md5(obj).hexdigest()\n",
    "            file_objects[sha256] = {'sha256':sha256,\n",
    "                                    'name':blob.name,\n",
    "                                    'size':blob.size,\n",
    "                                    'md5':md5,\n",
    "                                    'sha1':sha1,\n",
    "                                    'mime_type':mime_type,\n",
    "                                    'vt':'https://www.virustotal.com/gui/file/{}/detection'.format(sha256)}\n",
    "    df = pd.DataFrame(list(file_objects.values()),\n",
    "                      columns=['name','mime_type','sha256','vt','size','md5','sha1'])\n",
    "\n",
    "    \n",
    "    return df.style.format({'vt': linkify_vt})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = git2hash(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >name</th>        <th class=\"col_heading level0 col1\" >mime_type</th>        <th class=\"col_heading level0 col2\" >sha256</th>        <th class=\"col_heading level0 col3\" >vt</th>        <th class=\"col_heading level0 col4\" >size</th>        <th class=\"col_heading level0 col5\" >md5</th>        <th class=\"col_heading level0 col6\" >sha1</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row0_col0\" class=\"data row0 col0\" >..exe</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row0_col1\" class=\"data row0 col1\" >application/x-dosexec</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row0_col2\" class=\"data row0 col2\" >c4eada327d83caebe0929b3aa638db533a2d30c4ef15a3dc4f445245dfd53797</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row0_col3\" class=\"data row0 col3\" ><a target=\"_blank\" href=\"https://www.virustotal.com/gui/file/c4eada327d83caebe0929b3aa638db533a2d30c4ef15a3dc4f445245dfd53797/detection\">virustotal</a></td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row0_col4\" class=\"data row0 col4\" >156104</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row0_col5\" class=\"data row0 col5\" >a6763ae35acd41ec0f50bdfcc559d83b</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row0_col6\" class=\"data row0 col6\" >7ad583aa228ab1cc01af4d69b8a1256d3ffbef23</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row1_col0\" class=\"data row1 col0\" >S258745.cmd</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row1_col1\" class=\"data row1 col1\" >text/x-msdos-batch</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row1_col2\" class=\"data row1 col2\" >9635850e30112bd8427ece8a738bf8e93b37ebcd1c48c4ce501fa3924cdb8742</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row1_col3\" class=\"data row1 col3\" ><a target=\"_blank\" href=\"https://www.virustotal.com/gui/file/9635850e30112bd8427ece8a738bf8e93b37ebcd1c48c4ce501fa3924cdb8742/detection\">virustotal</a></td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row1_col4\" class=\"data row1 col4\" >1293</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row1_col5\" class=\"data row1 col5\" >14791338aa4495f24044c3cecadaca50</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row1_col6\" class=\"data row1 col6\" >565c1d84cc1091a2105f00d98d5a8018d27c7934</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row2_col0\" class=\"data row2 col0\" >README.md</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row2_col1\" class=\"data row2 col1\" >text/plain</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row2_col2\" class=\"data row2 col2\" >a0c81baa03e82fbab08f1258c767f44fc26adc7f6833cb0c352668f1d7a70c74</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row2_col3\" class=\"data row2 col3\" ><a target=\"_blank\" href=\"https://www.virustotal.com/gui/file/a0c81baa03e82fbab08f1258c767f44fc26adc7f6833cb0c352668f1d7a70c74/detection\">virustotal</a></td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row2_col4\" class=\"data row2 col4\" >12</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row2_col5\" class=\"data row2 col5\" >1edbbc0a486566abbff55e1da8218309</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row2_col6\" class=\"data row2 col6\" >82be576aa21709abcb9994f238282bcfbd53cdb0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row3_col0\" class=\"data row3 col0\" >LICENSE</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row3_col1\" class=\"data row3 col1\" >text/plain</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row3_col2\" class=\"data row3 col2\" >1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row3_col3\" class=\"data row3 col3\" ><a target=\"_blank\" href=\"https://www.virustotal.com/gui/file/1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5/detection\">virustotal</a></td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row3_col4\" class=\"data row3 col4\" >16725</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row3_col5\" class=\"data row3 col5\" >9741c346eef56131163e13b9db1241b3</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row3_col6\" class=\"data row3 col6\" >d22157abc0fc0b4ae96380c09528e23cf77290a9</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row4_col0\" class=\"data row4 col0\" >.gitignore</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row4_col1\" class=\"data row4 col1\" >text/plain</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row4_col2\" class=\"data row4 col2\" >3ba2020da55f76620d61950cb88f0d6cc0d5852eae7b2c03f8487a64c87da959</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row4_col3\" class=\"data row4 col3\" ><a target=\"_blank\" href=\"https://www.virustotal.com/gui/file/3ba2020da55f76620d61950cb88f0d6cc0d5852eae7b2c03f8487a64c87da959/detection\">virustotal</a></td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row4_col4\" class=\"data row4 col4\" >270</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row4_col5\" class=\"data row4 col5\" >2b61ce16bc0d8f33d40ce4c8bc5e90cf</td>\n",
       "                        <td id=\"T_49d5020e_81ae_11ea_9a8a_acde48001122row4_col6\" class=\"data row4 col6\" >99d352abda5528933a0b16e017c9ef1c2a275951</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x11d094d10>"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}