RemoteBuild.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import functools\n",
    "import os\n",
    "import shlex\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from fabric import api as fapi\n",
    "from fabric import network as fnet\n",
    "import paramiko"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "fapi.env.warn_only = True\n",
    "fapi.env.use_ssh_config = True\n",
    "user = fapi.env.user = 'cc'\n",
    "host = fapi.env.host_string = '129.114.108.248'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def run(command, **kwargs):\n",
    "    runargs = {\n",
    "        'stdout': subprocess.PIPE,\n",
    "        'stderr': subprocess.PIPE,\n",
    "        'universal_newlines': True,\n",
    "        'shell': False\n",
    "    }\n",
    "    runargs.update(kwargs)\n",
    "    if not runargs['shell']:\n",
    "        command = shlex.split(command)\n",
    "    return subprocess.run(command, **runargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# proc = subprocess.run('git -C CC-Ubuntu16.04 status', shell=True, \n",
    "#           stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)\n",
    "# print(proc.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import errno\n",
    "import time\n",
    "\n",
    "from paramiko import SSHException\n",
    "from fabric.network import NetworkError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME=\"Ubuntu\"\r\n",
      "VERSION=\"16.04.1 LTS (Xenial Xerus)\"\r\n",
      "ID=ubuntu\r\n",
      "ID_LIKE=debian\r\n",
      "PRETTY_NAME=\"Ubuntu 16.04.1 LTS\"\r\n",
      "VERSION_ID=\"16.04\"\r\n",
      "HOME_URL=\"http://www.ubuntu.com/\"\r\n",
      "SUPPORT_URL=\"http://help.ubuntu.com/\"\r\n",
      "BUG_REPORT_URL=\"http://bugs.launchpad.net/ubuntu/\"\r\n",
      "VERSION_CODENAME=xenial\r\n",
      "UBUNTU_CODENAME=xenial\n"
     ]
    }
   ],
   "source": [
    "# import io\n",
    "# buf = io.BytesIO()\n",
    "# fapi.get('/etc/os-release', buf)\n",
    "# print(buf.getvalue().decode('ascii'))\n",
    "while True:\n",
    "    try:\n",
    "        print(fapi.run('cat /etc/os-release', quiet=True))\n",
    "        break\n",
    "    except OSError as e:\n",
    "        if e.errno == errno.ENETUNREACH:\n",
    "            pass\n",
    "        raise\n",
    "    except (NetworkError, SSHException):\n",
    "        pass\n",
    "    print('.', end='')\n",
    "    time.sleep(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.4.0-57-generic\n"
     ]
    }
   ],
   "source": [
    "print(fapi.run('uname -r', quiet=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On branch gpu\n",
      "nothing to commit, working tree clean\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'299a3ad78331c735014de40baed03cb2c43112cb'"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proc = run('git status', cwd='CC-Ubuntu16.04')\n",
    "print(proc.stdout)\n",
    "head = run('git rev-parse HEAD', cwd='CC-Ubuntu16.04').stdout.strip()\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized empty Git repository in /home/cc/build.git/\n"
     ]
    }
   ],
   "source": [
    "fapi.run('rm -rf ~/build.git', quiet=True)\n",
    "out = fapi.run('git init --bare build.git', quiet=True)\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/npt/Code/Ar/dibs'"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "\n",
      "Warning: Permanently added '129.114.108.248' (ECDSA) to the list of known hosts.\n",
      "To ssh://129.114.108.248/~/build.git\n",
      " * [new branch]      cuda-submission -> cuda-submission\n",
      " * [new branch]      gpu -> gpu\n",
      " * [new branch]      master -> master\n",
      "\n"
     ]
    }
   ],
   "source": [
    "proc = run('git push --all ssh://{}@{}/~/build.git'.format(user, host), cwd='CC-Ubuntu16.04', env={\n",
    "    'GIT_SSH_COMMAND': 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no',\n",
    "})\n",
    "print(proc.returncode)\n",
    "print(proc.stdout)\n",
    "print(proc.stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[129.114.108.248] run: git -C ~/build -c advice.detachedHead=false checkout 299a3ad78331c735014de40baed03cb2c43112cb\n",
      "[129.114.108.248] out: HEAD is now at 299a3ad... Drop 'image type' as it's fixed\n",
      "[129.114.108.248] out: \n",
      "\n",
      ".   create-image.sh  .git\t install-reqs.sh  VERSION\n",
      "..  elements\t     .gitignore  README.md\n"
     ]
    }
   ],
   "source": [
    "fapi.run('rm -rf ~/build', quiet=True)\n",
    "fapi.run('git clone ~/build.git ~/build', quiet=True)\n",
    "fapi.run('git -C ~/build -c advice.detachedHead=false checkout {}'.format(head))\n",
    "print(fapi.run('ls -a build', quiet=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r)\r\n",
      "Requirement already satisfied (use --upgrade to upgrade): pytz>=0a in /usr/lib/python2.7/dist-packages (from Babel>=2.3.4->diskimage-builder)\r\n",
      "\u001b[33mYou are using pip version 8.1.1, however version 9.0.1 is available.\r\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# out = fapi.run('sudo bash ~/build/install-reqs.sh', pty=True, quiet=True)\n",
    "out = fapi.sudo('apt-get -qq install -y qemu-utils python-pip', quiet=True)\n",
    "out = fapi.sudo('pip install diskimage-builder', quiet=True)\n",
    "print(out[-300:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[129.114.108.248] run: bash create-image.sh\n",
      "\n",
      "1013882\n"
     ]
    }
   ],
   "source": [
    "import io\n",
    "out = io.StringIO()\n",
    "# err = io.StringIO()\n",
    "with fapi.cd('/home/cc/build/'):\n",
    "#     out = fapi.run('bash create-image.sh', pty=False, quiet=True)\n",
    "    fapi.run('bash create-image.sh', pty=True, quiet=False, capture_buffer_size=5000, stdout=out)#, stderr=err)\n",
    "    \n",
    "with open('build.log', 'w') as f:\n",
    "    print(f.write(out.getvalue()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.108.248] out: sudo: unable to resolve host image-builder-gpu\n",
      "[129.114.108.248] out: Converting image using qemu-img convert\n",
      "[129.114.108.248] out: Image file /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2 created...\n",
      "[129.114.108.248] out: sudo: unable to resolve host image-builder-gpu\n",
      "[129.114.108.248] out: + '[' -f /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2.qcow2 ']'\n",
      "[129.114.108.248] out: + COMPRESSED_OUTPUT_FILE=/tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2-compressed\n",
      "[129.114.108.248] out: + qemu-img convert /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2 -O qcow2 -c /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2-compressed\n",
      "[129.114.108.248] out: + echo 'mv /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2-compressed /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2'\n",
      "[129.114.108.248] out: mv /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2-compressed /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2\n",
      "[129.114.108.248] out: + mv /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2-compressed /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2\n",
      "[129.114.108.248] out: + '[' 0 -eq 0 ']'\n",
      "[129.114.108.248] out: + echo 'Image built in /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2'\n",
      "[129.114.108.248] out: Image built in /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2\n",
      "[129.114.108.248] out: + '[' -f /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2 ']'\n",
      "[129.114.108.248] out: + echo 'to add the image in glance run the following command:'\n",
      "[129.114.108.248] out: to add the image in glance run the following command:\n",
      "[129.114.108.248] out: + echo 'glance image-create --name \"CC-Ubuntu16.04-CUDA8\" --disk-format qcow2 --container-format bare --file /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2'\n",
      "[129.114.108.248] out: glance image-create --name \"CC-Ubuntu16.04-CUDA8\" --disk-format qcow2 --container-format bare --file /tmp/tmp.7g2B322gMa/common/CC-Ubuntu16.04-CUDA8.qcow2\n",
      "[129.114.108.248] out: \n",
      "\n",
      "1008674\n"
     ]
    }
   ],
   "source": [
    "print(out.getvalue()[-2000:])\n",
    "print(len(out.getvalue()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/tmp/tmp.3VW9wNQiG0/common/CC-Ubuntu16.04.qcow2'"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.seek(0)\n",
    "ibi = f'[{host}] out: Image built in '\n",
    "for line in out:\n",
    "    if not line.startswith(ibi):\n",
    "        continue\n",
    "    output_file = line[len(ibi):].strip()\n",
    "output_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c6c933d5c6d4e25c5399e5cb47327838'"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checksum = fapi.run(f'md5sum {output_file}', quiet=True).split()[0]\n",
    "checksum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "uploaded my RC and ran this command with an SSH shell...\n",
    "\n",
    "```bash\n",
    "glance image-create --name \"test-CC-Ubuntu16.04-CUDA8\" --disk-format qcow2 --container-format bare --file gpu\n",
    "```\n",
    "\n",
    "rebuilt the instance with that image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"**************************************************************************\\r\\n# This system is using the EC2 Metadata Service, but does not appear to  #\\r\\n# be running on Amazon EC2 or one of cloud-init's known platforms that   #\\r\\n# provide a EC2 Metadata service. In the future, cloud-init may stop     #\\r\\n# reading metadata from the EC2 Metadata Service unless the platform can #\\r\\n# be identified.                                                         #\\r\\n#                                                                        #\\r\\n# If you are seeing this message, please file a bug against              #\\r\\n# cloud-init at                                                          #\\r\\n#    https://bugs.launchpad.net/cloud-init/+filebug?field.tags=dsid      #\\r\\n# Make sure to include the cloud provider your instance is               #\\r\\n# running on.                                                            #\\r\\n#                                                                        #\\r\\n# For more information see                                               #\\r\\n#   https://bugs.launchpad.net/bugs/1660385                              #\\r\\n#                                                                        #\\r\\n# After you have filed a bug, you can disable this warning by            #\\r\\n# launching your instance with the cloud-config below, or                #\\r\\n# putting that content into                                              #\\r\\n#    /etc/cloud/cloud.cfg.d/99-ec2-datasource.cfg                        #\\r\\n#                                                                        #\\r\\n# #cloud-config                                                          #\\r\\n# datasource:                                                            #\\r\\n#  Ec2:                                                                  #\\r\\n#   strict_id: false                                                     #\\r\\n**************************************************************************\\r\\n\\r\\nDisable the warnings above by:\\r\\n  touch /home/cc/.cloud-warnings.skip\\r\\nor\\r\\n  touch /var/lib/cloud/instance/warnings/.skip\""
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fapi.run('ls', quiet=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fapi.run('rm -rf ubu ubu.git', quiet=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "options = {\n",
    "    'username': 'cc',\n",
    "    'hostname': '129.114.109.148',\n",
    "}\n",
    "\n",
    "client = paramiko.SSHClient()\n",
    "client._policy = paramiko.WarningPolicy()\n",
    "client.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n",
    "\n",
    "ssh_config = paramiko.SSHConfig()\n",
    "user_config_file = os.path.expanduser(\"~/.ssh/config\")\n",
    "if os.path.exists(user_config_file):\n",
    "    with open(user_config_file) as f:\n",
    "        ssh_config.parse(f)\n",
    "\n",
    "cfg = {'hostname': options['hostname'], 'username': options[\"username\"]}\n",
    "\n",
    "user_config = ssh_config.lookup(cfg['hostname'])\n",
    "for k in ('hostname', 'username', 'port'):\n",
    "    if k in user_config:\n",
    "        cfg[k] = user_config[k]\n",
    "\n",
    "if 'proxycommand' in user_config:\n",
    "    cfg['sock'] = paramiko.ProxyCommand(user_config['proxycommand'])\n",
    "\n",
    "client.connect(**cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "stdin, stdout, stderr = client.exec_command('ls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b''"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stdout.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "OS Clients (3.6)",
   "language": "python",
   "name": "os36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}