{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '/Users/joel/src/thundr/tracker')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "l = logging.getLogger('tracker.s3')\n",
    "l.setLevel(logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from importlib import reload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tracker import config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "config.insta_username = 'joellst.in'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'tracker.s3' from '/Users/joel/src/thundr/tracker/tracker/s3.py'>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tracker.instagram import instascrape as insta\n",
    "from tracker import s3\n",
    "reload(insta)\n",
    "reload(s3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "driver = insta.get_configured_driver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "driver.get('https://whtlst.in')\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename = \"/tmp/{prefix}_{ts:%Y%m%d}_{ts:%H%M%S}_{ts:%f}.png\".format(\n",
    "    prefix='jupyter', ts=datetime.today()\n",
    ")\n",
    "driver.save_screenshot(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename = \"/tmp/{prefix}_{ts:%Y%m%d}_{ts:%H%M%S}_{ts:%f}.png\".format(\n",
    "    prefix='jupyter', ts=datetime.today()\n",
    ")\n",
    "driver.save_screenshot(filename)\n",
    "from IPython.display import Image\n",
    "from IPython.core.display import HTML \n",
    "# Image(url=r, width=200)\n",
    "Image(filename=filename, width=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "driver.quit()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'tracker.instagram.instascrape' from '/Users/joel/src/thundr/tracker/tracker/instagram/instascrape.py'>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(insta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "insta._login_to_instagram(driver, 'joellst.in', 'thisisinstagram')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# artist_inid = 'sd.muni'\n",
    "# artist_inid = 'thatshitgraye'\n",
    "# artist_inid = 'joesefjoesefjoesef'\n",
    "# artist_inid = 'quarrywastaken'\n",
    "artist_inid = 'mathilda_homer'\n",
    "artist_inid = 'muguncov'\n",
    "artist_inid = 'rushyfromthe3'\n",
    "artist_inid = 'ale.e.sha'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "driver.get('https://instagram.com/{}/'.format(artist_inid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "driver.find_element_by_partial_link_text(\"followers\").click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "els = driver.find_element_by_css_selector(\"div[role=dialog]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dialoghtml = els.get_attribute(\"innerHTML\")\n",
    "soup = BeautifulSoup(dialoghtml, \"html5lib\")\n",
    "sib = [\n",
    "    d\n",
    "    for d in soup.find_all(\"div\")\n",
    "    if d.text.strip() == (\"Followers\")\n",
    "][0]\n",
    "scrollClass = list(sib.parent.children)[-1]\n",
    "scroll = scrollClass.attrs[\"class\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'isgrP'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scroll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scrolling... 0\n",
      "Now 36\n",
      "Scrolling... 1\n",
      "Now 48\n",
      "Scrolling... 2\n",
      "Now 60\n",
      "Scrolling... 3\n",
      "Now 72\n",
      "Scrolling... 4\n",
      "Now 84\n",
      "Scrolling... 5\n",
      "Now 96\n",
      "Scrolling... 6\n",
      "Now 108\n",
      "Scrolling... 7\n",
      "Now 120\n",
      "Scrolling... 8\n",
      "Now 132\n",
      "Scrolling... 9\n",
      "Now 144\n",
      "Scrolling... 10\n",
      "Now 156\n",
      "Scrolling... 11\n",
      "Now 168\n",
      "Scrolling... 12\n",
      "Now 180\n",
      "Scrolling... 13\n",
      "Now 192\n",
      "Scrolling... 14\n",
      "Now 204\n",
      "Scrolling... 15\n",
      "Now 216\n",
      "Scrolling... 16\n",
      "Now 228\n",
      "Scrolling... 17\n",
      "Now 240\n",
      "Scrolling... 18\n",
      "Now 252\n",
      "Scrolling... 19\n",
      "Now 264\n",
      "Scrolling... 20\n",
      "Now 276\n",
      "Scrolling... 21\n",
      "Now 288\n",
      "Scrolling... 22\n",
      "Now 300\n",
      "Scrolling... 23\n",
      "Now 312\n",
      "Scrolling... 24\n",
      "Now 324\n",
      "Scrolling... 25\n",
      "Now 336\n",
      "Scrolling... 26\n",
      "Now 348\n",
      "Scrolling... 27\n",
      "Now 360\n",
      "Scrolling... 28\n",
      "Now 372\n",
      "Scrolling... 29\n",
      "Now 384\n",
      "Scrolling... 30\n",
      "Now 396\n",
      "Scrolling... 31\n",
      "Now 408\n",
      "Scrolling... 32\n",
      "Now 420\n",
      "Scrolling... 33\n",
      "Now 432\n",
      "Scrolling... 34\n",
      "Now 444\n",
      "Scrolling... 35\n",
      "Now 456\n",
      "Scrolling... 36\n",
      "Now 468\n",
      "Scrolling... 37\n",
      "Now 480\n",
      "Scrolling... 38\n",
      "Now 492\n",
      "Scrolling... 39\n",
      "Now 504\n",
      "Scrolling... 40\n",
      "Now 516\n",
      "Scrolling... 41\n",
      "Now 528\n",
      "Scrolling... 42\n",
      "Now 540\n",
      "Scrolling... 43\n",
      "Now 552\n",
      "Scrolling... 44\n",
      "Now 564\n",
      "Scrolling... 45\n",
      "Now 576\n",
      "Scrolling... 46\n",
      "Now 588\n",
      "Scrolling... 47\n",
      "Now 600\n",
      "Scrolling... 48\n",
      "Now 612\n",
      "Scrolling... 49\n",
      "Now 624\n",
      "Scrolling... 50\n",
      "Now 636\n",
      "Scrolling... 51\n",
      "Now 648\n",
      "Scrolling... 52\n",
      "Now 660\n",
      "Scrolling... 53\n",
      "Now 672\n",
      "Scrolling... 54\n",
      "Now 684\n",
      "Scrolling... 55\n",
      "Now 696\n",
      "Scrolling... 56\n",
      "Now 708\n",
      "Scrolling... 57\n",
      "Now 720\n",
      "Scrolling... 58\n",
      "Now 732\n",
      "Scrolling... 59\n",
      "Now 744\n",
      "Scrolling... 60\n",
      "Now 756\n",
      "Scrolling... 61\n",
      "Now 768\n",
      "Scrolling... 62\n",
      "Now 780\n",
      "Scrolling... 63\n",
      "Now 792\n",
      "Scrolling... 64\n",
      "Now 804\n",
      "Scrolling... 65\n",
      "Now 816\n",
      "Scrolling... 66\n",
      "Now 828\n",
      "Scrolling... 67\n",
      "Now 840\n",
      "Scrolling... 68\n",
      "Now 852\n",
      "Scrolling... 69\n",
      "Now 864\n",
      "Scrolling... 70\n",
      "Now 876\n",
      "Scrolling... 71\n",
      "Now 888\n",
      "Scrolling... 72\n",
      "Now 900\n",
      "Scrolling... 73\n",
      "Now 912\n",
      "Scrolling... 74\n",
      "Now 924\n",
      "Scrolling... 75\n",
      "Now 936\n",
      "Scrolling... 76\n",
      "Now 948\n",
      "Scrolling... 77\n",
      "Now 960\n",
      "Scrolling... 78\n",
      "Now 972\n",
      "Scrolling... 79\n",
      "Now 984\n",
      "Scrolling... 80\n",
      "Now 996\n",
      "Scrolling... 81\n",
      "Now 1008\n",
      "Scrolling... 82\n",
      "Now 1020\n",
      "Scrolling... 83\n",
      "Now 1032\n",
      "Scrolling... 84\n",
      "Now 1044\n",
      "Scrolling... 85\n",
      "Now 1056\n",
      "Scrolling... 86\n",
      "Now 1068\n",
      "Scrolling... 87\n",
      "Now 1080\n",
      "Scrolling... 88\n",
      "Now 1092\n",
      "Scrolling... 89\n",
      "Now 1104\n",
      "Scrolling... 90\n",
      "Now 1116\n",
      "Scrolling... 91\n",
      "Now 1128\n",
      "Scrolling... 92\n",
      "Now 1140\n",
      "Scrolling... 93\n",
      "Now 1152\n",
      "Scrolling... 94\n",
      "Now 1164\n",
      "Scrolling... 95\n",
      "Now 1176\n",
      "Scrolling... 96\n",
      "Now 1188\n",
      "Scrolling... 97\n",
      "Now 1200\n",
      "Scrolling... 98\n",
      "Now 1212\n",
      "Scrolling... 99\n",
      "Now 1224\n",
      "Scrolling... 100\n",
      "Now 1236\n",
      "Scrolling... 101\n",
      "Now 1248\n",
      "Scrolling... 102\n",
      "Now 1260\n",
      "Scrolling... 103\n",
      "Now 1272\n",
      "Scrolling... 104\n",
      "Now 1284\n",
      "Scrolling... 105\n",
      "Now 1296\n",
      "Scrolling... 106\n",
      "Now 1308\n",
      "Scrolling... 107\n",
      "Now 1320\n",
      "Scrolling... 108\n",
      "Now 1332\n",
      "Scrolling... 109\n",
      "Now 1344\n",
      "Scrolling... 110\n",
      "Now 1356\n",
      "Scrolling... 111\n",
      "Now 1368\n",
      "Scrolling... 112\n",
      "Now 1380\n",
      "Scrolling... 113\n",
      "Now 1392\n",
      "Scrolling... 114\n",
      "Now 1404\n",
      "Scrolling... 115\n",
      "Now 1416\n",
      "Scrolling... 116\n",
      "Now 1428\n",
      "Scrolling... 117\n",
      "Now 1440\n",
      "Scrolling... 118\n",
      "Now 1452\n",
      "Scrolling... 119\n",
      "Now 1464\n",
      "Scrolling... 120\n",
      "Now 1476\n",
      "Scrolling... 121\n",
      "Now 1488\n",
      "Scrolling... 122\n",
      "Now 1500\n",
      "Scrolling... 123\n",
      "Now 1512\n",
      "Scrolling... 124\n",
      "Now 1524\n",
      "Scrolling... 125\n",
      "Now 1536\n",
      "Scrolling... 126\n",
      "Now 1548\n",
      "Scrolling... 127\n",
      "Now 1560\n",
      "Scrolling... 128\n",
      "Now 1572\n",
      "Scrolling... 129\n",
      "Now 1584\n",
      "Scrolling... 130\n",
      "Now 1596\n",
      "Scrolling... 131\n",
      "Now 1608\n",
      "Scrolling... 132\n",
      "Now 1620\n",
      "Scrolling... 133\n",
      "Now 1632\n",
      "Scrolling... 134\n",
      "Now 1644\n",
      "Scrolling... 135\n",
      "Now 1656\n",
      "Scrolling... 136\n",
      "Now 1668\n",
      "Scrolling... 137\n",
      "Now 1680\n",
      "Scrolling... 138\n",
      "Now 1692\n",
      "Scrolling... 139\n",
      "Now 1704\n",
      "Scrolling... 140\n",
      "Now 1716\n",
      "Scrolling... 141\n",
      "Now 1728\n",
      "Scrolling... 142\n",
      "Now 1740\n",
      "Scrolling... 143\n",
      "Now 1752\n",
      "Scrolling... 144\n",
      "Now 1764\n",
      "Scrolling... 145\n",
      "Now 1776\n",
      "Scrolling... 146\n",
      "Now 1788\n",
      "Scrolling... 147\n",
      "Now 1800\n",
      "Scrolling... 148\n",
      "Now 1812\n",
      "Scrolling... 149\n",
      "Now 1824\n",
      "Scrolling... 150\n",
      "Now 1836\n",
      "Scrolling... 151\n",
      "Now 1848\n",
      "Scrolling... 152\n",
      "Now 1860\n",
      "Scrolling... 153\n",
      "Now 1872\n",
      "Scrolling... 154\n",
      "Now 1884\n",
      "Scrolling... 155\n",
      "Now 1896\n",
      "Scrolling... 156\n",
      "Now 1908\n",
      "Scrolling... 157\n",
      "Now 1920\n",
      "Scrolling... 158\n",
      "Now 1932\n",
      "Scrolling... 159\n",
      "Now 1944\n",
      "Scrolling... 160\n",
      "Now 1956\n",
      "Scrolling... 161\n",
      "Now 1968\n",
      "Scrolling... 162\n",
      "Now 1980\n",
      "Scrolling... 163\n",
      "Now 1992\n",
      "Scrolling... 164\n",
      "Now 2004\n",
      "Scrolling... 165\n",
      "Now 2016\n",
      "Scrolling... 166\n",
      "Now 2028\n",
      "Scrolling... 167\n",
      "Now 2040\n",
      "Scrolling... 168\n",
      "Now 2052\n",
      "Scrolling... 169\n",
      "Now 2064\n",
      "Scrolling... 170\n",
      "Now 2076\n",
      "Scrolling... 171\n",
      "Now 2088\n",
      "Scrolling... 172\n",
      "Now 2100\n",
      "Scrolling... 173\n",
      "Now 2112\n",
      "Scrolling... 174\n",
      "Now 2124\n",
      "Scrolling... 175\n",
      "Now 2136\n",
      "Scrolling... 176\n",
      "Now 2148\n",
      "Scrolling... 177\n",
      "Now 2160\n",
      "Scrolling... 178\n",
      "Now 2172\n",
      "Scrolling... 179\n",
      "Now 2184\n",
      "Scrolling... 180\n",
      "Now 2196\n",
      "Scrolling... 181\n",
      "Now 2208\n",
      "Scrolling... 182\n",
      "Now 2220\n",
      "Scrolling... 183\n",
      "Now 2232\n",
      "Scrolling... 184\n",
      "Now 2244\n",
      "Scrolling... 185\n",
      "Now 2256\n",
      "Scrolling... 186\n",
      "Now 2268\n",
      "Scrolling... 187\n",
      "Now 2280\n",
      "Scrolling... 188\n",
      "Now 2292\n",
      "Scrolling... 189\n",
      "Now 2304\n",
      "Scrolling... 190\n",
      "Now 2316\n",
      "Scrolling... 191\n",
      "Now 2328\n",
      "Scrolling... 192\n",
      "Now 2340\n",
      "Scrolling... 193\n",
      "Now 2352\n",
      "Scrolling... 194\n",
      "Now 2364\n",
      "Scrolling... 195\n",
      "Now 2376\n",
      "Scrolling... 196\n",
      "Now 2388\n",
      "Scrolling... 197\n",
      "Now 2400\n",
      "Scrolling... 198\n",
      "Now 2412\n",
      "Scrolling... 199\n",
      "Now 2424\n",
      "Scrolling... 200\n",
      "Now 2436\n",
      "Scrolling... 201\n",
      "Now 2448\n",
      "Scrolling... 202\n",
      "Now 2460\n",
      "Scrolling... 203\n",
      "Now 2472\n",
      "Scrolling... 204\n",
      "Now 2484\n",
      "Scrolling... 205\n",
      "Now 2496\n",
      "Scrolling... 206\n",
      "Now 2508\n",
      "Scrolling... 207\n",
      "Now 2520\n",
      "Scrolling... 208\n",
      "Now 2532\n",
      "Scrolling... 209\n",
      "Now 2544\n",
      "Scrolling... 210\n",
      "Now 2556\n",
      "Scrolling... 211\n",
      "Now 2568\n",
      "Scrolling... 212\n",
      "Now 2580\n",
      "Scrolling... 213\n",
      "Now 2592\n",
      "Scrolling... 214\n",
      "Now 2604\n",
      "Scrolling... 215\n",
      "Now 2616\n",
      "Scrolling... 216\n",
      "Now 2628\n",
      "Scrolling... 217\n",
      "Now 2640\n",
      "Scrolling... 218\n",
      "Now 2652\n",
      "Scrolling... 219\n",
      "Now 2664\n",
      "Scrolling... 220\n",
      "Now 2676\n",
      "Scrolling... 221\n",
      "Now 2688\n",
      "Scrolling... 222\n",
      "Now 2700\n",
      "Scrolling... 223\n",
      "Now 2712\n",
      "Scrolling... 224\n",
      "Now 2724\n",
      "Scrolling... 225\n",
      "Now 2736\n",
      "Scrolling... 226\n",
      "Now 2748\n",
      "Scrolling... 227\n",
      "Now 2760\n",
      "Scrolling... 228\n",
      "Now 2772\n",
      "Scrolling... 229\n",
      "Now 2784\n",
      "Scrolling... 230\n",
      "Now 2796\n",
      "Scrolling... 231\n",
      "Now 2808\n",
      "Scrolling... 232\n",
      "Now 2820\n",
      "Scrolling... 233\n",
      "Now 2832\n",
      "Scrolling... 234\n",
      "Now 2844\n",
      "Scrolling... 235\n",
      "Now 2856\n",
      "Scrolling... 236\n",
      "Now 2868\n",
      "Scrolling... 237\n",
      "Now 2880\n",
      "Scrolling... 238\n",
      "Now 2892\n",
      "Scrolling... 239\n",
      "Now 2904\n",
      "Scrolling... 240\n",
      "Now 2916\n",
      "Scrolling... 241\n",
      "Now 2928\n",
      "Scrolling... 242\n",
      "Now 2940\n",
      "Scrolling... 243\n",
      "Now 2952\n",
      "Scrolling... 244\n",
      "Now 2964\n",
      "Scrolling... 245\n",
      "Now 2976\n",
      "Scrolling... 246\n",
      "Now 2988\n",
      "Scrolling... 247\n",
      "Now 3000\n",
      "Scrolling... 248\n",
      "Now 3012\n",
      "Scrolling... 249\n",
      "Now 3024\n",
      "Scrolling... 250\n",
      "Now 3036\n",
      "Scrolling... 251\n",
      "Now 3048\n",
      "Scrolling... 252\n",
      "Now 3060\n",
      "Scrolling... 253\n",
      "Now 3072\n",
      "Scrolling... 254\n",
      "Now 3084\n",
      "Scrolling... 255\n",
      "Now 3096\n",
      "Scrolling... 256\n",
      "Now 3108\n",
      "Scrolling... 257\n",
      "Now 3120\n",
      "Scrolling... 258\n",
      "Now 3132\n",
      "Scrolling... 259\n",
      "Now 3144\n",
      "Scrolling... 260\n",
      "Now 3156\n",
      "Scrolling... 261\n",
      "Now 3168\n",
      "Scrolling... 262\n",
      "Now 3180\n",
      "Scrolling... 263\n",
      "Now 3187\n",
      "Scrolling... 264\n",
      "Now 3187\n",
      "Scrolling... 265\n",
      "Now 3187\n",
      "Scrolling... 266\n",
      "Now 3187\n",
      "Scrolling... 267\n",
      "Now 3187\n",
      "Scrolling... 268\n",
      "Now 3187\n",
      "I Think we are done!\n"
     ]
    }
   ],
   "source": [
    "prev_lis = 0\n",
    "num_times_the_same = 0\n",
    "for i in range(10000):\n",
    "    print(\"Scrolling... {}\".format(i))\n",
    "    driver.execute_script(\n",
    "        \"document.getElementsByClassName('{}')[0].scrollTop = 1000000; document.activeElement.blur(); return false;\".format(scroll)\n",
    "    )\n",
    "    import time; time.sleep(6)\n",
    "    with open('/tmp/{}'.format(artist_inid), 'w') as fout:\n",
    "        ul = driver.find_element_by_class_name(scroll).get_attribute(\"innerHTML\")\n",
    "        lis = BeautifulSoup(str(ul), \"html5lib\").select(\"li\")\n",
    "        num_lis = len(lis)\n",
    "        if num_lis < 10:\n",
    "            print(\"TROUBLE\")\n",
    "            break\n",
    "            \n",
    "        print(\"Now {}\".format(num_lis))\n",
    "        fout.write(str(ul))\n",
    "        if num_lis == prev_lis:\n",
    "            num_times_the_same += 1\n",
    "            \n",
    "        if num_times_the_same > 4:\n",
    "            print(\"I Think we are done!\")\n",
    "            break\n",
    "            \n",
    "        prev_lis = num_lis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ul = driver.find_element_by_class_name(scroll).get_attribute(\"innerHTML\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open ('/tmp/quarrywastaken', 'w') as fout:\n",
    "    fout.write(str(ul))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open ('/tmp/quarrywastaken') as fin:\n",
    "    print(fin.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lis = lis = BeautifulSoup(open('/tmp/{}'.format(artist_inid)).read(), \"html5lib\").select(\"li\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(lis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "[insta.parse_li(li) for li in lis[-20:-1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "driver.get('https://whitelistrecords.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "r = insta.store_screenshot(driver, 'testing')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Insta Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "driver.get('https://www.instagram.com/accounts/login/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "driver.wait_and_get_element_by_name(\"username\").send_keys('joellst.in')\n",
    "driver.wait_and_get_element_by_name(\"password\").send_keys('thisisinstagram')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = next(iter([el for el in driver.find_elements_by_css_selector('form button') if el.text.lower() == 'log in']), None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "r.click()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "https://s3-eu-west-1.amazonaws.com/whtlst-selenium-screenshots/testing_20180621_120650_114657.png"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### --------------  Selenium IP Address ------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ecs_client = boto3.client(\"ecs\")\n",
    "cw_client = boto3.client(\"logs\")\n",
    "events_client = boto3.client(\"events\")\n",
    "ec2_client = boto3.client('ec2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_vnc():\n",
    "    taskInfo = []\n",
    "    taskArns = ecs_client.list_tasks(cluster='tracker')['taskArns']\n",
    "    if not taskArns:\n",
    "        print(\"No running tasks\")\n",
    "        return\n",
    "\n",
    "    described = ecs_client.describe_tasks(tasks=taskArns, cluster='tracker')['tasks']\n",
    "\n",
    "    for task in described:\n",
    "        selenium_container = next((c for c in task['containers'] if c['name'] == 'selenium'), None)\n",
    "        if selenium_container:\n",
    "            portDef = next((r for r in selenium_container.get('networkBindings') or []), None)\n",
    "            if not portDef:\n",
    "                continue\n",
    "            port = portDef.get('hostPort')\n",
    "            containerInstanceArn = task['containerInstanceArn']\n",
    "            ec2InstanceId = ecs_client.describe_container_instances(\n",
    "                containerInstances=[containerInstanceArn], cluster='tracker'\n",
    "            )['containerInstances'][0]['ec2InstanceId']\n",
    "            ip_address = ec2_client.describe_instances(InstanceIds=[ec2InstanceId])['Reservations'][0]['Instances'][0]['PublicIpAddress']\n",
    "            taskInfo.append(\n",
    "                (task.get('group'), \"vnc://{}:{}\".format(ip_address, port))\n",
    "            )\n",
    "\n",
    "    print(\"\\n\".join(\"{}| {}\".format(*t) for t in taskInfo))\n",
    "    return taskInfo\n",
    "    \n",
    "\n",
    "tasks = get_vnc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "for t in tasks:\n",
    "    if t[1]:\n",
    "        print(t)\n",
    "        os.system('''osascript -e 'open location \"{}\" ' '''.format(t[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
