{ "cells": [ { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import sys\n", "sys.path.insert(0, '.')\n", "sys.path.insert(0, '../..')" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "\n", "import csv" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import requests" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": true }, "outputs": [], "source": [ "sources = [('global', 'Global'),\n", " ('us', 'United States'),\n", " ('gb', 'United Kingdom'),\n", " ('ad', 'Andorra'),\n", " ('ar', 'Argentina'),\n", " ('at', 'Austria'),\n", " ('au', 'Australia'),\n", " ('be', 'Belgium'),\n", " ('bg', 'Bulgaria'),\n", " ('bo', 'Bolivia'),\n", " ('br', 'Brazil'),\n", " ('ca', 'Canada'),\n", " ('ch', 'Switzerland'),\n", " ('cl', 'Chile'),\n", " ('co', 'Colombia'),\n", " ('cr', 'Costa Rica'),\n", " ('cy', 'Cyprus'),\n", " ('cz', 'Czech Republic'),\n", " ('de', 'Germany'),\n", " ('dk', 'Denmark'),\n", " ('do', 'Dominican Republic'),\n", " ('ec', 'Ecuador'),\n", " ('ee', 'Estonia'),\n", " ('es', 'Spain'),\n", " ('fi', 'Finland'),\n", " ('fr', 'France'),\n", " ('gr', 'Greece'),\n", " ('gt', 'Guatemala'),\n", " ('hk', 'Hong Kong'),\n", " ('hn', 'Honduras'),\n", " ('hu', 'Hungary'),\n", " ('id', 'Indonesia'),\n", " ('ie', 'Ireland'),\n", " ('il', 'Israel'),\n", " ('is', 'Iceland'),\n", " ('it', 'Italy'),\n", " ('jp', 'Japan'),\n", " ('lt', 'Lithuania'),\n", " ('lu', 'Luxembourg'),\n", " ('lv', 'Latvia'),\n", " ('mc', 'Monaco'),\n", " ('mt', 'Malta'),\n", " ('mx', 'Mexico'),\n", " ('my', 'Malaysia'),\n", " ('ni', 'Nicaragua'),\n", " ('nl', 'Netherlands'),\n", " ('no', 'Norway'),\n", " ('nz', 'New Zealand'),\n", " ('pa', 'Panama'),\n", " ('pe', 'Peru'),\n", " ('ph', 'Philippines'),\n", " ('pl', 'Poland'),\n", " ('pt', 'Portugal'),\n", " ('py', 'Paraguay'),\n", " ('ro', 'Romania'),\n", " ('se', 'Sweden'),\n", " ('sg', 'Singapore'),\n", " ('sk', 'Slovakia'),\n", " ('sv', 'El Salvador'),\n", " ('th', 'Thailand'),\n", " ('tr', 'Turkey'),\n", " ('tw', 'Taiwan'),\n", " ('uy', 'Uruguay'),\n", " ('vn', 'Viet Nam')]" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": true }, "outputs": [], "source": [ "r = requests.get('https://spotifycharts.com')" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "collapsed": true }, "outputs": [], "source": [ "sources = [(a.attrs.get('data-value'), a.text) for a in BeautifulSoup(r.content, 'html5lib').select('.chart-filters-list [data-type=\"country\"] li')]" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import urllib.request\n", "from datetime import datetime\n", "import shutil" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": true }, "outputs": [], "source": [ "f = datetime.today()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'20190115_123648'" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "f.strftime('%Y%m%d_%H%M%S')" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def download_chart(country_code, as_of_date=None): \n", " if as_of_date is None:\n", " as_of_date = datetime.today().date()\n", " url_code = 'latest'\n", " else:\n", " url_code = as_of_date.strftime('%Y-%m-%d')\n", " \n", " url = 'https://spotifycharts.com/regional/{}/daily/{}/download'.format(country_code, url_code)\n", " filename = '/tmp/spotifycharts_' + as_of_date.strftime('%Y%m%d') + '_' + country_code + '.csv'\n", " print('downloading {} / {}'.format(url, filename))\n", " r = requests.get(url, stream=True)\n", " with open(filename, 'wb') as f:\n", " shutil.copyfileobj(r.raw, f)\n", " \n", " return filename\n", " \n", " " ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "scrolled": true }, "outputs": [], "source": [ "def parse_chart_csv(filename, as_of, country_code):\n", " rdr = csv.reader(open(filename))\n", " for row in rdr:\n", " if len(row) < 5 or 'https://open.spotify.com/track/' not in row[4]:\n", " print('Skipping row {}'.format(row))\n", " continue\n", " \n", " spyid = row[4].split('https://open.spotify.com/track/')[1]\n", " data = {\n", " 'country_code': country_code, \n", " 'as_of': as_of, \n", " 'spyid': spyid,\n", " 'streams': int(row[3]),\n", " 'position': int(row[0]),\n", " }\n", " # spotify.upsert_track_by_id(spyid)\n", " print(\"Would Save:\\n\", data)\n" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "downloading https://spotifycharts.com/regional/gb/daily/2019-01-11/download / /tmp/spotifycharts_20190111_gb.csv\n" ] } ], "source": [ "as_of = datetime(2019, 1, 11, 0, 0, 0).date()\n", "country_code = sources[2][0]\n", "filename = download_chart(country_code, as_of)\n" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Skipping row ['', '', '', 'Note that these figures are generated using a formula that protects against any artificial inflation of chart positions.', '']\n", "Skipping row ['Position', 'Track Name', 'Artist', 'Streams', 'URL']\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '25sgk305KZfyuqVBQIahim', 'streams': 469827, 'position': 1}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2rPE9A1vEgShuZxxzR2tZH', 'streams': 374593, 'position': 2}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6MWtB6iiXyIwun0YzU6DFP', 'streams': 372692, 'position': 3}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3KkXRkHbMCARz0aVfEt68P', 'streams': 348574, 'position': 4}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5p7ujcrUXASCNwRaWNHR1C', 'streams': 338627, 'position': 5}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6Qs4SXO9dwPj5GKvVOv8Ki', 'streams': 316363, 'position': 6}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '27rdGxbavYJeBphck5MZAF', 'streams': 265294, 'position': 7}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2xLMifQCjDGFmkHkpNLD9h', 'streams': 250192, 'position': 8}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5itOtNx0WxtJmi1TQ3RuRd', 'streams': 230299, 'position': 9}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1Yi8XPnCPoimWwqxwm3uz9', 'streams': 226167, 'position': 10}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0FZ4Dmg8jJJAPJnvBIzD9z', 'streams': 221338, 'position': 11}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '29R1IMTTbDDA3VNlk6UEW5', 'streams': 220568, 'position': 12}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0G1GIn5gGoe5iLHvVC5FM0', 'streams': 220474, 'position': 13}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5tAa8Uaqr4VvA3693mbIhU', 'streams': 217155, 'position': 14}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2IRZnDFmlqMuOrYOLnZZyc', 'streams': 208438, 'position': 15}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7FGq80cy8juXBCD2nrqdWU', 'streams': 203388, 'position': 16}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7wFybC8jBH3zE139OpCtpG', 'streams': 203124, 'position': 17}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5JEx7HbmvHQQswJCsoo9rA', 'streams': 199823, 'position': 18}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5N5k9nd479b1xpDZ4usjrg', 'streams': 199800, 'position': 19}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4ofwffwvvnbSkrMSCKQDaC', 'streams': 192214, 'position': 20}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2VxeLyX666F8uXCJ0dZF8B', 'streams': 183911, 'position': 21}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '01IQ4aQgOf0Kkq3a273hmO', 'streams': 181146, 'position': 22}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1rqqCSm0Qe4I9rUvWncaom', 'streams': 177732, 'position': 23}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2dpaYNEQHiRxtZbfNsse99', 'streams': 170844, 'position': 24}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1xzBco0xcoJEDXktl7Jxrr', 'streams': 166878, 'position': 25}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '222B3aljXx6fFwKmFjuBED', 'streams': 161130, 'position': 26}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2cytBOLpwFRX7J9URCrFIe', 'streams': 160219, 'position': 27}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '14JzyD6FlBD5z0wV5P07YI', 'streams': 158702, 'position': 28}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2xGjteMU3E1tkEPVFBO08U', 'streams': 158629, 'position': 29}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7K7MUBCnzgBAvMVW2RTWNs', 'streams': 157524, 'position': 30}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3ji4bcQugJj5B6NRGYDNwP', 'streams': 154503, 'position': 31}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7dt6x5M1jzdTEt8oCbisTK', 'streams': 153626, 'position': 32}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5IaHrVsrferBYDm0bDyABy', 'streams': 151887, 'position': 33}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5kfNriitmkNE8mUbZ7gbq8', 'streams': 150231, 'position': 34}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6zeeWid2sgw4lap2jV61PZ', 'streams': 144576, 'position': 35}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5Fla3zyOCcIRqrDoKLU0DP', 'streams': 144473, 'position': 36}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0todbbT9g5uQGbeFCiBf2F', 'streams': 144163, 'position': 37}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '43ay9lQZ5rfNcOOHhRF2cM', 'streams': 144134, 'position': 38}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6mQLN3zRtAp6ovjusyYKrV', 'streams': 137913, 'position': 39}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '14mHfNsSHN0FbLftyjrV9g', 'streams': 136359, 'position': 40}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6Gy7rXB6Ku5vIWC7WGWsl3', 'streams': 132794, 'position': 41}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '630sXRhIcfwr2e4RdNtjKN', 'streams': 131243, 'position': 42}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '78QR3Wp35dqAhFEc2qAGjE', 'streams': 131139, 'position': 43}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '13hvHEstJ4sNbzdroPrPI3', 'streams': 128974, 'position': 44}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0RoA7ObU6phWpqhlC9zH4Z', 'streams': 128765, 'position': 45}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4WYRFY0yFa2s1WeX0w84z6', 'streams': 128441, 'position': 46}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '285pBltuF7vW8TeWk8hdRR', 'streams': 126683, 'position': 47}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3ee8Jmje8o58CHK66QrVC2', 'streams': 123708, 'position': 48}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7uiBocndm12aKbsdnQ3Scx', 'streams': 120665, 'position': 49}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5CpkVOosLFc83LbwqO6hnI', 'streams': 120604, 'position': 50}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1uru26I2JKd2mQZt0MDCUe', 'streams': 120086, 'position': 51}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0e7ipj03S05BNilyu5bRzt', 'streams': 119502, 'position': 52}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1fp2uoWXPca3mIdxRHIgQm', 'streams': 119338, 'position': 53}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4M8x1TR36VMQMFMbGCdoyA', 'streams': 118825, 'position': 54}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2TIlqbIneP0ZY1O0EzYLlc', 'streams': 116221, 'position': 55}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1PfUKiMDKIlPEF9FfdJawc', 'streams': 112912, 'position': 56}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0Gl5s8IhMmQE5YQwM8Qx1J', 'streams': 112469, 'position': 57}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5N4erncE7kuUccm7zEmwzk', 'streams': 112180, 'position': 58}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2LskIZrCeLxRvCiGP8gxlh', 'streams': 111933, 'position': 59}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2G7V7zsVDxg1yRsu7Ew9RJ', 'streams': 109519, 'position': 60}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '38zwkK6TtTjIW9tpYBfZ3D', 'streams': 109392, 'position': 61}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '42bbDWZ8WmXTH7PkYAlGLu', 'streams': 108595, 'position': 62}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0E9ZjEAyAwOXZ7wJC0PD33', 'streams': 108273, 'position': 63}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '04MLEeAMuV9IlHEsD8vF6A', 'streams': 107709, 'position': 64}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2BgEsaKNfHUdlh97KmvFyo', 'streams': 107520, 'position': 65}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4w8niZpiMy6qz1mntFA5uM', 'streams': 104567, 'position': 66}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1kQhi22kyluRCnIPAXtxEZ', 'streams': 102044, 'position': 67}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7tFiyTwD0nx5a1eklYtX2J', 'streams': 101938, 'position': 68}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2qT1uLXPVPzGgFOx4jtEuo', 'streams': 101903, 'position': 69}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2GGMabyHXnJmjY6CXhhB2e', 'streams': 101800, 'position': 70}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5rFj8OzffT0iggSjcBaeND', 'streams': 101306, 'position': 71}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4eQBL0cuF3ismYCzBff2Ii', 'streams': 99882, 'position': 72}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '73FebDuELlAEinHw1xWJCZ', 'streams': 99443, 'position': 73}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6DCZcSspjsKoFjzjrWoCdn', 'streams': 99004, 'position': 74}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4EtIza2cKMmR2XP7idebXG', 'streams': 98179, 'position': 75}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2cKovq3l6OJjhVVDbVKOsr', 'streams': 97682, 'position': 76}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7ef4DlsgrMEH11cDZd32M6', 'streams': 97421, 'position': 77}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0TlLq3lA83rQOYtrqBqSct', 'streams': 97363, 'position': 78}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5XWCc9wQes7X5uyYH9h5JR', 'streams': 96026, 'position': 79}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3CA9pLiwRIGtUBiMjbZmRw', 'streams': 95143, 'position': 80}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3IWDUiOsV0Q8XhQKJNxxML', 'streams': 94821, 'position': 81}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7oK9VyNzrYvRFo7nQEYkWN', 'streams': 93682, 'position': 82}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3EPXxR3ImUwfayaurPi3cm', 'streams': 93588, 'position': 83}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4OafepJy2teCjYJbvFE60J', 'streams': 93375, 'position': 84}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7tGEAA1f8MydT7eVbbO9Zy', 'streams': 92959, 'position': 85}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0DC7Zs2qvAbNK1owWQ4mlL', 'streams': 92181, 'position': 86}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '05mAIVLkIWc2d1UBYZBCp8', 'streams': 90234, 'position': 87}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0JP9xo3adEtGSdUEISiszL', 'streams': 90213, 'position': 88}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2iUXsYOEPhVqEBwsqP70rE', 'streams': 89061, 'position': 89}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7fa9MBXhVfQ8P8Df9OEbD8', 'streams': 88938, 'position': 90}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6EelrDeZGS0Z81NhXevtrJ', 'streams': 88542, 'position': 91}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1NJcg626wLln4pGHpiV7mf', 'streams': 88156, 'position': 92}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '116H0KvKr2Zl4RPuVBruDO', 'streams': 86738, 'position': 93}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '58q2HKrzhC3ozto2nDdN4z', 'streams': 86568, 'position': 94}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3swc6WTsr7rl9DqQKQA55C', 'streams': 86258, 'position': 95}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '75ZvA4QfFiZvzhj2xkaWAh', 'streams': 85215, 'position': 96}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7ycWLEP1GsNjVvcjawXz3z', 'streams': 84772, 'position': 97}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6XqvFyJGdUD5IWee02ARKU', 'streams': 83919, 'position': 98}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3IkWmgGsXbsafrunwPojXO', 'streams': 83789, 'position': 99}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4S8d14HvHb70ImctNgVzQQ', 'streams': 83659, 'position': 100}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5xNUR50KxswPRAvx7S163g', 'streams': 83636, 'position': 101}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4WWa4Po5YCEN23DbFzML6M', 'streams': 82891, 'position': 102}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3Ol2xnObFdKV9pmRD2t9x8', 'streams': 81641, 'position': 103}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3TgMcrV32NUKjEG2ujn9eh', 'streams': 80426, 'position': 104}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1wJRveJZLSb1rjhnUHQiv6', 'streams': 79337, 'position': 105}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5CLGzJsGqhCEECcpnFQA8x', 'streams': 78997, 'position': 106}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3wAyZwtyyLMsJNG7Mb17ky', 'streams': 78600, 'position': 107}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6DMXPshekQJR7uOMj0103y', 'streams': 78355, 'position': 108}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6eVWOyIA4iacH7UaQ1TqzM', 'streams': 77867, 'position': 109}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7m9OqQk4RVRkw9JJdeAw96', 'streams': 77443, 'position': 110}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3NuK5xMlSlB6K2Qp16zf3h', 'streams': 76389, 'position': 111}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3KLygy1YWhmrQx0GRs7IOx', 'streams': 76094, 'position': 112}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '254bXAqt3zP6P50BdQvEsq', 'streams': 74519, 'position': 113}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6CTWathupIiDs7U4InHnDA', 'streams': 74308, 'position': 114}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1k65BavLispRXs7vzBuRLV', 'streams': 74030, 'position': 115}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5OCJzvD7sykQEKHH7qAC3C', 'streams': 74013, 'position': 116}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0oagwWkGlBoACUuKvTmqZV', 'streams': 73391, 'position': 117}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0mfEHC4z5a7533o7VNOH2c', 'streams': 72715, 'position': 118}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '40T5GIqQ1CegGm2PTEl8Bu', 'streams': 72271, 'position': 119}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5T8EDUDqKcs6OSOwEsfqG7', 'streams': 71840, 'position': 120}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1mXVgsBdtIVeCLJnSnmtdV', 'streams': 71648, 'position': 121}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7pdF27mSDuPWhppnHAmWHa', 'streams': 71620, 'position': 122}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0jAfdqv18goRTUxm3ilRjb', 'streams': 71268, 'position': 123}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4kWO6O1BUXcZmaxitpVUwp', 'streams': 70838, 'position': 124}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5GNjiM8jZCgbqjHklAcT9e', 'streams': 70159, 'position': 125}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2tUBqZG2AbRi7Q0BIrVrEj', 'streams': 69747, 'position': 126}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '76cy1WJvNGJTj78UqeA5zr', 'streams': 69035, 'position': 127}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2374M0fQpWi3dLnB54qaLX', 'streams': 67684, 'position': 128}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2t8yVaLvJ0RenpXUIAC52d', 'streams': 67641, 'position': 129}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5qqabIl2vWzo9ApSC317sa', 'streams': 66115, 'position': 130}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7qiZfU4dY1lWllzX7mPBI3', 'streams': 65589, 'position': 131}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4qKcDkK6siZ7Jp1Jb4m0aL', 'streams': 65555, 'position': 132}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2xJCMIJfcNYDc5iR0sAm2O', 'streams': 64756, 'position': 133}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6TR0FGw4zhlGbQALN065AI', 'streams': 64426, 'position': 134}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6QgjcU0zLnzq5OrUoSZ3OK', 'streams': 64335, 'position': 135}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3Fzlg5r1IjhLk2qRw667od', 'streams': 63989, 'position': 136}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '09IStsImFySgyp0pIQdqAc', 'streams': 63968, 'position': 137}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4o6BgsqLIBViaGVbx5rbRk', 'streams': 63572, 'position': 138}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2rbDhOo9Fh61Bbu23T2qCk', 'streams': 63371, 'position': 139}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2r7kFZo4Ua517FKCW39fo9', 'streams': 63185, 'position': 140}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5vdp5UmvTsnMEMESIF2Ym7', 'streams': 63088, 'position': 141}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3B54sVLJ402zGa6Xm4YGNe', 'streams': 63048, 'position': 142}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0JpxNq2SYwPt2uSqdcSvyQ', 'streams': 62996, 'position': 143}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6ho0GyrWZN3mhi9zVRW7xi', 'streams': 62944, 'position': 144}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '60SdxE8apGAxMiRrpbmLY0', 'streams': 62766, 'position': 145}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1mqlc0vEP9mU1kZgTi6LIQ', 'streams': 61844, 'position': 146}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6lOWoTqVnAWXchddtTH31W', 'streams': 61557, 'position': 147}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4Gw1SXoN1MlAmdM3J1NjmR', 'streams': 61360, 'position': 148}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1rfofaqEpACxVEHIZBJe6W', 'streams': 61143, 'position': 149}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6u9RqxALwkjJ1ukB1y8vuP', 'streams': 60720, 'position': 150}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '03CbQgFxnBFoaaVJTfEqMD', 'streams': 60603, 'position': 151}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2FUNBaa5DwItJtYEBgAblU', 'streams': 60534, 'position': 152}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '51NFxnQvaosfDDutk0tams', 'streams': 60219, 'position': 153}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '55GiOwtxNajXVbpjjIzMnD', 'streams': 60164, 'position': 154}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '33DzKnwuBE6lfOiADwzd5E', 'streams': 60118, 'position': 155}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7GJClzimvMSghjcrKxuf1M', 'streams': 59700, 'position': 156}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0lstGz2advSx5VqY8olvzt', 'streams': 59655, 'position': 157}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0GVuLQtPXFaL18ijEOqoAa', 'streams': 59418, 'position': 158}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5uCax9HTNlzGybIStD3vDh', 'streams': 58730, 'position': 159}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2H9zpvV33I48HE5jnNvWVB', 'streams': 58328, 'position': 160}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7bkzlEJJLmV6wMJIBhYAMi', 'streams': 58152, 'position': 161}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2fuCquhmrzHpu5xcA1ci9x', 'streams': 58097, 'position': 162}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3a1lNhkSLSkpJE4MSHpDu9', 'streams': 58060, 'position': 163}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3GCdLUSnKSMJhs4Tj6CV3s', 'streams': 57600, 'position': 164}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '10Igtw8bSDyyFs7KIsKngZ', 'streams': 57379, 'position': 165}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2uIX8YMNjGMD7441kqyyNU', 'streams': 57097, 'position': 166}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0PowxVSTY2uh1CPc8EPKyX', 'streams': 56406, 'position': 167}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3rq5w4bQGigXOfdN30ATJt', 'streams': 56130, 'position': 168}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6GvhPVjvqhspuWvnuCkvhF', 'streams': 55989, 'position': 169}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3IpGVLqOyhPrbhqSpUs0dc', 'streams': 54972, 'position': 170}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '22An65gG31JLf9n18qjdhZ', 'streams': 54969, 'position': 171}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7e4KQkp5ZR1gXCAalRjX6B', 'streams': 54587, 'position': 172}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '3S0OXQeoh0w6AY8WQVckRW', 'streams': 54098, 'position': 173}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7AFASza1mXqntmGtbxXprO', 'streams': 54000, 'position': 174}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0pqnGHJpmpxLKifKRmU6WP', 'streams': 53858, 'position': 175}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5f14ylFBAuwqVHXvafqtmd', 'streams': 53833, 'position': 176}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '180WImKPbvdUrboKUov5f4', 'streams': 53745, 'position': 177}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '66y7x28jXOPrcmu3D5Zjh6', 'streams': 53738, 'position': 178}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4QIyEq7TcTdAcxjI2AFlAK', 'streams': 53470, 'position': 179}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2G1tXoGBaEMJ7FKGnkf6ud', 'streams': 53465, 'position': 180}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2RlgNHKcydI9sayD2Df2xp', 'streams': 53443, 'position': 181}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7BHPGtpuuWWsvE7cCaMuEU', 'streams': 53293, 'position': 182}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7GX5flRQZVHRAGd6B4TmDO', 'streams': 53055, 'position': 183}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4w3tQBXhn5345eUXDGBWZG', 'streams': 52887, 'position': 184}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '45Egmo7icyopuzJN0oMEdk', 'streams': 52776, 'position': 185}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6PCUP3dWmTjcTtXY02oFdT', 'streams': 52669, 'position': 186}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2n48BjaFSczfeIG3HaIaPv', 'streams': 52570, 'position': 187}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4jvjzW7Hm0yK4LvvE0Paz9', 'streams': 52546, 'position': 188}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0tgVpDi06FyKpA1z0VMD4v', 'streams': 52495, 'position': 189}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '0UvCh63URrLFcPkKt99hHd', 'streams': 52412, 'position': 190}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6cFZ4PLC19taNlpl9pbGMf', 'streams': 52334, 'position': 191}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1KGi9sZVMeszgZOWivFpxs', 'streams': 52246, 'position': 192}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1zi7xx7UVEFkmKfv06H8x0', 'streams': 52227, 'position': 193}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '2RSHsoi04658QL5xgQVov3', 'streams': 52187, 'position': 194}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '7lAK3oHbfEnvUQWosrMMpR', 'streams': 52178, 'position': 195}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '1cS0TgbR263ey9jn0MwD2s', 'streams': 52175, 'position': 196}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '4hQ6UGyWQIGJmHSo0J88JW', 'streams': 51815, 'position': 197}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '5Z01UMMf7V1o0MzF86s6WJ', 'streams': 51739, 'position': 198}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '6lU7kJ6k1oUew36PbPl15E', 'streams': 50942, 'position': 199}\n", "Would Save:\n", " {'country_code': datetime.date(2019, 1, 11), 'as_of': 'gb', 'spyid': '15DwFznkBJir7AK9PyMyRR', 'streams': 50305, 'position': 200}\n" ] } ], "source": [ "parse_chart_csv(filename, country_code, as_of)" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "collapsed": true }, "outputs": [], "source": [ "rrr = requests.get('https://spotifycharts.com/regional/global/daily/latest/download')" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import csv" ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "collapsed": true }, "outputs": [], "source": [ "rrr.iter_lines?" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "<_csv.reader at 0x10582f518>" ] }, "execution_count": 59, "metadata": {}, "output_type": "execute_result" } ], "source": [ "csv.reader(rrr.iter_lines())" ] }, { "cell_type": "code", "execution_count": 60, "metadata": { "collapsed": true }, "outputs": [], "source": [ "lines = _" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [ { "ename": "Error", "evalue": "iterator should return strings, not bytes (did you open the file in text mode?)", "output_type": "error", "traceback": [ "\u001b[0;31m----------------------------------\u001b[0m", "\u001b[0;31mError\u001b[0mTraceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlines\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mError\u001b[0m: iterator should return strings, not bytes (did you open the file in text mode?)" ] } ], "source": [ "for line in lines:\n", " print(line)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "r.json()['info']['sc']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Schema:\n", "\n", "arid\n", "scid\n", "spyid\n", "eid\n", "inid\n", "\n", "avatar: 'http://example.com/avatar.png',\n", "bio: ''\n", "countryCodes: []\n", "followerCounts: { sc: 0, spy: 0, in: 0, tw: 0 }\n" ] }, { "cell_type": "code", "execution_count": 69, "metadata": { "collapsed": true }, "outputs": [], "source": [ "m = requests.get('https://armas1743.lnk.to/SPLA', allow_redirects=False)" ] }, { "cell_type": "code", "execution_count": 78, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['https://st.linkfire.com/Api/Click/click/d53fc61fb3d5118338d3ba61320a7c5e/422894?boardUuid=8fc13bce-4c47-11e6-9fd0-066c3e7a8751&linkUuid=bbf32bcf-b5af-4535-8def-13f72230e806&ouuid=7fe92092-046b-4ed6-9ef8-5edebf27c87f&tlouuid=7fe92092-046b-4ed6-9ef8-5edebf27c87f&country=GB&visitorToken=d53fc61fb3d5118338d3ba61320a7c5e&boardId=1894&uid=5b437c1fdcf959.03045908&shortlink=ARMAS1743.lnk.to%2FSPLA&referrer=&linkChannel=0&url=https%3A%2F%2Fopen.spotify.com%2Ftrack%2F6KUsVERTI2fY3Swi5mo27R&artists=Mokita&tags=&orgId=107&browser=python-requests&device=crawler&os=unknown&querystring=&city=Hornsey&activity=&channelType=Original&channelId=&organisation=Armada+Music&artist=Mokita&album=Love+Alone&track=Love+Alone&label=&board=Armada+Music&genre=&countryName=United+Kingdom&puuid=c5af5c00-4013-4675-945f-f41c55b4f415&sourcetype=landingpage&linkId=422894&sourcetype=landingpage&newSession=True',\n", " 'https://st.linkfire.com/Api/Click/app/d53fc61fb3d5118338d3ba61320a7c5e/422894?boardUuid=8fc13bce-4c47-11e6-9fd0-066c3e7a8751&linkUuid=bbf32bcf-b5af-4535-8def-13f72230e806&ouuid=7fe92092-046b-4ed6-9ef8-5edebf27c87f&tlouuid=7fe92092-046b-4ed6-9ef8-5edebf27c87f&country=GB&visitorToken=d53fc61fb3d5118338d3ba61320a7c5e&boardId=1894&uid=5b437c1fdcf959.03045908&shortlink=ARMAS1743.lnk.to%2FSPLA&referrer=&linkChannel=0&url=https%3A%2F%2Fopen.spotify.com%2Ftrack%2F6KUsVERTI2fY3Swi5mo27R&artists=Mokita&tags=&orgId=107&browser=python-requests&device=crawler&os=unknown&querystring=&city=Hornsey&activity=&channelType=Original&channelId=&organisation=Armada+Music&artist=Mokita&album=Love+Alone&track=Love+Alone&label=&board=Armada+Music&genre=&countryName=United+Kingdom&puuid=c5af5c00-4013-4675-945f-f41c55b4f415&sourcetype=landingpage&linkId=422894&channel=spotify&apptype=single&eventCategoryType=play']" ] }, "execution_count": 78, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[i.attrs.get('src') for i in BeautifulSoup(m.content, 'html5lib').select('img')]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "soup = BeautifulSoup(m.content, 'html5lib')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "a0 = soup.select_one('#prefpage').select('a')[0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "[a.attrs for a in soup.select_one('#prefpage').select('a')]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from urllib.parse import urlparse" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "urlparse('www.instagram.com')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import re" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "rrr = re.compile('spotify.com\\/user\\/.*\\/playlist\\/')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "rrr.search('https://open.spotify.com/user/jal648/playlist/5xQKMpDH3AGhtap4Mq6FVU?si=PUimPDN3RoKQxblT8lnKJA')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "rrr.split('https://open.spotify.com/track/5xQKMpDH3AGhtap4Mq6FVU?si=PUimPDN3RoKQxblT8lnKJA')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "t = requests.get('http://smarturl.it/SACHIspotify', allow_redirects=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "t.headers['Location']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "t.status_code" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "sss = requests.get('http://snd.sc/MYIhlx', allow_redirects=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "sss" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "sss.headers['Location']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "kp = requests.get('https://twitter.com/intent/user?screen_name=kingprincess69', allow_redirects=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "kp" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "kpsoup = BeautifulSoup(kp.content, 'html5lib')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "kpsoup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "kpsoup.select_one('.summary .nickname').text" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "[a.attrs.get('data-expanded-url') for a in kpsoup.select('.summary .note a')]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "sm2 = requests.get('http://smarturl.it/makemybed')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mmb = BeautifulSoup(sm2.content, 'html5lib')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "[a.attrs.get('data-url') for a in mmb.select('#provider-list a') if a.get('data-url')]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from tracker.spidering import url_spider" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "R = url_spider._get_handler('http://smarturl.it/makemybed')" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": true }, "outputs": [], "source": [ "ll = R('http://smarturl.it/makemybed').spider_for_urls()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['https://www.amazon.co.uk/Make-Bed-Explicit-King-Princess/dp/B07DMCCTWF/ref=sr_1_1?ie=UTF8&qid=1529342645&sr=8-1&keywords=make+my+bed+king+princess&tag=sonmus07-21',\n", " 'https://itunes.apple.com/gb/album/make-my-bed-ep/1396670015?uo=4&ls=1&app=music&at=11l79w&ct=4kmmm4',\n", " 'https://itunes.apple.com/gb/album/make-my-bed-ep/1396670015?uo=4&ls=1&app=itunes&at=11l79w&ct=4kmmm4',\n", " 'http://open.spotify.com/album/6VnoKML5dWRxOOEbMAPfG5',\n", " 'https://youtu.be/WTuJYwrygSg']" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ll" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "requests.get('https://bit.ly/2KoHdyA', allow_redirects=False)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": true }, "outputs": [], "source": [ "bt = _" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'https://twitter.com/KingPrincess69'" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bt.headers.get('Location')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "screen_names = [\"TheAMAUK\", \"tracy_ma\", \"Ocasio2018\", \"bbgirlbritt\", \"StormiMaya\", \"orbitalpx\", \"ShowtimeLA818\", \"MemphiaMM\", \"NowUnitedMusic\", \"LillyMilman\", \"CultofRezz\", \"wsvarghese\", \"georgia_m_mckay\", \"RenshawHS\", \"AaronMazor\", \"JeremiahSilva13\", \"ASphotoNYC\", \"SameronHiles\", \"bariilyssa\", \"nagruhs\", \"mr_merrick\", \"faintmag\", \"RSPolitics\", \"D_Tal\", \"followneal\", \"_pauLaLaLa\", \"MEGASHUFFLE\", \"a_paul_g\", \"SandwichGomesIV\", \"ThisisEarth\", \"ohmara\", \"schumer\", \"openhousenyc\", \"sfrieds\", \"Phish_FTR\", \"DBTH\", \"PopUpMusic_\", \"JFSica\", \"maxdalton\", \"BRappy55\", \"drewStadler\", \"lillistcupcake\", \"BRBLasagnaJR\", \"aviator31\", \"shararose\", \"adamdesiderio\", \"zeeceeskie\", \"theSchabes\", \"RyanShorr\", \"philipbobby\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "for n in screen_names[-20:-1]:\n", " r1 = requests.get('https://twitter.com/intent/user?screen_name={}'.format(n), allow_redirects=False)\n", " soup = BeautifulSoup(r1.content, 'html5lib')\n", " urls = [a.attrs.get('data-expanded-url') for a in soup.select('.summary .note a') if a.attrs.get('data-expanded-url') ]\n", "\n", " print(\"\\n\\n{}: \\n\\n{}\".format(n, json.dumps(urls, indent=2)))\n", " import time; time.sleep(2)" ] }, { "cell_type": "code", "execution_count": 79, "metadata": { "collapsed": true }, "outputs": [], "source": [ "top_tracks = '''\n", "402893568\n", "388292009\n", "379770545\n", "379402637\n", "404323284\n", "371421017\n", "374023478\n", "374459129\n", "393951015\n", "381320963\n", "382143683\n", "397560597\n", "373180994\n", "398491623\n", "388018229\n", "394356345\n", "396530439\n", "370614416\n", "381996059\n", "393455898\n", "372207440\n", "402812445\n", "406735815\n", "397650279\n", "399690738\n", "388779204\n", "403560963\n", "397650231\n", "406260021\n", "392117283\n", "389495529\n", "374292974\n", "388682223\n", "389191530\n", "395477778\n", "388153226\n", "378385937\n", "404649510\n", "382284143\n", "385833719\n", "388585878\n", "397650252\n", "397650270\n", "380345714\n", "391943448\n", "395030925\n", "374292995\n", "378370727\n", "382387046\n", "400580565\n", "400873743\n", "397650264\n", "385543409\n", "374293010\n", "397650243\n", "393539214\n", "385838504\n", "396557802\n", "397650276\n", "378873662\n", "399256818\n", "397650267\n", "388005032\n", "378504179\n", "371851634\n", "402586104\n", "399028485\n", "402291849\n", "397999140\n", "395686416\n", "381534737\n", "395562192\n", "374231543\n", "407674584\n", "379027904\n", "400580571\n", "386068955\n", "390966405\n", "397650258\n", "373197143\n", "405563529\n", "380401943\n", "380740907\n", "397650234\n", "372836567\n", "397131156\n", "371851592\n", "406287219\n", "373638059\n", "373196852\n", "397650273\n", "399245250\n", "397650255\n", "397650246\n", "393284094\n", "400668606\n", "400580553\n", "403994976\n", "404514528\n", "389463651\n", "400061817\n", "391944393\n", "377246855\n", "398550276\n", "400305990\n", "404516217\n", "400580589\n", "405439029\n", "404517909\n", "371851529\n", "392034117\n", "404515326\n", "395989218\n", "378551831\n", "376230341\n", "399681696\n", "401322114\n", "381281378\n", "385050590\n", "371851616\n", "372832748\n", "373383650\n", "400148778\n", "397650237\n", "393197340\n", "371851568\n", "399679659\n", "402564591\n", "405464790\n", "404748891\n", "397650228\n", "403788930\n", "399073782\n", "397650261\n", "371851535\n", "392117274\n", "382587857\n", "399003024\n", "395404896\n", "393651297\n", "371851553\n", "371851496\n", "400580619\n", "392576553\n", "402256656\n", "406078437\n", "373196825\n", "396976650\n", "404089296\n", "373778930\n", "399683865\n", "404514885\n", "397225269\n", "371851511\n", "404516526\n", "387477809\n", "373383614\n", "375775727\n", "400305921\n", "406623909\n", "373196864\n", "399259248\n", "399059862\n", "382251146\n", "396381273\n", "400305870\n", "400575042\n", "404101038\n", "404515686\n", "389452683\n", "406772127\n", "385532834\n", "391944897\n", "402010917\n", "388994568\n", "393522873\n", "404234529\n", "400605243\n", "400305822\n", "396976635\n", "407456001\n", "407687973\n", "382854626\n", "389218395\n", "398489595\n", "373227848\n", "399252729\n", "371851418\n", "373196882\n", "381866225\n", "397943190\n", "387981791\n", "373382729\n", "395256405\n", "382587407\n", "400654020\n", "385999628\n", "376008179\n", "383779571\n", "400227354\n", "401303412\n", "395987553\n", "404176707\n", "382566896\n", "370055771\n", "384466463\n", "398718498\n", "404170821\n", "399047223\n", "400623150\n", "406589676\n", "400305684\n", "393018072\n", "391130103\n", "374292965\n", "394807815\n", "396849891\n", "400624170\n", "373383575\n", "373197041\n", "407751912\n", "374293007\n", "372832721\n", "400668588\n", "404508435\n", "404125434\n", "373196837\n", "405543303\n", "383105120\n", "400664706\n", "373196870\n", "387891932\n", "373383461\n", "394944771\n", "378446735\n", "388659339\n", "398590401\n", "403954530\n", "406356261\n", "403930359\n", "398812056\n", "385500641\n", "379814969\n", "391787976\n", "378821477\n", "381659921\n", "393516180\n", "398494113\n", "395931129\n", "394176015\n", "392489637\n", "373158839\n", "405095538\n", "398488872\n", "400305708\n", "393702642\n", "373196891\n", "407159697\n", "388817859\n", "401467758\n", "402945405\n", "402058524\n", "399592719\n", "376167884\n", "382863251\n", "407174856\n", "397623417\n", "371760581\n", "391912035\n", "388618983\n", "389264739\n", "403526670\n", "379036901\n", "407015799\n", "393485154\n", "389451408\n", "400163442\n", "398558556\n", "373419926\n", "387534212\n", "385555226\n", "370552208\n", "396422754\n", "373196885\n", "394878351\n", "389299845\n", "374292956\n", "404291982\n", "402640596\n", "404126034\n", "404033028\n", "402522765\n", "373382687\n", "404112447\n", "402122028\n", "385048004\n", "373196849\n", "400510083\n", "402385494\n", "'''" ] }, { "cell_type": "code", "execution_count": 80, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "',402893568,388292009,379770545,379402637,404323284,371421017,374023478,374459129,393951015,381320963,382143683,397560597,373180994,398491623,388018229,394356345,396530439,370614416,381996059,393455898,372207440,402812445,406735815,397650279,399690738,388779204,403560963,397650231,406260021,392117283,389495529,374292974,388682223,389191530,395477778,388153226,378385937,404649510,382284143,385833719,388585878,397650252,397650270,380345714,391943448,395030925,374292995,378370727,382387046,400580565,400873743,397650264,385543409,374293010,397650243,393539214,385838504,396557802,397650276,378873662,399256818,397650267,388005032,378504179,371851634,402586104,399028485,402291849,397999140,395686416,381534737,395562192,374231543,407674584,379027904,400580571,386068955,390966405,397650258,373197143,405563529,380401943,380740907,397650234,372836567,397131156,371851592,406287219,373638059,373196852,397650273,399245250,397650255,397650246,393284094,400668606,400580553,403994976,404514528,389463651,400061817,391944393,377246855,398550276,400305990,404516217,400580589,405439029,404517909,371851529,392034117,404515326,395989218,378551831,376230341,399681696,401322114,381281378,385050590,371851616,372832748,373383650,400148778,397650237,393197340,371851568,399679659,402564591,405464790,404748891,397650228,403788930,399073782,397650261,371851535,392117274,382587857,399003024,395404896,393651297,371851553,371851496,400580619,392576553,402256656,406078437,373196825,396976650,404089296,373778930,399683865,404514885,397225269,371851511,404516526,387477809,373383614,375775727,400305921,406623909,373196864,399259248,399059862,382251146,396381273,400305870,400575042,404101038,404515686,389452683,406772127,385532834,391944897,402010917,388994568,393522873,404234529,400605243,400305822,396976635,407456001,407687973,382854626,389218395,398489595,373227848,399252729,371851418,373196882,381866225,397943190,387981791,373382729,395256405,382587407,400654020,385999628,376008179,383779571,400227354,401303412,395987553,404176707,382566896,370055771,384466463,398718498,404170821,399047223,400623150,406589676,400305684,393018072,391130103,374292965,394807815,396849891,400624170,373383575,373197041,407751912,374293007,372832721,400668588,404508435,404125434,373196837,405543303,383105120,400664706,373196870,387891932,373383461,394944771,378446735,388659339,398590401,403954530,406356261,403930359,398812056,385500641,379814969,391787976,378821477,381659921,393516180,398494113,395931129,394176015,392489637,373158839,405095538,398488872,400305708,393702642,373196891,407159697,388817859,401467758,402945405,402058524,399592719,376167884,382863251,407174856,397623417,371760581,391912035,388618983,389264739,403526670,379036901,407015799,393485154,389451408,400163442,398558556,373419926,387534212,385555226,370552208,396422754,373196885,394878351,389299845,374292956,404291982,402640596,404126034,404033028,402522765,373382687,404112447,402122028,385048004,373196849,400510083,402385494'" ] }, "execution_count": 80, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\",\".join(top_tracks.splitlines())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.1" } }, "nbformat": 4, "nbformat_minor": 2 }