{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "CrowdTangle_Post_Collect_Instagram.ipynb", "provenance": [], "collapsed_sections": [], "include_colab_link": true }, "kernelspec": { "name": "python2", "display_name": "Python 2" } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "code", "metadata": { "id": "qkvX8JFUIX0E", "colab_type": "code", "colab": {} }, "source": [ "import json\n", "import urllib2\n", "url=''\n", "response=''\n", "data=''\n", "values=''\n", "df=''\n", "df2=''\n", "page=''\n", "url = \"https://api.crowdtangle.com/posts/search?token=LmY6Yk5qp5LiJJ6lZFryJ9v0g7xUNKfS44nD3djL&searchTerm=camila_cabello,shawnmendes&sortBy=total_interactions&startDate=2019-06-17&endDate=2019-07-04&count=100&verifiedOnly=true\"\n", "response = urllib2.urlopen(url)\n", "data = response.read()\n", "values = json.loads(data)" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "O9jwmYK1Ie1X", "colab_type": "code", "colab": {} }, "source": [ "from itertools import chain\n", "import pandas as pd\n", "df=pd.DataFrame.from_dict(values)\n", "df2=pd.DataFrame.from_dict(df.iloc[1,0])\n", "dictlist=[]\n", "prev_page=[]\n", "for key, value in df.iloc[0,0].iteritems():\n", " temp = [key,value.encode('utf-8')]\n", " dictlist.append(temp)\n", " page=list(chain.from_iterable(dictlist))[1]" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "xmRbjW6uIjlj", "colab_type": "code", "outputId": "807d50bc-3b9f-4d87-cee3-846136b01965", "colab": { "base_uri": "https://localhost:8080/", "height": 153 } }, "source": [ "import time\n", "for x in range(1000):\n", " if (page not in prev_page):\n", " response = urllib2.urlopen(page)\n", " data = response.read()\n", " values = json.loads(data)\n", " df=pd.DataFrame.from_dict(values)\n", " df2=pd.concat([df2,pd.DataFrame.from_dict(df.iloc[1,0])])\n", " time.sleep(11)\n", " prev_page.append(list(chain.from_iterable(dictlist))[1])\n", " dictlist=[]\n", " for key, value in df.iloc[0,0].iteritems():\n", " temp = [key,value.encode('utf-8')]\n", " dictlist.append(temp)\n", " page=list(chain.from_iterable(dictlist))[1]" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:8: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n", "of pandas will change to not sort by default.\n", "\n", "To accept the future behavior, pass 'sort=False'.\n", "\n", "To retain the current behavior and silence the warning, pass 'sort=True'.\n", "\n", " \n" ], "name": "stderr" } ] }, { "cell_type": "code", "metadata": { "id": "0HXPB10iI0w_", "colab_type": "code", "outputId": "85f3fd76-2995-4a58-c16d-cb7642672830", "colab": { "base_uri": "https://localhost:8080/", "height": 68 } }, "source": [ "from fastparquet import write\n", "df_final=df2\n", "write('senorita.parq', df_final)\n", "!gsutil cp senorita.parq gs://epic-social/Instagram" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Copying file://senorita.parq [Content-Type=application/octet-stream]...\n", "/ [1 files][ 2.3 MiB/ 2.3 MiB] \n", "Operation completed over 1 objects/2.3 MiB. \n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "JIIRNe1cKcSX", "colab_type": "code", "outputId": "ba118fd9-748b-4eca-cc04-cbf185df11a9", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "source": [ "from google.colab import auth\n", "auth.authenticate_user()\n", "\n", "# https://cloud.google.com/resource-manager/docs/creating-managing-projects\n", "project_id = 'alien-iterator-227622'\n", "!gcloud config set project {project_id}" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Updated property [core/project].\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "KIL_138uLqq1", "colab_type": "code", "outputId": "8d5e8370-14e8-48e8-fc4b-4c34a8e12ca7", "colab": { "base_uri": "https://localhost:8080/", "height": 357 } }, "source": [ "df2.count()" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "account 3639\n", "caption 64\n", "date 1239\n", "description 1073\n", "expandedLinks 265\n", "id 3639\n", "link 240\n", "liveVideoStatus 1\n", "media 1238\n", "message 241\n", "platform 3639\n", "platformId 3639\n", "postUrl 1239\n", "score 3639\n", "statistics 3639\n", "subscriberCount 1239\n", "title 39\n", "type 1239\n", "updated 1239\n", "dtype: int64" ] }, "metadata": { "tags": [] }, "execution_count": 52 } ] }, { "cell_type": "code", "metadata": { "id": "9PF9NULeamMT", "colab_type": "code", "outputId": "8f883f12-ff84-4c79-9846-cfc95a25f4d3", "colab": { "base_uri": "https://localhost:8080/", "height": 221 } }, "source": [ "!pip install fastparquet" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Requirement already satisfied: fastparquet in /usr/local/lib/python2.7/dist-packages (0.3.1)\n", "Requirement already satisfied: six in /usr/local/lib/python2.7/dist-packages (from fastparquet) (1.12.0)\n", "Requirement already satisfied: thrift>=0.11.0 in /usr/local/lib/python2.7/dist-packages (from fastparquet) (0.11.0)\n", "Requirement already satisfied: numba>=0.28 in /usr/local/lib/python2.7/dist-packages (from fastparquet) (0.40.1)\n", "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python2.7/dist-packages (from fastparquet) (1.16.4)\n", "Requirement already satisfied: pandas>=0.19 in /usr/local/lib/python2.7/dist-packages (from fastparquet) (0.24.2)\n", "Requirement already satisfied: funcsigs in /usr/local/lib/python2.7/dist-packages (from numba>=0.28->fastparquet) (1.0.2)\n", "Requirement already satisfied: enum34 in /usr/local/lib/python2.7/dist-packages (from numba>=0.28->fastparquet) (1.1.6)\n", "Requirement already satisfied: llvmlite>=0.25.0dev0 in /usr/local/lib/python2.7/dist-packages (from numba>=0.28->fastparquet) (0.29.0)\n", "Requirement already satisfied: singledispatch in /usr/local/lib/python2.7/dist-packages (from numba>=0.28->fastparquet) (3.4.0.3)\n", "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python2.7/dist-packages (from pandas>=0.19->fastparquet) (2018.9)\n", "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python2.7/dist-packages (from pandas>=0.19->fastparquet) (2.5.3)\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "VO0qFXZe5dZS", "colab_type": "code", "outputId": "063f9cb8-3ff9-46c9-a368-52e4fbd9016e", "colab": { "base_uri": "https://localhost:8080/", "height": 102 } }, "source": [ "!pip install pyarrow" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Requirement already satisfied: pyarrow in /usr/local/lib/python2.7/dist-packages (0.13.0)\n", "Requirement already satisfied: six>=1.0.0 in /usr/local/lib/python2.7/dist-packages (from pyarrow) (1.12.0)\n", "Requirement already satisfied: enum34>=1.1.6; python_version < \"3.4\" in /usr/local/lib/python2.7/dist-packages (from pyarrow) (1.1.6)\n", "Requirement already satisfied: futures; python_version < \"3.2\" in /usr/local/lib/python2.7/dist-packages (from pyarrow) (3.2.0)\n", "Requirement already satisfied: numpy>=1.14 in /usr/local/lib/python2.7/dist-packages (from pyarrow) (1.16.3)\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "8FkCgbfbHeVh", "colab_type": "code", "outputId": "56a3e756-52c9-49ec-ef90-54c95fb49245", "colab": { "base_uri": "https://localhost:8080/", "height": 309 } }, "source": [ "!pip install pandavro" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Collecting pandavro\n", " Downloading https://files.pythonhosted.org/packages/ca/c2/307ee588db6497289a3119e98577b4a71c291eb888ecb765b034acc177bc/pandavro-1.5.0.tar.gz\n", "Collecting fastavro>=0.14.11 (from pandavro)\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/15/8b/5d9736fec68e94ce4f98e98407b500c21d2faffcbdec52d09a96a0f68ddc/fastavro-0.21.23-cp27-cp27mu-manylinux1_x86_64.whl (1.0MB)\n", "\u001b[K |████████████████████████████████| 1.0MB 2.9MB/s \n", "\u001b[?25hRequirement already satisfied: numpy>=1.7.0 in /usr/local/lib/python2.7/dist-packages (from pandavro) (1.16.3)\n", "Requirement already satisfied: pandas in /usr/local/lib/python2.7/dist-packages (from pandavro) (0.24.2)\n", "Requirement already satisfied: six>=1.9 in /usr/local/lib/python2.7/dist-packages (from pandavro) (1.12.0)\n", "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python2.7/dist-packages (from pandas->pandavro) (2018.9)\n", "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python2.7/dist-packages (from pandas->pandavro) (2.5.3)\n", "Building wheels for collected packages: pandavro\n", " Building wheel for pandavro (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Stored in directory: /root/.cache/pip/wheels/a8/7e/0d/5e6a27d47e4c74a6e4dcdb73d07ea7b9fc957b1453162832ff\n", "Successfully built pandavro\n", "Installing collected packages: fastavro, pandavro\n", "Successfully installed fastavro-0.21.23 pandavro-1.5.0\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "s_3Y0Ox5KpnL", "colab_type": "code", "outputId": "31c7eb39-cdbb-4f0c-8e9c-9507f2e21b93", "colab": { "base_uri": "https://localhost:8080/", "height": 394 } }, "source": [ "df2.head()" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
accountdatedescriptionexpandedLinksidlinkmediamessageplatformplatformIdpostUrlscorestatisticssubscriberCounttypeupdated
0{u'subscriberCount': 46527589, u'profileImage'...2019-06-25 23:52:14Scream therapy @camila_cabelloNaN63027061462NaN[{u'url': u'https://scontent.cdninstagram.com/...NaNInstagram2074390522188857733_212742998https://www.instagram.com/p/BzJtzB4gQGF/14482525.0{u'expected': {u'commentCount': 18209, u'viewC...45463841.0video2019-07-04 19:10:33
1{u'subscriberCount': 46527589, u'profileImage'...2019-06-30 05:22:23Thank you for all the love on Señorita ♥️ @cam...NaN63449743873NaN[{u'url': u'https://scontent.cdninstagram.com/...NaNInstagram2077455948334515185_212742998https://www.instagram.com/p/BzUmy2dAEPx/7011901.0{u'expected': {u'commentCount': 17756, u'viewC...46129592.0video2019-07-04 19:10:33
2{u'subscriberCount': 46527589, u'profileImage'...2019-06-21 17:39:48Señorita...@camila_cabello 🖤🌹NaN62681869480NaN[{u'url': u'https://scontent.xx.fbcdn.net/v/t5...NaNInstagram2071304744961541723_212742998https://www.instagram.com/p/By-wLD5A4Zb/6992520.0{u'expected': {u'commentCount': 29663, u'favor...45007944.0album2019-07-04 19:10:33
3{u'subscriberCount': 3899124, u'profileImage':...2019-06-27 20:10:46How to create Havana by Camila Cabello 😂 @imal...NaN63242103054NaN[{u'url': u'https://scontent.xx.fbcdn.net/v/t5...NaNInstagram2075728995598371345_4686316577https://www.instagram.com/p/BzOeIYYnooR/6934390.0{u'expected': {u'commentCount': 3219, u'viewCo...3789428.0video2019-07-04 19:16:59
4{u'subscriberCount': 46527589, u'profileImage'...2019-06-20 18:01:40#Señorita video with @camila_cabello midnight ...NaN62578870210NaN[{u'url': u'https://scontent.xx.fbcdn.net/v/t5...NaNInstagram2070590974857563499_212742998https://www.instagram.com/p/By8N4Vrg81r/4070097.0{u'expected': {u'commentCount': 26915, u'favor...44973020.0photo2019-07-04 19:10:33
\n", "
" ], "text/plain": [ " account ... updated\n", "0 {u'subscriberCount': 46527589, u'profileImage'... ... 2019-07-04 19:10:33\n", "1 {u'subscriberCount': 46527589, u'profileImage'... ... 2019-07-04 19:10:33\n", "2 {u'subscriberCount': 46527589, u'profileImage'... ... 2019-07-04 19:10:33\n", "3 {u'subscriberCount': 3899124, u'profileImage':... ... 2019-07-04 19:16:59\n", "4 {u'subscriberCount': 46527589, u'profileImage'... ... 2019-07-04 19:10:33\n", "\n", "[5 rows x 16 columns]" ] }, "metadata": { "tags": [] }, "execution_count": 39 } ] }, { "cell_type": "code", "metadata": { "id": "QvaHA5hzATmQ", "colab_type": "code", "colab": {} }, "source": [ "import json\n", "import urllib2\n", "url=''\n", "response=''\n", "data=''\n", "values=''\n", "df=''\n", "df2=''\n", "page=''\n", "url = \"https://api.crowdtangle.com/lists?token=07PCsFU8FOkUKLxmThNSHfLK7UP7KUWnHK0E7kCz\"\n", "response = urllib2.urlopen(url)\n", "data = response.read()\n", "values = json.loads(data)" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "7z4GyRq8BEog", "colab_type": "code", "outputId": "2818b03c-bb26-433e-ee9b-77f964a7ba40", "colab": { "base_uri": "https://localhost:8080/", "height": 204 } }, "source": [ "values" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "{u'result': {u'lists': [{u'id': 1124537,\n", " u'title': u'Camila Cabello',\n", " u'type': u'LIST'},\n", " {u'id': 1124559, u'title': u'French Montana', u'type': u'LIST'},\n", " {u'id': 1124561, u'title': u'AJ Mitchell', u'type': u'LIST'},\n", " {u'id': 1124562, u'title': u'Zara Larsson', u'type': u'LIST'},\n", " {u'id': 1124563, u'title': u'Tyla Yaweh', u'type': u'LIST'},\n", " {u'id': 1157446, u'title': u'Meghan Trainor', u'type': u'LIST'},\n", " {u'id': 1161953, u'title': u'DJ Khaled', u'type': u'LIST'},\n", " {u'id': 1183368, u'title': u'Senorita', u'type': u'LIST'}]},\n", " u'status': 200}" ] }, "metadata": { "tags": [] }, "execution_count": 3 } ] }, { "cell_type": "code", "metadata": { "id": "ovHcp46RBG1x", "colab_type": "code", "colab": {} }, "source": [ "" ], "execution_count": 0, "outputs": [] } ] }