{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "CrowdTangle_Post_Collect_Facebook.ipynb", "provenance": [], "include_colab_link": true }, "kernelspec": { "name": "python2", "display_name": "Python 2" } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "code", "metadata": { "id": "qkvX8JFUIX0E", "colab_type": "code", "colab": {} }, "source": [ "import json\n", "import urllib2\n", "url=''\n", "response=''\n", "data=''\n", "values=''\n", "df=''\n", "df2=''\n", "page=''\n", "url = \"https://api.crowdtangle.com/posts?token=ozXFeJawiB1Ak3OQgfGunx34MDEiynjV8HzB6ZBe&sortBy=date&startDate=2018-06-12&endDate=2019-06-11&count=100&listIds=1157447\"\n", "response = urllib2.urlopen(url)\n", "data = response.read()\n", "values = json.loads(data)" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "O9jwmYK1Ie1X", "colab_type": "code", "colab": {} }, "source": [ "from itertools import chain\n", "import pandas as pd\n", "df=pd.DataFrame.from_dict(values)\n", "df2=pd.DataFrame.from_dict(df.iloc[1,0])\n", "dictlist=[]\n", "prev_page=[]\n", "for key, value in df.iloc[0,0].iteritems():\n", " temp = [key,value.encode('utf-8')]\n", " dictlist.append(temp)\n", " page=list(chain.from_iterable(dictlist))[1]" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "xmRbjW6uIjlj", "colab_type": "code", "outputId": "86a930ad-70e7-4bec-8bba-1a8b0d9a7e83", "colab": { "base_uri": "https://localhost:8080/", "height": 153 } }, "source": [ "import time\n", "for x in range(100):\n", " if (page not in prev_page):\n", " response = urllib2.urlopen(page)\n", " data = response.read()\n", " values = json.loads(data)\n", " df=pd.DataFrame.from_dict(values)\n", " df2=pd.concat([df2,pd.DataFrame.from_dict(df.iloc[1,0])])\n", " time.sleep(11)\n", " prev_page.append(list(chain.from_iterable(dictlist))[1])\n", " dictlist=[]\n", " for key, value in df.iloc[0,0].iteritems():\n", " temp = [key,value.encode('utf-8')]\n", " dictlist.append(temp)\n", " page=list(chain.from_iterable(dictlist))[1]" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:8: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n", "of pandas will change to not sort by default.\n", "\n", "To accept the future behavior, pass 'sort=False'.\n", "\n", "To retain the current behavior and silence the warning, pass 'sort=True'.\n", "\n", " \n" ], "name": "stderr" } ] }, { "cell_type": "code", "metadata": { "id": "0HXPB10iI0w_", "colab_type": "code", "outputId": "ffa7da0a-f200-4ac3-aa45-70a39ad5dce1", "colab": { "base_uri": "https://localhost:8080/", "height": 68 } }, "source": [ "from fastparquet import write\n", "df_final=df2\n", "write('meghan-fb-5.parq', df_final)\n", "!gsutil cp meghan-fb-5.parq gs://epic-social/Facebook" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Copying file://meghan-fb-5.parq [Content-Type=application/octet-stream]...\n", "/ [1 files][440.8 KiB/440.8 KiB] \n", "Operation completed over 1 objects/440.8 KiB. \n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "JIIRNe1cKcSX", "colab_type": "code", "outputId": "eeba99f4-6b96-413e-b861-dac76b4e2ff8", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "source": [ "from google.colab import auth\n", "auth.authenticate_user()\n", "\n", "# https://cloud.google.com/resource-manager/docs/creating-managing-projects\n", "project_id = 'alien-iterator-227622'\n", "!gcloud config set project {project_id}" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Updated property [core/project].\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "KIL_138uLqq1", "colab_type": "code", "outputId": "e3850a68-f40b-446a-ac09-f72d53830644", "colab": { "base_uri": "https://localhost:8080/", "height": 340 } }, "source": [ "df2.count()" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "account 45\n", "caption 9\n", "date 45\n", "description 15\n", "expandedLinks 44\n", "id 45\n", "link 44\n", "media 45\n", "message 45\n", "platform 45\n", "platformId 45\n", "postUrl 45\n", "score 45\n", "statistics 45\n", "subscriberCount 45\n", "title 3\n", "type 45\n", "updated 45\n", "dtype: int64" ] }, "metadata": { "tags": [] }, "execution_count": 57 } ] }, { "cell_type": "code", "metadata": { "id": "9PF9NULeamMT", "colab_type": "code", "outputId": "fdac9b0e-fc25-45d6-b268-4eb4dc4540ec", "colab": { "base_uri": "https://localhost:8080/", "height": 221 } }, "source": [ "!pip install fastparquet" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Requirement already satisfied: fastparquet in /usr/local/lib/python2.7/dist-packages (0.3.1)\n", "Requirement already satisfied: six in /usr/local/lib/python2.7/dist-packages (from fastparquet) (1.12.0)\n", "Requirement already satisfied: thrift>=0.11.0 in /usr/local/lib/python2.7/dist-packages (from fastparquet) (0.11.0)\n", "Requirement already satisfied: numba>=0.28 in /usr/local/lib/python2.7/dist-packages (from fastparquet) (0.40.1)\n", "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python2.7/dist-packages (from fastparquet) (1.16.4)\n", "Requirement already satisfied: pandas>=0.19 in /usr/local/lib/python2.7/dist-packages (from fastparquet) (0.24.2)\n", "Requirement already satisfied: funcsigs in /usr/local/lib/python2.7/dist-packages (from numba>=0.28->fastparquet) (1.0.2)\n", "Requirement already satisfied: enum34 in /usr/local/lib/python2.7/dist-packages (from numba>=0.28->fastparquet) (1.1.6)\n", "Requirement already satisfied: llvmlite>=0.25.0dev0 in /usr/local/lib/python2.7/dist-packages (from numba>=0.28->fastparquet) (0.29.0)\n", "Requirement already satisfied: singledispatch in /usr/local/lib/python2.7/dist-packages (from numba>=0.28->fastparquet) (3.4.0.3)\n", "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python2.7/dist-packages (from pandas>=0.19->fastparquet) (2018.9)\n", "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python2.7/dist-packages (from pandas>=0.19->fastparquet) (2.5.3)\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "VO0qFXZe5dZS", "colab_type": "code", "outputId": "063f9cb8-3ff9-46c9-a368-52e4fbd9016e", "colab": { "base_uri": "https://localhost:8080/", "height": 102 } }, "source": [ "!pip install pyarrow" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Requirement already satisfied: pyarrow in /usr/local/lib/python2.7/dist-packages (0.13.0)\n", "Requirement already satisfied: six>=1.0.0 in /usr/local/lib/python2.7/dist-packages (from pyarrow) (1.12.0)\n", "Requirement already satisfied: enum34>=1.1.6; python_version < \"3.4\" in /usr/local/lib/python2.7/dist-packages (from pyarrow) (1.1.6)\n", "Requirement already satisfied: futures; python_version < \"3.2\" in /usr/local/lib/python2.7/dist-packages (from pyarrow) (3.2.0)\n", "Requirement already satisfied: numpy>=1.14 in /usr/local/lib/python2.7/dist-packages (from pyarrow) (1.16.3)\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "8FkCgbfbHeVh", "colab_type": "code", "outputId": "56a3e756-52c9-49ec-ef90-54c95fb49245", "colab": { "base_uri": "https://localhost:8080/", "height": 309 } }, "source": [ "!pip install pandavro" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Collecting pandavro\n", " Downloading https://files.pythonhosted.org/packages/ca/c2/307ee588db6497289a3119e98577b4a71c291eb888ecb765b034acc177bc/pandavro-1.5.0.tar.gz\n", "Collecting fastavro>=0.14.11 (from pandavro)\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/15/8b/5d9736fec68e94ce4f98e98407b500c21d2faffcbdec52d09a96a0f68ddc/fastavro-0.21.23-cp27-cp27mu-manylinux1_x86_64.whl (1.0MB)\n", "\u001b[K |████████████████████████████████| 1.0MB 2.9MB/s \n", "\u001b[?25hRequirement already satisfied: numpy>=1.7.0 in /usr/local/lib/python2.7/dist-packages (from pandavro) (1.16.3)\n", "Requirement already satisfied: pandas in /usr/local/lib/python2.7/dist-packages (from pandavro) (0.24.2)\n", "Requirement already satisfied: six>=1.9 in /usr/local/lib/python2.7/dist-packages (from pandavro) (1.12.0)\n", "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python2.7/dist-packages (from pandas->pandavro) (2018.9)\n", "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python2.7/dist-packages (from pandas->pandavro) (2.5.3)\n", "Building wheels for collected packages: pandavro\n", " Building wheel for pandavro (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Stored in directory: /root/.cache/pip/wheels/a8/7e/0d/5e6a27d47e4c74a6e4dcdb73d07ea7b9fc957b1453162832ff\n", "Successfully built pandavro\n", "Installing collected packages: fastavro, pandavro\n", "Successfully installed fastavro-0.21.23 pandavro-1.5.0\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "s_3Y0Ox5KpnL", "colab_type": "code", "colab": {} }, "source": [ "import json\n", "import urllib2\n", "url=''\n", "response=''\n", "data=''\n", "values=''\n", "df=''\n", "df2=''\n", "page=''\n", "url = \"https://api.crowdtangle.com/lists?token=ozXFeJawiB1Ak3OQgfGunx34MDEiynjV8HzB6ZBe\"\n", "response = urllib2.urlopen(url)\n", "data = response.read()\n", "values = json.loads(data)" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "Iuqiv_QnErBI", "colab_type": "code", "outputId": "fcdfa5fa-707a-4d76-91f8-76f9b61f4e85", "colab": { "base_uri": "https://localhost:8080/", "height": 170 } }, "source": [ "values" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "{u'result': {u'lists': [{u'id': 1125494,\n", " u'title': u'Camila Cabello',\n", " u'type': u'LIST'},\n", " {u'id': 1125752, u'title': u'AJ Mitchell', u'type': u'LIST'},\n", " {u'id': 1125753, u'title': u'Tyla Yaweh', u'type': u'LIST'},\n", " {u'id': 1125754, u'title': u'Zara Larsson', u'type': u'LIST'},\n", " {u'id': 1125763, u'title': u'French Montana', u'type': u'LIST'},\n", " {u'id': 1157447, u'title': u'Meghan Trainor', u'type': u'LIST'}]},\n", " u'status': 200}" ] }, "metadata": { "tags": [] }, "execution_count": 2 } ] }, { "cell_type": "code", "metadata": { "id": "2jd9w1O6EtpX", "colab_type": "code", "colab": {} }, "source": [ "" ], "execution_count": 0, "outputs": [] } ] }