{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import boto3\n",
    "import json\n",
    "import pandas as pd\n",
    "logging.getLogger().setLevel(logging.INFO)\n",
    "\n",
    "def parse_pipl_json(bucket, prefix):\n",
    "    logging.info(f'Attempting to read files from {bucket}/{prefix}')\n",
    "\n",
    "    # TODO! can we do it somehow so we wouldn't have to instantiate client and resource?\n",
    "    client = boto3.client('s3')\n",
    "    #s3 = boto3.resource('s3')\n",
    "\n",
    "    data_list = []\n",
    "    objects = client.list_objects(Bucket=bucket, Prefix=prefix, MaxKeys=9999)\n",
    "    \n",
    "    for key in objects['Contents']:\n",
    "        key = key['Key']\n",
    "        obj = client.get_object(Bucket=bucket, Key=key)['Body']\n",
    "        data = json.loads(obj.read())\n",
    "        data_list.append(data)\n",
    "\n",
    "    \"\"\" # In case you need JSON flattening, this flattens the JSON and puts it to a dataframe.\n",
    "        # Left it here for boilerplate\n",
    "    from pandas.io.json import json_normalize\n",
    "    print(repr(json_normalize(data)))\n",
    "    \"\"\"\n",
    "\n",
    "    results_list = []\n",
    "    for item in data_list:\n",
    "\n",
    "        results = {'email': None,\n",
    "                   #'available_data': None,\n",
    "                   'match': None,\n",
    "                   'gender': None,\n",
    "                   'age': None,\n",
    "                   'phones': None,\n",
    "                   'firstnames': None,\n",
    "                   'addresses': None,\n",
    "                   'facebook': None,\n",
    "                   'linkedin': None,\n",
    "                   'instagram': None,\n",
    "                   'twitter': None,\n",
    "                   'images': None\n",
    "                   }\n",
    "\n",
    "        try:\n",
    "            results['email'] = item['query']['emails'][0]['address']\n",
    "\n",
    "            available = item.get('available_data')\n",
    "            if available:\n",
    "                results['available_data'] = available.get('premium')\n",
    "\n",
    "            # Skip iteration if person not found\n",
    "            person = item.get('person')\n",
    "            if not person:\n",
    "                continue\n",
    "\n",
    "            results['match'] = person.get('@match')\n",
    "\n",
    "            gender = person.get('gender')\n",
    "            if gender:\n",
    "                results['gender'] = gender.get('content')\n",
    "\n",
    "            # Get the first part for display (e.g. 28 from \"28 years old\")\n",
    "            age = person.get('dob')\n",
    "            if age:\n",
    "                results['age'] = age.get('display').split(' ')[0]\n",
    "\n",
    "            # Get all phone numbers\n",
    "            phones = person.get('phones')\n",
    "            if phones:\n",
    "                phone_list = []\n",
    "                for phone in phones:\n",
    "                    # We only care about mobile phones\n",
    "                    if phone.get('@type') == 'mobile':\n",
    "                        phone_list.append(phone.get('display_international'))\n",
    "                results['phones'] = phone_list\n",
    "\n",
    "            # Get all first names\n",
    "            names = person.get('names')\n",
    "            if names:\n",
    "                names_list = []\n",
    "                for name in names:\n",
    "                    names_list.append(name.get('first'))\n",
    "                results['firstnames'] = names_list\n",
    "\n",
    "            # Get all addresses\n",
    "            addresses = person.get('addresses')\n",
    "            if addresses:\n",
    "                addresses_list = []\n",
    "                for address in addresses:\n",
    "                    addresses_list.append(address.get('display'))\n",
    "                results['addresses'] = addresses_list\n",
    "\n",
    "            # Get social media (stores only the fact that this exists)\n",
    "            social = person.get('user_ids')\n",
    "            if social:\n",
    "                for user_id in social:\n",
    "                    social = user_id.get('content').split('@')[-1]\n",
    "                    if social in ['facebook', 'linkedin', 'instagram', 'twitter']:\n",
    "                        results[social] = social\n",
    "\n",
    "            images = person.get('images')\n",
    "            if images:\n",
    "                images_list = []\n",
    "                for image in images:\n",
    "                    images_list.append(image.get('thumbnail_token'))\n",
    "                results['images'] = images_list\n",
    "                # TODO! do smth with those images\n",
    "\n",
    "            # Add parsed results of this person to list\n",
    "            results_list.append(results)\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"Exception {e}: {item}\")\n",
    "\n",
    "    dataframe = pd.DataFrame(results_list)\n",
    "    return dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_pipl_json_summary(bucket, prefix):\n",
    "    logging.info(f'Attempting to read files from {bucket}/{prefix}')\n",
    "\n",
    "    # TODO! can we do it somehow so we wouldn't have to instantiate client and resource?\n",
    "    client = boto3.client('s3')\n",
    "    #s3 = boto3.resource('s3')\n",
    "\n",
    "    data_list = []\n",
    "    objects = client.list_objects(Bucket=bucket, Prefix=prefix, MaxKeys=9999)\n",
    "    \n",
    "    for key in objects['Contents']:\n",
    "        key = key['Key']\n",
    "        obj = client.get_object(Bucket=bucket, Key=key)['Body']\n",
    "        data = json.loads(obj.read())\n",
    "        data_list.append(data)\n",
    "\n",
    "    \"\"\" # In case you need JSON flattening, this flattens the JSON and puts it to a dataframe.\n",
    "        # Left it here for boilerplate\n",
    "    from pandas.io.json import json_normalize\n",
    "    print(repr(json_normalize(data)))\n",
    "    \"\"\"\n",
    "\n",
    "    results_list = []\n",
    "    for item in data_list:\n",
    "\n",
    "        results = {'email': \"\",\n",
    "                   #'available_data': None,\n",
    "                   'match': \"\",\n",
    "                   'gender': \"\",\n",
    "                   'gender_value':\"\",\n",
    "                   'age': \"\",\n",
    "                   'age_value':\"\",\n",
    "                   'phones': \"\",\n",
    "                   'firstnames': \"\",\n",
    "                   'addresses': \"\",\n",
    "                   'addresses_value': \"\",\n",
    "                   'facebook': \"\",\n",
    "                   'linkedin': \"\",\n",
    "                   'instagram': \"\",\n",
    "                   'twitter': \"\",\n",
    "                   'images': \"\"\n",
    "                   }\n",
    "\n",
    "        try:\n",
    "            results['email'] = item['query']['emails'][0]['address']\n",
    "\n",
    "            available = item.get('available_data')\n",
    "            if available:\n",
    "                results['available_data'] = \"Y\"\n",
    "\n",
    "            # Skip iteration if person not found\n",
    "            person = item.get('person')\n",
    "            if not person:\n",
    "                continue\n",
    "\n",
    "            results['match'] = person.get('@match')\n",
    "\n",
    "            gender = person.get('gender')\n",
    "            if gender:\n",
    "                results['gender'] = \"Y\"\n",
    "                results['gender_value']=gender['content']\n",
    "\n",
    "            age = person.get('dob')\n",
    "            if age:\n",
    "                results['age'] = \"Y\"\n",
    "                results['age_value']=age.get('display')\n",
    "\n",
    "            phones = person.get('phones')\n",
    "            if phones:\n",
    "                results['phones'] = \"Y\"\n",
    "\n",
    "            names = person.get('names')\n",
    "            if names:\n",
    "                results['firstnames'] = \"Y\"\n",
    "\n",
    "            addresses = person.get('addresses')\n",
    "            if addresses:\n",
    "                results['addresses'] = \"Y\"\n",
    "                results['addresses_value']=addresses.get('display')\n",
    "\n",
    "            social = person.get('user_ids')\n",
    "            if social:\n",
    "                for user_id in social:\n",
    "                    social = user_id.get('content').split('@')[-1]\n",
    "                    if social in ['facebook', 'linkedin', 'instagram', 'twitter']:\n",
    "                        results[social] = \"Y\"\n",
    "\n",
    "            images = person.get('images')\n",
    "            if images:\n",
    "                images_list = []\n",
    "                for image in images:\n",
    "                    images_list.append(image.get('thumbnail_token'))\n",
    "                results['images'] = \"Y\"\n",
    "\n",
    "            # Add parsed results of this person to list\n",
    "            results_list.append(results)\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"Exception {e}: {item}\")\n",
    "\n",
    "    dataframe = pd.DataFrame(results_list)\n",
    "    return dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Attempting to read files from frontend-api-devel-filestore/a7a460d855e229dd0384001e1c9f5d6fb59381f967ba1b7ec36a7591/50bd3222-a32a-410f-8293-7c07512fe3dc/enrichment/6/2020-07-20T19:51:47\n"
     ]
    }
   ],
   "source": [
    "# prefix = 'a7a460d855e229dd0384001e1c9f5d6fb59381f967ba1b7ec36a7591/50bd3222-a32a-410f-8293-7c07512fe3dc/enrichment/5/2020-06-11T23:34:14'\n",
    "# prefix for Janis Joplin data\n",
    "prefix = 'a7a460d855e229dd0384001e1c9f5d6fb59381f967ba1b7ec36a7591/50bd3222-a32a-410f-8293-7c07512fe3dc/enrichment/6/2020-07-20T19:51:47'\n",
    "df = parse_pipl_json('frontend-api-devel-filestore', prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Attempting to read files from frontend-api-devel-filestore/a7a460d855e229dd0384001e1c9f5d6fb59381f967ba1b7ec36a7591/50bd3222-a32a-410f-8293-7c07512fe3dc/enrichment/6/2020-07-20T19:51:47\n",
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_summary = parse_pipl_json_summary('frontend-api-devel-filestore', prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.to_csv('superfan_enrichments_jj.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_summary.to_csv('superfan_enrichments_jj_summary.csv', sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    male\n",
      "Name: gender_value, dtype: object\n",
      "0    63 years old\n",
      "Name: age_value, dtype: object\n",
      "0    [{'@valid_since': '2011-09-06', '@last_seen': ...\n",
      "Name: addresses_value, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(df_summary['gender_value'].head(1))\n",
    "print(df_summary['age_value'].head(1))\n",
    "print(df_summary['addresses_value'].head(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500, 13)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_summary.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}