{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "##### MAKE SURE THIS IS THE GITHUB FILE" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting pandas-profiling\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/e8/b0/bd5e3aaf37302fbe581b6947dc5ec1cda02a0ffe50fc823123def73e4d7a/pandas-profiling-2.5.0.tar.gz (192kB)\n", "\u001b[K |████████████████████████████████| 194kB 5.2MB/s eta 0:00:01\n", "\u001b[?25hRequirement already satisfied: numpy>=1.16.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pandas-profiling) (1.17.4)\n", "Collecting scipy>=1.4.1\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/85/7a/ae480be23b768910a9327c33517ced4623ba88dc035f9ce0206657c353a9/scipy-1.4.1-cp37-cp37m-macosx_10_6_intel.whl (28.4MB)\n", "\u001b[K |████████████████████████████████| 28.4MB 15.8MB/s eta 0:00:01\n", "\u001b[?25hRequirement already satisfied: pandas==0.25.3 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pandas-profiling) (0.25.3)\n", "Requirement already satisfied: matplotlib>=3.0.3 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pandas-profiling) (3.1.1)\n", "Processing /Users/barr001/Library/Caches/pip/wheels/b0/b2/96/2074eee7dbf7b7df69d004c9b6ac4e32dad04fb7666cf943bd/confuse-1.0.0-cp37-none-any.whl\n", "Collecting jinja2==2.11.1\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/27/24/4f35961e5c669e96f6559760042a55b9bcfcdb82b9bdb3c8753dbe042e35/Jinja2-2.11.1-py2.py3-none-any.whl (126kB)\n", "\u001b[K |████████████████████████████████| 133kB 40.3MB/s eta 0:00:01\n", "\u001b[?25hCollecting visions==0.2.2\n", " Downloading https://files.pythonhosted.org/packages/07/4a/ab37f8bafda516b66c4f475b221a6c170097c0db203750a4aafb01023339/visions-0.2.2.tar.gz\n", "Processing /Users/barr001/Library/Caches/pip/wheels/43/07/ac/7c5a9d708d65247ac1f94066cf1db075540b85716c30255459/htmlmin-0.1.12-cp37-none-any.whl\n", "Collecting missingno==0.4.2\n", " Using cached https://files.pythonhosted.org/packages/2b/de/6e4dd6d720c49939544352155dc06a08c9f7e4271aa631a559dfbeaaf9d4/missingno-0.4.2-py3-none-any.whl\n", "Collecting phik==0.9.9\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/03/cf/b8cef2778104dc5d319f36dd836efaceb07a037cbf63f27c966b5a193ce9/phik-0.9.9-py3-none-any.whl (607kB)\n", "\u001b[K |████████████████████████████████| 614kB 53.0MB/s eta 0:00:01\n", "\u001b[?25hRequirement already satisfied: astropy>=3.2.3 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pandas-profiling) (3.2.3)\n", "Collecting tangled-up-in-unicode==0.0.3\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/43/fc/e3c970c5007b405827a4623e70fc4eb966ee49bc1edbd56c33e85e1c6534/tangled_up_in_unicode-0.0.3.tar.gz (1.5MB)\n", "\u001b[K |████████████████████████████████| 1.5MB 42.8MB/s eta 0:00:01\n", "\u001b[?25hCollecting tqdm==4.42.0\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/cc/2e/4307206db63f05ed37e21d4c0d843d0fbcacd62479f8ce99ba0f2c0875e0/tqdm-4.42.0-py2.py3-none-any.whl (59kB)\n", "\u001b[K |████████████████████████████████| 61kB 14.5MB/s eta 0:00:01\n", "\u001b[?25hCollecting kaggle==1.5.6\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/62/ab/bb20f9b9e24f9a6250f95a432f8d9a7d745f8d24039d7a5a6eaadb7783ba/kaggle-1.5.6.tar.gz (58kB)\n", "\u001b[K |████████████████████████████████| 61kB 18.3MB/s eta 0:00:01\n", "\u001b[?25hRequirement already satisfied: ipywidgets==7.5.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pandas-profiling) (7.5.1)\n", "Requirement already satisfied: requests==2.22.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pandas-profiling) (2.22.0)\n", "Requirement already satisfied: pytz>=2017.2 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pandas==0.25.3->pandas-profiling) (2019.3)\n", "Requirement already satisfied: python-dateutil>=2.6.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pandas==0.25.3->pandas-profiling) (2.8.1)\n", "Requirement already satisfied: cycler>=0.10 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from matplotlib>=3.0.3->pandas-profiling) (0.10.0)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from matplotlib>=3.0.3->pandas-profiling) (1.1.0)\n", "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from matplotlib>=3.0.3->pandas-profiling) (2.4.5)\n", "Requirement already satisfied: pyyaml in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from confuse==1.0.0->pandas-profiling) (5.1.2)\n", "Requirement already satisfied: MarkupSafe>=0.23 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from jinja2==2.11.1->pandas-profiling) (1.1.1)\n", "Requirement already satisfied: networkx in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from visions==0.2.2->pandas-profiling) (2.4)\n", "Collecting attr\n", " Downloading https://files.pythonhosted.org/packages/de/be/ddc7f84d4e087144472a38a373d3e319f51a6faf6e5fc1ae897173675f21/attr-0.3.1.tar.gz\n", "Requirement already satisfied: seaborn in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from missingno==0.4.2->pandas-profiling) (0.9.0)\n", "Requirement already satisfied: jupyter-client>=5.2.3 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from phik==0.9.9->pandas-profiling) (5.3.4)\n", "Collecting pytest-pylint>=0.13.0\n", " Downloading https://files.pythonhosted.org/packages/0f/2e/12d3e83e90341fc7aff0e42955ab560df4e7f07b847f98302e23eba96289/pytest_pylint-0.15.0-py3-none-any.whl\n", "Requirement already satisfied: nbconvert>=5.3.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from phik==0.9.9->pandas-profiling) (5.6.1)\n", "Requirement already satisfied: numba>=0.38.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from phik==0.9.9->pandas-profiling) (0.46.0)\n", "Requirement already satisfied: pytest>=4.0.2 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from phik==0.9.9->pandas-profiling) (5.3.1)\n", "Collecting joblib>=0.14.1\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/28/5c/cf6a2b65a321c4a209efcdf64c2689efae2cb62661f8f6f4bb28547cf1bf/joblib-0.14.1-py2.py3-none-any.whl (294kB)\n", "\u001b[K |████████████████████████████████| 296kB 43.3MB/s eta 0:00:01\n", "\u001b[?25hCollecting urllib3<1.25,>=1.21.1\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/01/11/525b02e4acc0c747de8b6ccdab376331597c569c42ea66ab0a1dbd36eca2/urllib3-1.24.3-py2.py3-none-any.whl (118kB)\n", "\u001b[K |████████████████████████████████| 122kB 64.2MB/s eta 0:00:01\n", "\u001b[?25hRequirement already satisfied: six>=1.10 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from kaggle==1.5.6->pandas-profiling) (1.13.0)\n", "Requirement already satisfied: certifi in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from kaggle==1.5.6->pandas-profiling) (2019.11.28)\n", "Collecting python-slugify\n", " Downloading https://files.pythonhosted.org/packages/92/5f/7b84a0bba8a0fdd50c046f8b57dcf179dc16237ad33446079b7c484de04c/python-slugify-4.0.0.tar.gz\n", "Requirement already satisfied: nbformat>=4.2.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipywidgets==7.5.1->pandas-profiling) (4.4.0)\n", "Requirement already satisfied: ipython>=4.0.0; python_version >= \"3.3\" in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipywidgets==7.5.1->pandas-profiling) (7.10.1)\n", "Requirement already satisfied: ipykernel>=4.5.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipywidgets==7.5.1->pandas-profiling) (5.1.3)\n", "Requirement already satisfied: widgetsnbextension~=3.5.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipywidgets==7.5.1->pandas-profiling) (3.5.1)\n", "Requirement already satisfied: traitlets>=4.3.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipywidgets==7.5.1->pandas-profiling) (4.3.3)\n", "Requirement already satisfied: idna<2.9,>=2.5 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from requests==2.22.0->pandas-profiling) (2.8)\n", "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from requests==2.22.0->pandas-profiling) (3.0.4)\n", "Requirement already satisfied: setuptools in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib>=3.0.3->pandas-profiling) (42.0.2.post20191203)\n", "Requirement already satisfied: decorator>=4.3.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from networkx->visions==0.2.2->pandas-profiling) (4.4.1)\n", "Requirement already satisfied: jupyter-core>=4.6.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from jupyter-client>=5.2.3->phik==0.9.9->pandas-profiling) (4.6.1)\n", "Requirement already satisfied: tornado>=4.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from jupyter-client>=5.2.3->phik==0.9.9->pandas-profiling) (6.0.3)\n", "Requirement already satisfied: pyzmq>=13 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from jupyter-client>=5.2.3->phik==0.9.9->pandas-profiling) (18.1.0)\n", "Requirement already satisfied: pylint>=2.0.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pytest-pylint>=0.13.0->phik==0.9.9->pandas-profiling) (2.4.4)\n", "Requirement already satisfied: defusedxml in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbconvert>=5.3.1->phik==0.9.9->pandas-profiling) (0.6.0)\n", "Requirement already satisfied: mistune<2,>=0.8.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbconvert>=5.3.1->phik==0.9.9->pandas-profiling) (0.8.4)\n", "Requirement already satisfied: bleach in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbconvert>=5.3.1->phik==0.9.9->pandas-profiling) (3.1.0)\n", "Requirement already satisfied: pygments in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbconvert>=5.3.1->phik==0.9.9->pandas-profiling) (2.5.2)\n", "Requirement already satisfied: testpath in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbconvert>=5.3.1->phik==0.9.9->pandas-profiling) (0.4.4)\n", "Requirement already satisfied: entrypoints>=0.2.2 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbconvert>=5.3.1->phik==0.9.9->pandas-profiling) (0.3)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: pandocfilters>=1.4.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbconvert>=5.3.1->phik==0.9.9->pandas-profiling) (1.4.2)\n", "Requirement already satisfied: llvmlite>=0.30.0dev0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from numba>=0.38.1->phik==0.9.9->pandas-profiling) (0.30.0)\n", "Requirement already satisfied: py>=1.5.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pytest>=4.0.2->phik==0.9.9->pandas-profiling) (1.8.0)\n", "Requirement already satisfied: packaging in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pytest>=4.0.2->phik==0.9.9->pandas-profiling) (19.2)\n", "Requirement already satisfied: attrs>=17.4.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pytest>=4.0.2->phik==0.9.9->pandas-profiling) (19.3.0)\n", "Requirement already satisfied: more-itertools>=4.0.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pytest>=4.0.2->phik==0.9.9->pandas-profiling) (7.2.0)\n", "Requirement already satisfied: pluggy<1.0,>=0.12 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pytest>=4.0.2->phik==0.9.9->pandas-profiling) (0.13.1)\n", "Requirement already satisfied: wcwidth in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pytest>=4.0.2->phik==0.9.9->pandas-profiling) (0.1.7)\n", "Requirement already satisfied: importlib-metadata>=0.12 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pytest>=4.0.2->phik==0.9.9->pandas-profiling) (1.2.0)\n", "Collecting text-unidecode>=1.3\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl (78kB)\n", "\u001b[K |████████████████████████████████| 81kB 20.2MB/s eta 0:00:01\n", "\u001b[?25hRequirement already satisfied: jsonschema!=2.5.0,>=2.4 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbformat>=4.2.0->ipywidgets==7.5.1->pandas-profiling) (3.2.0)\n", "Requirement already satisfied: ipython-genutils in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from nbformat>=4.2.0->ipywidgets==7.5.1->pandas-profiling) (0.2.0)\n", "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets==7.5.1->pandas-profiling) (3.0.2)\n", "Requirement already satisfied: backcall in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets==7.5.1->pandas-profiling) (0.1.0)\n", "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets==7.5.1->pandas-profiling) (4.7.0)\n", "Requirement already satisfied: appnope; sys_platform == \"darwin\" in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets==7.5.1->pandas-profiling) (0.1.0)\n", "Requirement already satisfied: jedi>=0.10 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets==7.5.1->pandas-profiling) (0.14.1)\n", "Requirement already satisfied: pickleshare in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets==7.5.1->pandas-profiling) (0.7.5)\n", "Requirement already satisfied: notebook>=4.4.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from widgetsnbextension~=3.5.0->ipywidgets==7.5.1->pandas-profiling) (6.0.2)\n", "Requirement already satisfied: isort<5,>=4.2.5 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pylint>=2.0.0->pytest-pylint>=0.13.0->phik==0.9.9->pandas-profiling) (4.3.21)\n", "Requirement already satisfied: mccabe<0.7,>=0.6 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pylint>=2.0.0->pytest-pylint>=0.13.0->phik==0.9.9->pandas-profiling) (0.6.1)\n", "Requirement already satisfied: astroid<2.4,>=2.3.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pylint>=2.0.0->pytest-pylint>=0.13.0->phik==0.9.9->pandas-profiling) (2.3.3)\n", "Requirement already satisfied: webencodings in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from bleach->nbconvert>=5.3.1->phik==0.9.9->pandas-profiling) (0.5.1)\n", "Requirement already satisfied: zipp>=0.5 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from importlib-metadata>=0.12->pytest>=4.0.2->phik==0.9.9->pandas-profiling) (0.6.0)\n", "Requirement already satisfied: pyrsistent>=0.14.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets==7.5.1->pandas-profiling) (0.15.6)\n", "Requirement already satisfied: ptyprocess>=0.5 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from pexpect; sys_platform != \"win32\"->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets==7.5.1->pandas-profiling) (0.6.0)\n", "Requirement already satisfied: parso>=0.5.0 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from jedi>=0.10->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets==7.5.1->pandas-profiling) (0.5.1)\n", "Requirement already satisfied: terminado>=0.8.1 in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets==7.5.1->pandas-profiling) (0.8.3)\n", "Requirement already satisfied: prometheus-client in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets==7.5.1->pandas-profiling) (0.7.1)\n", "Requirement already satisfied: Send2Trash in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets==7.5.1->pandas-profiling) (1.5.0)\n", "Collecting typed-ast<1.5,>=1.4.0; implementation_name == \"cpython\" and python_version < \"3.8\"\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/0c/f0/c351d6aeb7f5ba00da2e5aac43ef09b71be95aed45bfca943a5016854269/typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl (223kB)\n", "\u001b[K |████████████████████████████████| 225kB 41.0MB/s eta 0:00:01\n", "\u001b[?25hRequirement already satisfied: wrapt==1.11.* in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from astroid<2.4,>=2.3.0->pylint>=2.0.0->pytest-pylint>=0.13.0->phik==0.9.9->pandas-profiling) (1.11.2)\n", "Requirement already satisfied: lazy-object-proxy==1.4.* in /Users/barr001/anaconda3/envs/python3.7/lib/python3.7/site-packages (from astroid<2.4,>=2.3.0->pylint>=2.0.0->pytest-pylint>=0.13.0->phik==0.9.9->pandas-profiling) (1.4.3)\n", "Building wheels for collected packages: pandas-profiling, visions, tangled-up-in-unicode, kaggle, attr, python-slugify\n", " Building wheel for pandas-profiling (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for pandas-profiling: filename=pandas_profiling-2.5.0-py2.py3-none-any.whl size=241328 sha256=dcbffa352b2710ba57b3f9c81a27142a341e80906b1c008cf32fa184173eda24\n", " Stored in directory: /Users/barr001/Library/Caches/pip/wheels/9b/c9/f1/4a2f30c760e017f3e2f46be999c4597a93d126ef5ea38e276f\n", " Building wheel for visions (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for visions: filename=visions-0.2.2-cp37-none-any.whl size=53059 sha256=6319debb5ecd60b20d6b96f9d67c80ca27c6ae60ccf39c8a9b50b49d562358a9\n", " Stored in directory: /Users/barr001/Library/Caches/pip/wheels/53/87/68/294a9e88d82e395b38571df18f7cb71e9ab51cedae77dd6f31\n", " Building wheel for tangled-up-in-unicode (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for tangled-up-in-unicode: filename=tangled_up_in_unicode-0.0.3-cp37-none-any.whl size=1554154 sha256=d58cc92a04a33a3ec5444f5fed503078b27a3c295600d4ed0b81c429edc3c26c\n", " Stored in directory: /Users/barr001/Library/Caches/pip/wheels/c4/57/cc/5f58206efb00418d4dcae8d08a3cb40627778ea29622f664c6\n", " Building wheel for kaggle (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for kaggle: filename=kaggle-1.5.6-cp37-none-any.whl size=72859 sha256=cd8ca00da94001000908a83c755073cbcf098a40c5288a04476ed5a9f4f6407b\n", " Stored in directory: /Users/barr001/Library/Caches/pip/wheels/57/4e/e8/bb28d035162fb8f17f8ca5d42c3230e284c6aa565b42b72674\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Building wheel for attr (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for attr: filename=attr-0.3.1-cp37-none-any.whl size=2459 sha256=49a45f914bd2aad95aacff89c59a4753cb9e9190946dd3887313b72bef59e88f\n", " Stored in directory: /Users/barr001/Library/Caches/pip/wheels/f0/96/9b/1f8892a707d17095b5a6eab0275da9d39e68e03a26aee2e726\n", " Building wheel for python-slugify (setup.py) ... \u001b[?25ldone\n", "\u001b[?25h Created wheel for python-slugify: filename=python_slugify-4.0.0-py2.py3-none-any.whl size=5487 sha256=a65bed414f52ca17d660b2b5a53e1c629608969588dbe113c0f062cd56ff13a0\n", " Stored in directory: /Users/barr001/Library/Caches/pip/wheels/11/94/81/312969455540cb0e6a773e5d68a73c14128bfdfd4a7969bb4f\n", "Successfully built pandas-profiling visions tangled-up-in-unicode kaggle attr python-slugify\n", "Installing collected packages: scipy, confuse, jinja2, tangled-up-in-unicode, attr, visions, htmlmin, missingno, pytest-pylint, joblib, phik, tqdm, urllib3, text-unidecode, python-slugify, kaggle, pandas-profiling, typed-ast\n", " Found existing installation: scipy 1.3.1\n", " Uninstalling scipy-1.3.1:\n", " Successfully uninstalled scipy-1.3.1\n", " Found existing installation: Jinja2 2.10.3\n", " Uninstalling Jinja2-2.10.3:\n", " Successfully uninstalled Jinja2-2.10.3\n", " Found existing installation: joblib 0.14.0\n", " Uninstalling joblib-0.14.0:\n", " Successfully uninstalled joblib-0.14.0\n", " Found existing installation: tqdm 4.40.0\n", " Uninstalling tqdm-4.40.0:\n", " Successfully uninstalled tqdm-4.40.0\n", " Found existing installation: urllib3 1.25.7\n", " Uninstalling urllib3-1.25.7:\n", " Successfully uninstalled urllib3-1.25.7\n", "Successfully installed attr-0.3.1 confuse-1.0.0 htmlmin-0.1.12 jinja2-2.11.1 joblib-0.14.1 kaggle-1.5.6 missingno-0.4.2 pandas-profiling-2.5.0 phik-0.9.9 pytest-pylint-0.15.0 python-slugify-4.0.0 scipy-1.4.1 tangled-up-in-unicode-0.0.3 text-unidecode-1.3 tqdm-4.42.0 typed-ast-1.4.1 urllib3-1.24.3 visions-0.2.2\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install pandas-profiling" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "from datetime import datetime, timedelta \n", "import os\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "import glob\n", "from pandas_profiling import ProfileReport" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
primary_artist_keyproject_keyproduct_family_nomincount
0367626-12890241990-02-208
1599458-12790511994-02-282
2353277-13236671999-01-182
3273915-13976962004-05-151
4104059658524384772008-03-041
\n", "
" ], "text/plain": [ " primary_artist_key project_key product_family_no min count\n", "0 367626 -1 289024 1990-02-20 8\n", "1 599458 -1 279051 1994-02-28 2\n", "2 353277 -1 323667 1999-01-18 2\n", "3 273915 -1 397696 2004-05-15 1\n", "4 104059 65852 438477 2008-03-04 1" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data = pd.read_csv('/users/BARR001/Documents/Finance Stream Predictions/one_more_try.csv')\n", "projects = pd.read_csv('/users/BARR001/Documents/Finance Stream Predictions/project_mapping_with_release_date.csv')\n", "projects.head()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(9810730, 21)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.shape" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Index(['report_date', 'week_end_date_thursday', 'month_end_date',\n", " 'product_family_no', 'product_family_name', 'primary_artist_name',\n", " 'fin_label_name', 'project_id', 'source_name', 'age_group_intl_name',\n", " 'gender_name', 'type_group_name', 'zip_code', 'playlist_type_name',\n", " 'media_duration', 'playlist_count', 'product_count', 'max_streams',\n", " 'listener_count', 'streams', 'artist_count'],\n", " dtype='object')" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.columns" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "report_date object\n", "week_end_date_thursday object\n", "month_end_date object\n", "product_family_no int64\n", "product_family_name object\n", "primary_artist_name object\n", "fin_label_name object\n", "project_id int64\n", "source_name object\n", "age_group_intl_name object\n", "gender_name object\n", "type_group_name object\n", "zip_code object\n", "playlist_type_name object\n", "media_duration int64\n", "playlist_count int64\n", "product_count int64\n", "max_streams int64\n", "listener_count int64\n", "streams int64\n", "artist_count int64\n", "dtype: object" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.dtypes" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "data['report_date'] = pd.to_datetime(data['report_date'])\n", "data['week_end_date_thursday'] = pd.to_datetime(data['week_end_date_thursday'])\n", "data['month_end_date'] = pd.to_datetime(data['month_end_date'])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "report_date datetime64[ns]\n", "week_end_date_thursday datetime64[ns]\n", "month_end_date datetime64[ns]\n", "product_family_no int64\n", "product_family_name object\n", "primary_artist_name object\n", "fin_label_name object\n", "project_id int64\n", "source_name object\n", "age_group_intl_name object\n", "gender_name object\n", "type_group_name object\n", "zip_code object\n", "playlist_type_name object\n", "media_duration int64\n", "playlist_count int64\n", "product_count int64\n", "max_streams int64\n", "listener_count int64\n", "streams int64\n", "artist_count int64\n", "dtype: object" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.dtypes" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
report_dateweek_end_date_thursdaymonth_end_dateproduct_family_noproduct_family_nameprimary_artist_namefin_label_nameproject_idsource_nameage_group_intl_name...type_group_namezip_codeplaylist_type_namemedia_durationplaylist_countproduct_countmax_streamslistener_countstreamsartist_count
02016-12-132016-12-152016-12-312748192Esperando Que VuelvasNelson Velásquez & Morre RomeroCatalog Digital1120240Others Playlist40-44...FreeNonOther Playlist2871147111
12017-01-212017-01-262017-01-313858865Dancing in the StreetDavid Bowie & Mick JaggerColumbia Legacy Cat1184974Album15-20...FreeNonNaN19111432111
22017-01-162017-01-192017-01-312602827Mi Última CartaChelo SilvaCatalog Digital1094464Artist50-54...Paid1NaN18511129111
32017-03-102017-03-162017-03-31349775Juan Diecinueve Cuarenta y UnoCamilo SestoCatalog BMG Int'l1035964Collection45-49...Free752NaN16111143111
42017-03-072017-03-092017-03-31349775Juan Diecinueve Cuarenta y UnoCamilo SestoCatalog BMG Int'l1035964Others Playlist50-54...Paid921Other Playlist16111143111
\n", "

5 rows × 21 columns

\n", "
" ], "text/plain": [ " report_date week_end_date_thursday month_end_date product_family_no \\\n", "0 2016-12-13 2016-12-15 2016-12-31 2748192 \n", "1 2017-01-21 2017-01-26 2017-01-31 3858865 \n", "2 2017-01-16 2017-01-19 2017-01-31 2602827 \n", "3 2017-03-10 2017-03-16 2017-03-31 349775 \n", "4 2017-03-07 2017-03-09 2017-03-31 349775 \n", "\n", " product_family_name primary_artist_name \\\n", "0 Esperando Que Vuelvas Nelson Velásquez & Morre Romero \n", "1 Dancing in the Street David Bowie & Mick Jagger \n", "2 Mi Última Carta Chelo Silva \n", "3 Juan Diecinueve Cuarenta y Uno Camilo Sesto \n", "4 Juan Diecinueve Cuarenta y Uno Camilo Sesto \n", "\n", " fin_label_name project_id source_name age_group_intl_name ... \\\n", "0 Catalog Digital 1120240 Others Playlist 40-44 ... \n", "1 Columbia Legacy Cat 1184974 Album 15-20 ... \n", "2 Catalog Digital 1094464 Artist 50-54 ... \n", "3 Catalog BMG Int'l 1035964 Collection 45-49 ... \n", "4 Catalog BMG Int'l 1035964 Others Playlist 50-54 ... \n", "\n", " type_group_name zip_code playlist_type_name media_duration playlist_count \\\n", "0 Free Non Other Playlist 287 1 \n", "1 Free Non NaN 191 1 \n", "2 Paid 1 NaN 185 1 \n", "3 Free 752 NaN 161 1 \n", "4 Paid 921 Other Playlist 161 1 \n", "\n", " product_count max_streams listener_count streams artist_count \n", "0 1 47 1 1 1 \n", "1 1 432 1 1 1 \n", "2 1 129 1 1 1 \n", "3 1 143 1 1 1 \n", "4 1 143 1 1 1 \n", "\n", "[5 rows x 21 columns]" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.head()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10983" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(data['product_family_no'].unique())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "So what we're aiming for here is a row per pfn (by month?) with the breakdowns (%free vs paid, %M vs F, %age group, etc)" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
media_durationstreams
countmeansum
product_family_no
2755931249.0000001
2773571180.0000001
2774634180.0000004
27746410165.00000010
2778071216.0000001
............
775368596108406.222222124
816259891117330.461538137
824506455110236.327273119
879900041668197.179641724
99978342374220.29729780
\n", "

10983 rows × 3 columns

\n", "
" ], "text/plain": [ " media_duration streams\n", " count mean sum\n", "product_family_no \n", "275593 1 249.000000 1\n", "277357 1 180.000000 1\n", "277463 4 180.000000 4\n", "277464 10 165.000000 10\n", "277807 1 216.000000 1\n", "... ... ... ...\n", "775368596 108 406.222222 124\n", "816259891 117 330.461538 137\n", "824506455 110 236.327273 119\n", "879900041 668 197.179641 724\n", "999783423 74 220.297297 80\n", "\n", "[10983 rows x 3 columns]" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.groupby('product_family_no').agg({'media_duration': ['count', 'mean'],'streams': ['sum']})" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "22c033311a0a4b1fbe9b8f61d7aa30d5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='variables', max=21.0, style=ProgressStyle(description_wid…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ebdf08696f2b46758d11721be392e5a2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='correlations', max=6.0, style=ProgressStyle(description_w…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4f6e05c5ca7941a6b0ec49deec605713", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='interactions [continuous]', max=64.0, style=ProgressStyle…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "11dbc210a7874b00b54abd203dddb5c9", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='table', max=1.0, style=ProgressStyle(description_width='i…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7d43fa0916a4447daa2078838bfb34cd", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='missing', max=3.0, style=ProgressStyle(description_width=…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "28bccd7a63254adbbe93d964453577ae", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='warnings', max=3.0, style=ProgressStyle(description_width…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "e8e6dc07788e4bdf909bdb3adda0f399", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='package', max=1.0, style=ProgressStyle(description_width=…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "2fcdb448eb7f4d41a6c47dd051092ceb", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='build report structure', max=1.0, style=ProgressStyle(des…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "profile = ProfileReport(data, title='Pandas Profiling Report', html={'style':{'full_width':True}})" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "