{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys, os" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "sys.path.insert(0, '/Users/joel/src/delphi-api')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from delphi_api import const as const" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "APP_NAME=delphi-api\n", "ENVIRONMENT=dev\n", "ROOT_PATH=/Users/joel/src/delphi-api/delphi_api\n", "STATIC_PATH=/Users/joel/src/delphi-api/delphi_api/static\n", "SERVICE_PORT=8000\n", "AWS_DEFAULT_REGION=us-east-1\n", "AUTH0_DOMAIN=dev-delphi.auth0.com\n", "AUTH0_API_AUDIENCE=dev-delphi-api\n", "AUTH0_ALGORITHMS=['RS256']\n", "DEFAULT_OAUTH2_FLOW=client_credentials\n", "OAUTH2_CALLBACK_URL=/oauth/callback\n", "OAUTH2_TOKEN_URL=/oauth/token\n", "ARCHIVE_BUCKET=dev-sme-data-archive\n", "DEFAULT_PAGE_SIZE=5000\n", "MINUTES=60\n", "HOURS=3600\n", "DEFAULT_CACHE_TTL=7200\n", "DEFAULT_CACHE_MAX_SIZE=128\n", "AUTH0_KEY_SIG_CACHE_TTL=36000\n", "RSA_KEY_CACHE_TTL=3600\n", "TRACING_CACHE_TTL=86400\n", "CLIENTS_LIST_SECRET_NAME=delphi/dev/delphi-api/clients_list\n", "CLIENTS_LIST_CACHE_TTL=7200\n", "DEBUG=False\n", "RELOAD_WORKERS=False\n", "LOG_LEVEL=INFO\n", "BIGTABLE_INSTANCE_ID=dev-delphi-consumer-analytics-hdd\n", "BIGTABLE_APP_PROFILE_ID=None\n", "SENTRY_DSN=None\n", "SENTRY_ENABLE=False\n", "SENTRY_PROJECT=delphi-api\n", "POSTGRES_URL=postgres://postgres:postgres@localhost:5432/test_db\n", "POSTGRES_SCHEMA_NAME=public\n", "SQLALCHEMY_DATABASE_URI=postgres://postgres:postgres@localhost:5432/test_db\n", "SQLALCHEMY_TRACK_MODIFICATIONS=True\n", "ENABLE_FAKE_DATA=True\n" ] } ], "source": [ "print(\"\\n\".join(\n", " [f\"{k}={v}\" for k,v in const.__dict__.items() if k.upper() == k]\n", "))" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "import gzip" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "filename = '/Users/joel/src/delphi-api/delphi_api/seeds/data/beta_only/artist.csv.gz'" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "ename": "SyntaxError", "evalue": "unexpected EOF while parsing (, line 1)", "output_type": "error", "traceback": [ "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m with open(filename) as fin:\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m unexpected EOF while parsing\n" ] } ], "source": [ "with open(filename) as fin:\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "artist_df = pd.read_csv(filename)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
artist_idfull_nameparticip_nofirst_namelast_nameisni
0GRAS_7513Ozzy Osbourne7513OzzyOsbourneNaN
1GRAS_154321Future154321NaNNaNNaN
2GRAS_221051Elias221051NaNEliasNaN
3GRAS_620928Sam Fischer620928SamFischerNaN
4GRAS_740645Calvin Harris740645CalvinHarrisNaN
.....................
61GRAS_544492Tiësto544492NaNNaNNaN
62GRAS_582651Ferruccio Tebaldi582651NaNNaNNaN
63GRAS_64596Wu-Tang Clan64596NaNNaNNaN
64GRAS_70813Mike Inez70813MikeInezNaN
65GRAS_71638Zakk Wylde71638ZakkWyldeNaN
\n", "

66 rows × 6 columns

\n", "
" ], "text/plain": [ " artist_id full_name particip_no first_name last_name isni\n", "0 GRAS_7513 Ozzy Osbourne 7513 Ozzy Osbourne NaN\n", "1 GRAS_154321 Future 154321 NaN NaN NaN\n", "2 GRAS_221051 Elias 221051 NaN Elias NaN\n", "3 GRAS_620928 Sam Fischer 620928 Sam Fischer NaN\n", "4 GRAS_740645 Calvin Harris 740645 Calvin Harris NaN\n", ".. ... ... ... ... ... ...\n", "61 GRAS_544492 Tiësto 544492 NaN NaN NaN\n", "62 GRAS_582651 Ferruccio Tebaldi 582651 NaN NaN NaN\n", "63 GRAS_64596 Wu-Tang Clan 64596 NaN NaN NaN\n", "64 GRAS_70813 Mike Inez 70813 Mike Inez NaN\n", "65 GRAS_71638 Zakk Wylde 71638 Zakk Wylde NaN\n", "\n", "[66 rows x 6 columns]" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "artist_df" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "df1 = pd.read_parquet(\"/Users/joel/Downloads/data_for_bigtable.parquet\")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
partner_keypartner_codereport_datecountry_codeartist_idartist_nameprimary_artist_nameproduct_namesuppl_titleproduct_family_nameproduct_keygrid_noisrc_cdproduct_family_noupc_cdfin_label_namefin_label_parent_namerep_owner_nametotal_streams
01spotify2015-09-13FR154321FutureFutureCovered N MoneyCovered N Money3239407A10301A0003133755LUSSM114013893082441886444533263Epic/Legacy CatLegacyEpic62
15amusic2017-01-20PT1192524Travis ScottTravis Scottbeibs in the trapbeibs in the trap3797941A10301A0003608699YUSSM116075513749105Epic/Grand Hustle/Travis Scott LP2Epic Records GroupEpic48
21spotify2017-07-12US7513Ozzy OsbourneOzzy OsbourneI Don't Want To Change The WorldI Don't Want To Change The World1212368A10301A0000647012UUSSM10204870605666Epic/Legacy CatLegacyEpic54
35amusic2016-02-16UA7513Ozzy OsbourneOzzy OsbourneSuicide SolutionSuicide Solution2675688A10301A0002641714ZUSSM11002848605501Epic/Legacy CatLegacyEpic2
41spotify2017-11-08HU7513Ozzy OsbourneOzzy OsbourneCrazy TrainCrazy Train1073338A10301A0000491039XUSSM10204723605491Epic/Legacy CatLegacyEpic29
............................................................
726623085amusic2019-12-11BM154321FutureFutureLove Thy EnemiesLove Thy Enemies4377984A10301A0004109695PUSSM119038324547156EpicEpic Records GroupEpic5
7266230960prime-music2018-11-22IT7513Ozzy OsbourneOzzy OsbourneRock 'n' Roll RebelRock 'n' Roll Rebel1073362A10301A0000491064VUSSM18300024605515Epic/Legacy CatLegacyEpic1
7266231010vevo2018-11-25BS1241815Camila CabelloCamila CabelloShe Loves ControlAudioShe Loves Control4110144A10301A0003873750AUSSM218000594033262886446944647EpicEpic Records GroupEpic2
726623111spotify2019-03-04TN1192524Travis ScottTravis ScottHOUSTONFORNICATIONHOUSTONFORNICATION4225365A10301A0003976522TUSSM118066714327174Epic/Grand Hustle/Travis Scott LP3/All Other T...Epic Records GroupEpic22
7266231210vevo2019-08-01JE1192524Travis ScottTravis ScottWAKE UPAudioWAKE UP4225897A10301A0003976880IUSSM218010254327167886447253298Epic/Grand Hustle/Travis Scott LP3/Weekend/Wak...Epic Records GroupEpic3
\n", "

72662313 rows × 19 columns

\n", "
" ], "text/plain": [ " partner_key partner_code report_date country_code artist_id \\\n", "0 1 spotify 2015-09-13 FR 154321 \n", "1 5 amusic 2017-01-20 PT 1192524 \n", "2 1 spotify 2017-07-12 US 7513 \n", "3 5 amusic 2016-02-16 UA 7513 \n", "4 1 spotify 2017-11-08 HU 7513 \n", "... ... ... ... ... ... \n", "72662308 5 amusic 2019-12-11 BM 154321 \n", "72662309 60 prime-music 2018-11-22 IT 7513 \n", "72662310 10 vevo 2018-11-25 BS 1241815 \n", "72662311 1 spotify 2019-03-04 TN 1192524 \n", "72662312 10 vevo 2019-08-01 JE 1192524 \n", "\n", " artist_name primary_artist_name \\\n", "0 Future Future \n", "1 Travis Scott Travis Scott \n", "2 Ozzy Osbourne Ozzy Osbourne \n", "3 Ozzy Osbourne Ozzy Osbourne \n", "4 Ozzy Osbourne Ozzy Osbourne \n", "... ... ... \n", "72662308 Future Future \n", "72662309 Ozzy Osbourne Ozzy Osbourne \n", "72662310 Camila Cabello Camila Cabello \n", "72662311 Travis Scott Travis Scott \n", "72662312 Travis Scott Travis Scott \n", "\n", " product_name suppl_title \\\n", "0 Covered N Money \n", "1 beibs in the trap \n", "2 I Don't Want To Change The World \n", "3 Suicide Solution \n", "4 Crazy Train \n", "... ... ... \n", "72662308 Love Thy Enemies \n", "72662309 Rock 'n' Roll Rebel \n", "72662310 She Loves Control Audio \n", "72662311 HOUSTONFORNICATION \n", "72662312 WAKE UP Audio \n", "\n", " product_family_name product_key grid_no \\\n", "0 Covered N Money 3239407 A10301A0003133755L \n", "1 beibs in the trap 3797941 A10301A0003608699Y \n", "2 I Don't Want To Change The World 1212368 A10301A0000647012U \n", "3 Suicide Solution 2675688 A10301A0002641714Z \n", "4 Crazy Train 1073338 A10301A0000491039X \n", "... ... ... ... \n", "72662308 Love Thy Enemies 4377984 A10301A0004109695P \n", "72662309 Rock 'n' Roll Rebel 1073362 A10301A0000491064V \n", "72662310 She Loves Control 4110144 A10301A0003873750A \n", "72662311 HOUSTONFORNICATION 4225365 A10301A0003976522T \n", "72662312 WAKE UP 4225897 A10301A0003976880I \n", "\n", " isrc_cd product_family_no upc_cd \\\n", "0 USSM11401389 3082441 886444533263 \n", "1 USSM11607551 3749105 \n", "2 USSM10204870 605666 \n", "3 USSM11002848 605501 \n", "4 USSM10204723 605491 \n", "... ... ... ... \n", "72662308 USSM11903832 4547156 \n", "72662309 USSM18300024 605515 \n", "72662310 USSM21800059 4033262 886446944647 \n", "72662311 USSM11806671 4327174 \n", "72662312 USSM21801025 4327167 886447253298 \n", "\n", " fin_label_name \\\n", "0 Epic/Legacy Cat \n", "1 Epic/Grand Hustle/Travis Scott LP2 \n", "2 Epic/Legacy Cat \n", "3 Epic/Legacy Cat \n", "4 Epic/Legacy Cat \n", "... ... \n", "72662308 Epic \n", "72662309 Epic/Legacy Cat \n", "72662310 Epic \n", "72662311 Epic/Grand Hustle/Travis Scott LP3/All Other T... \n", "72662312 Epic/Grand Hustle/Travis Scott LP3/Weekend/Wak... \n", "\n", " fin_label_parent_name rep_owner_name total_streams \n", "0 Legacy Epic 62 \n", "1 Epic Records Group Epic 48 \n", "2 Legacy Epic 54 \n", "3 Legacy Epic 2 \n", "4 Legacy Epic 29 \n", "... ... ... ... \n", "72662308 Epic Records Group Epic 5 \n", "72662309 Legacy Epic 1 \n", "72662310 Epic Records Group Epic 2 \n", "72662311 Epic Records Group Epic 22 \n", "72662312 Epic Records Group Epic 3 \n", "\n", "[72662313 rows x 19 columns]" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.1" } }, "nbformat": 4, "nbformat_minor": 2 }