import datetime as dt import logging import os import pandas as pd from tadas.domain.trending import prep_timeseries, iron_out_trends from tadas.platform import config from tadas.snowflake import client as snowflake_utils from tadas.platform.caching import cached_df logger = logging.getLogger(__name__) @cached_df() def version_v3(report_date, group_by_country, select_tracks_table): source_schema = config.get('SOURCE_SCHEMA') days_back: int = config.get('TRENDING_DAYS_LOOKBACK_PERIOD_DAYS') logger.info(f"--- {dt.datetime.now().strftime('%H:%M:%S')} pulling streams_active...") q = f""" select f.download_activity_date as report_date, t.isrc_cd, t.geo_country, sum( // lean_forward aka streams_active (spotify + apple + others...) // expected: spotify: 'Artist','Album','Search','Play_Queue', 'Collection' // streams_active has only: ('artist', 'album', 'search', 'play_queue') // so we add collections streams_active + streams_sos_spotify_collection ) AS streams from {source_schema}v_streams_by_track_country_feed_distributor_daily f inner join {select_tracks_table} t on f.isrc = t.isrc_cd {'and f.country_code = t.geo_country' if group_by_country else ''} where f.download_activity_date between %(report_date)s::date - %(days_back)s and %(report_date)s::date group by all """ params = { 'report_date': report_date, 'days_back': days_back, } with snowflake_utils.snowflake_connection() as conn: big_raw_df = snowflake_utils.query_snowflake_to_df(query=q, params=params, connection=conn) big_raw_df = big_raw_df.groupby(['report_date', 'isrc_cd', 'geo_country']).sum().reset_index() big_raw_df['pfn_geo'] = big_raw_df['isrc_cd'].astype(str) + '_' + big_raw_df['geo_country'].astype(str) logger.info(f"--- {dt.datetime.now().strftime('%H:%M:%S')} : got big_raw_df! {len(big_raw_df)} rows of data. Adjusting for Lag.") logger.info(f"--- {dt.datetime.now().strftime('%H:%M:%S')} : prepping the time series - {len(big_raw_df)} rows of data.") timeseries_df = prep_timeseries(big_raw_df) logger.info(f"--- {dt.datetime.now().strftime('%H:%M:%S')} : ironing out trends") ironed_out_intl_df = iron_out_trends(timeseries_df) ironed_out_intl_df['vs_forecast_lift'] = ironed_out_intl_df['streams'] / ironed_out_intl_df[ 'combined_forecast'] - 1.0 return ironed_out_intl_df def save_days_trending(df, table_name): assert 'days_trending' in table_name, 'table name must contain "days_trending"' df['report_date'] = pd.to_datetime(df['report_date']).dt.date snowflake_utils.saveto_snowflake( df=df, myschema=os.getenv('SNOWFLAKE_SCHEMA'), table=table_name, mode='replace', )