from datetime import datetime, timedelta from collections import defaultdict from smelog.factory import BoundLogger from download_spotify_charts import config from download_spotify_charts import utils from download_spotify_charts.charts import SpotifyCharts from download_spotify_charts.constants import ChartBreakdown, ChartType from download_spotify_charts.s3 import Client as S3Client from download_spotify_charts.state_manager import StateManager def handler(logger: BoundLogger, event: dict): """Lambda job. Args: logger: Logger instance. event: Lambda event obj. Returns: JSON serializable response """ s3_client = S3Client() spotify_charts = SpotifyCharts(logger=logger) charts_list = spotify_charts.get_charts(config.CHART_TYPE_LIST, config.CHART_BREAKDOWN_LIST) latest_dates_mapping = spotify_charts.get_latest_date_mapping() state_mapping = defaultdict(dict) try: for chart_item in charts_list: chart_type, chart_breakdown, country_code, chart_id = ( ChartType(chart_item["type"]), ChartBreakdown(chart_item["breakdown"]), chart_item["country_code"], chart_item["chart_id"], ) state = state_mapping[chart_type].get(chart_breakdown) if state is None: state = StateManager(s3_client, chart_type, chart_breakdown, logger) state_mapping[chart_type][chart_breakdown] = state default_date = utils.get_default_date(chart_breakdown) if chart_breakdown == ChartBreakdown.WEEKLY and not state.is_empty and config.SKIP_WEEKLY_DAYS_COUNT_RANGE: date_diff = (datetime.utcnow().date() - state.get_min_date(chart_breakdown)).days if config.SKIP_WEEKLY_DAYS_COUNT_RANGE[0] <= date_diff <= config.SKIP_WEEKLY_DAYS_COUNT_RANGE[1]: logger.info(f"{chart_type.value}/{chart_breakdown.value} skipping") continue last_date, ts = state.get_country_item(chart_breakdown, country_code, default_date) if last_date < default_date: last_date = default_date max_date = latest_dates_mapping[chart_breakdown][chart_type][country_code] current_dates = utils.generate_dates(chart_breakdown, last_date, max_date) if not current_dates: ts = datetime.fromisoformat(ts) last_update_hours = ( datetime.utcnow() + timedelta(minutes=config.RELOAD_ROUND_MINUTES) - ts ).total_seconds() / 3600 if ( last_update_hours > config.RELOAD_EACH_HOURS and ( chart_breakdown != ChartBreakdown.WEEKLY or not config.RELOAD_WEEKLY_DAYS_COUNT or (datetime.utcnow().date() - max_date).days <= config.RELOAD_WEEKLY_DAYS_COUNT ) ): current_dates.append(max_date) logger.info( f"{chart_type.value}/{country_code}/{chart_breakdown.value} last {last_date} " + (f"dates: {','.join([i.isoformat() for i in current_dates])}" if current_dates else "up-to-date") ) if not current_dates: continue for chart_date in sorted(current_dates): data = spotify_charts.get_charts_tracks(chart_id, chart_date) if data: data = utils.data_to_csv(chart_type, data) if data: s3_client.upload_csv( utils.get_csv_filename(chart_type, chart_breakdown, country_code, chart_date), data ) state.set_country_item(country_code, chart_date) finally: for states in state_mapping.values(): for state in states.values(): state.save()