import json from typing import Any, Mapping from analytics.constants import cache from analytics.logic.data_availability import get_max_available_date from analytics.queries.format import format_row from analytics.queries.tiktok import ( TiktokAggregatedSummary, TiktokAggregatedTimeSeries, TiktokTimeSeries, ) from analytics.utils import date as date_utils from analytics.utils.cache import cache_in_redis AGGREGATION_FIELDS = {"BY_CONTENT_TYPE": "content_type", "BY_COUNTRY": "country_code"} DEFAULT_TYPE = "ALL" @cache_in_redis(ttl=cache.ONE_DAY) def get_tiktok_time_series( query_params: Mapping[str, Any], permissions: Mapping[str, Any] ): query_type = query_params.get("type") aggregation_field = AGGREGATION_FIELDS.get(query_type, DEFAULT_TYPE) if aggregation_field: query_params["aggregation_field"] = aggregation_field max_available_date = get_max_available_date().strftime("%Y-%m-%d") days_back = query_params.get("days_back") if days_back: days_back = int(days_back) query_params["days_back"] = days_back query_params = {**query_params, "max_available_date": max_available_date} query = TiktokTimeSeries({**query_params, **permissions}) time_series = query.execute() time_series = [format_row(data_point) for data_point in time_series] if not query_params.get("summary") and days_back: master_calendar = set( date_utils.generate_date_range_starting_from_date( start_date=max_available_date, days_back=days_back ) ) if aggregation_field != "ALL": if aggregation_field == "content_type": distinct_ids = {"UGC", "PGC"} else: distinct_ids = {item["id"] for item in time_series} for date in master_calendar: for id in distinct_ids: empty_data_point = { "date": date, "id": id, "streams": 0, "views": 0, "likes": 0, "favorites": 0, "creations": 0, "shares": 0, "comments": 0, } if not any( ( item["date"] == empty_data_point["date"] and item["id"] == empty_data_point["id"] ) for item in time_series ): time_series.append(empty_data_point) else: for date in master_calendar: empty_data_point = { "date": date, "streams": 0, "views": 0, "likes": 0, "favorites": 0, "creations": 0, "shares": 0, "comments": 0, } if not any( (item["date"] == empty_data_point["date"]) for item in time_series ): time_series.append(empty_data_point) if aggregation_field != "ALL": time_series = sorted(time_series, key=lambda k: (k["id"], k["date"])) else: time_series = sorted(time_series, key=lambda k: k["date"]) return time_series @cache_in_redis(ttl=cache.ONE_DAY) def get_tiktok_aggregated_summary( query_params: Mapping[str, Any], permissions: Mapping[str, Any] ): query = TiktokAggregatedSummary({**query_params, **permissions}) aggregate = [format_row(data_point) for data_point in query.execute()] if aggregate: parsed = json.loads(aggregate[0]["result"].lower()) else: parsed = {} return parsed @cache_in_redis(ttl=cache.ONE_DAY) def get_tiktok_aggregated_time_series( query_params: Mapping[str, Any], permissions: Mapping[str, Any] ): max_available_date = get_max_available_date().strftime("%Y-%m-%d") query_params = {**query_params, "max_available_date": max_available_date} query = TiktokAggregatedTimeSeries({**query_params, **permissions}) aggregated_time_series = query.execute() aggregated_time_series = [ format_row(data_point) for data_point in aggregated_time_series ] master_calendar = set( date_utils.generate_date_range_starting_from_date( start_date=max_available_date, days_back=query_params.get("days_back", 28) ) ) if aggregated_time_series: parsed = json.loads(aggregated_time_series[0]["result"].lower()) all_other_countries_creations = parsed["all_other_countries_creations"] all_other_countries_creations_dates_with_data = set() for data_point in parsed["all_other_countries_creations"]: all_other_countries_creations_dates_with_data.add( data_point["download_activity_date"] ) all_other_countries_creations_dates_with_no_data_to_add = ( master_calendar - all_other_countries_creations_dates_with_data ) for date in all_other_countries_creations_dates_with_no_data_to_add: empty_data_point = { "download_activity_date": date, "creations": 0, "views": 0, } all_other_countries_creations.append(empty_data_point) if all_other_countries_creations_dates_with_no_data_to_add: parsed["all_other_countries_creations"] = sorted( all_other_countries_creations, key=lambda k: k["download_activity_date"] ) all_other_countries_views = parsed["all_other_countries_views"] all_other_countries_views_dates_with_data = set() for data_point in parsed["all_other_countries_views"]: all_other_countries_views_dates_with_data.add( data_point["download_activity_date"] ) all_other_countries_views_dates_with_no_data_to_add = ( master_calendar - all_other_countries_views_dates_with_data ) for date in all_other_countries_views_dates_with_no_data_to_add: empty_data_point = { "download_activity_date": date, "creations": 0, "views": 0, } all_other_countries_views.append(empty_data_point) if all_other_countries_views_dates_with_no_data_to_add: parsed["all_other_countries_views"] = sorted( all_other_countries_views, key=lambda k: k["download_activity_date"] ) topn_countries_ordered_by_all_time_creations = parsed[ "topn_countries_ordered_by_all_time_creations" ] all_distinct_countries = set() for data_point in topn_countries_ordered_by_all_time_creations: all_distinct_countries.add(data_point["country_code"]) for country in all_distinct_countries: for date in master_calendar: empty_data_point = { "download_activity_date": date, "creations": 0, "views": 0, "country_code": country, } if not any( ( item["download_activity_date"] == empty_data_point["download_activity_date"] and item["country_code"] == empty_data_point["country_code"] ) for item in topn_countries_ordered_by_all_time_creations ): topn_countries_ordered_by_all_time_creations.append( empty_data_point ) if topn_countries_ordered_by_all_time_creations: parsed["topn_countries_ordered_by_all_time_creations"] = sorted( topn_countries_ordered_by_all_time_creations, key=lambda k: (k["country_code"], k["download_activity_date"]), ) topn_countries_ordered_by_all_time_views = parsed[ "topn_countries_ordered_by_all_time_views" ] all_distinct_countries = set() for data_point in topn_countries_ordered_by_all_time_views: all_distinct_countries.add(data_point["country_code"]) for country in all_distinct_countries: for date in master_calendar: empty_data_point = { "download_activity_date": date, "creations": 0, "views": 0, "country_code": country, } if not any( ( item["download_activity_date"] == empty_data_point["download_activity_date"] and item["country_code"] == empty_data_point["country_code"] ) for item in topn_countries_ordered_by_all_time_views ): topn_countries_ordered_by_all_time_views.append(empty_data_point) if topn_countries_ordered_by_all_time_views: parsed["topn_countries_ordered_by_all_time_views"] = sorted( topn_countries_ordered_by_all_time_views, key=lambda k: (k["country_code"], k["download_activity_date"]), ) else: parsed = {} return parsed