"""IngestionFeed class and related status, finder methods.""" import collections from feed_status import flows from feed_status import util from feed_status.models.orm import feed_status INGESTION_STATUS_NOT_AVAILABLE = 'not_available' INGESTION_STATUS_DOWNLOADED = 'downloaded' INGESTION_STATUS_POPULATED_RAW_TABLE = 'populated_raw_table' INGESTION_STATUS_INGESTED = 'ingested' INGESTION_STATUS_NOT_INGESTED = 'not_ingested' INGESTION_TYPE_DAILY = 'daily' INGESTION_TYPE_WEEKLY = 'weekly' INGESTION_TYPE_MONTHLY = 'monthly' class IngestionFeed(object): """IngestionFeed class to store configuration about each feed.""" def __init__(self, feed_id, feed_name, status_source_db, update_freq, feed_type, status=None, status_details=None, metrics=None): """Initialize class with attributes. Args: feed_type (str): one of INGESTION_TYPE_xxx. """ self.feed_id = feed_id self.feed_name = feed_name self.status_source_db = status_source_db self.update_freq = update_freq # todo: move feed_type to feed monitor class self.feed_type = feed_type self.status = status self.status_details = status_details self.metrics = metrics def match(self, **kwargs): """Whether instance of IngestionFeed contains provided attributes. Args: kwargs: keys/values to match on. Returns: boolean: True if contains all provided attributes, False otherwise. """ return all( getattr(self, key, None) == val for (key, val) in kwargs.items()) # todo: move to table active_feeds = [ IngestionFeed(feed_id='amazon_digital_services_theorchard', feed_name='Amazon Digital Services Inc. (TheOrchard)', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='amazon_digital_services_sme', feed_name='Amazon Digital Services Inc. (SME)', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='pandora_theorchard', feed_name='Pandora TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='pandora_sme', feed_name='Pandora SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='spotify_theorchard_streams', feed_name='Spotify TheOrchard Streams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='spotify_sme_streams', feed_name='Spotify SME Streams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='spotify_smejpintl_streams', feed_name='Spotify SME intl Streams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='spotify_sme_sub_30_sec_streams', feed_name='Spotify SME Sub 30 Sec Streams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='spotify_charts_daily_top_songs', feed_name='Spotify Charts daily_top_songs', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='spotify_charts_weekly_top_songs', feed_name='Spotify Charts weekly_top_songs', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY), IngestionFeed(feed_id='itunes_theorchard', feed_name='iTunes TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='itunes_sme', feed_name='iTunes SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_id_mapping_sme', feed_name='Apple id mapping sme', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='amazon_music_theorchard_unlimited', feed_name='Amazon Music TheOrchard Unlimited', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='amazon_music_theorchard_prime', feed_name='Amazon Music TheOrchard Prime', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='amazon_music_theorchard_adsupported', feed_name='Amazon Music TheOrchard AdSupported', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='amazon_music_sme_unlimited', feed_name='Amazon Music SME Unlimited', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='amazon_music_sme_prime', feed_name='Amazon Music SME Prime', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='amazon_music_sme_adsupported', feed_name='Amazon Music SME AdSupported', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='deezer_daily_theorchard', feed_name='Deezer Streams + Fraud Daily TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='deezer_daily_sme', feed_name='Deezer Streams + Fraud Daily SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amContentDemographics', feed_name='Apple Music TheOrchard amContentDemographics', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amShazam', feed_name='Apple Music TheOrchard amShazam', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amNonRoyaltySummaryStreams', feed_name='Apple Music TheOrchard ' 'amNonRoyaltySummaryStreams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amLibraryEvents', feed_name='Apple Music TheOrchard amLibraryEvents', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amTotalLibraryAdds', feed_name='Apple Music TheOrchard amTotalLibraryAdds', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amArtists', feed_name='Apple Music TheOrchard amArtists', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amPlaylists', feed_name='Apple Music TheOrchard amPlaylists', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amSongs', feed_name='Apple Music TheOrchard amSongs', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amStreamsSummary', feed_name='Apple Music TheOrchard amStreamsSummary', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_theorchard_amContainer', feed_name='Apple Music TheOrchard amContainer', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amContentDemographics', feed_name='Apple Music SME amContentDemographics', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amShazam', feed_name='Apple Music SME amShazam', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amNonRoyaltySummaryStreams', feed_name='Apple Music SME amNonRoyaltySummaryStreams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amLibraryEvents', feed_name='Apple Music SME amLibraryEvents', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amTotalLibraryAdds', feed_name='Apple Music SME amTotalLibraryAdds', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amArtists', feed_name='Apple Music SME amArtists', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amPlaylists', feed_name='Apple Music SME amPlaylists', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amSongs', feed_name='Apple Music SME amSongs', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amStreamsSummary', feed_name='Apple Music SME amStreamsSummary', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_sme_amContainer', feed_name='Apple Music SME amContainer', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amContentDemographics', feed_name='Apple Music Awal amContentDemographics', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amShazam', feed_name='Apple Music Awal amShazam', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amNonRoyaltySummaryStreams', feed_name='Apple Music Awal ' 'amNonRoyaltySummaryStreams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amLibraryEvents', feed_name='Apple Music Awal amLibraryEvents', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amTotalLibraryAdds', feed_name='Apple Music Awal amTotalLibraryAdds', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amArtists', feed_name='Apple Music Awal amArtists', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amPlaylists', feed_name='Apple Music Awal amPlaylists', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amSongs', feed_name='Apple Music Awal amSongs', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amStreamsSummary', feed_name='Apple Music Awal amStreamsSummary', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='apple_music_awal_amContainer', feed_name='Apple Music Awal amContainer', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='physical-warehouse-reports', feed_name='Physical Warehouse Reports Membran', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='physical_reporting_dpworld', feed_name='Physical Reporting DP World', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), # IngestionFeed(feed_id='proper_daily_shortages', # feed_name='Proper Shortages', # status_source_db='feed_status', # update_freq=1, # feed_type=INGESTION_TYPE_DAILY), # IngestionFeed(feed_id='proper_daily_stock', # feed_name='Proper Stock', # status_source_db='feed_status', # update_freq=1, # feed_type=INGESTION_TYPE_DAILY), # IngestionFeed(feed_id='proper_daily_goodsin', # feed_name='Proper Goodsin', # status_source_db='feed_status', # update_freq=1, # feed_type=INGESTION_TYPE_DAILY), # IngestionFeed(feed_id='proper_daily_sales', # feed_name='Proper Sales', # status_source_db='feed_status', # update_freq=1, # feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='spotify_theorchard_aggregated_streams', feed_name='Spotify TheOrchard Aggregated Streams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='spotify_sme_aggregated_streams', feed_name='Spotify SME Aggregated Streams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='line_theorchard', feed_name='Line TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='line_smej', feed_name='Line SME-J', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='tiktok_theorchard', feed_name='TikTok TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='tiktok_sme', feed_name='TikTok SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='tiktok_smejp', feed_name='TikTok SMEJP (Japan)', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='tiktok_smejpintl', feed_name='TikTok SMEJP (International)', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='delphi_crm_complaint_tracking', feed_name='Delphi CRM complaint_tracking', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='delphi_crm_held_emails', feed_name='Delphi CRM held_emails', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='delphi_crm_mc_unsubscribe_data', feed_name='Delphi CRM MC unsubscribe_data', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='soundcloud_theorchard', feed_name='SoundCloud TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='soundcloud_sme', feed_name='SoundCloud SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='awa_theorchard', feed_name='AWA TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='awa_smej', feed_name='AWA SME-J', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='itunes_tickets', feed_name='iTunes Tickets', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='itunes_hides', feed_name='iTunes Hides', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_channel_names_theorchard', feed_name='YouTube Channel Names TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_channel_names_sme', feed_name='YouTube Channel Names SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_neo4j', feed_name='YouTube Neo4j', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='feature_fm_facebook', feed_name='Feature.FM', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='vevo_theorchard', feed_name='VEVO TheOrchard Channels', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='vevo_sme', feed_name='VEVO SME Channels', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='wynk', feed_name='WYNK', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='beatport', feed_name='Beatport', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='qq_theorchard', feed_name='QQ TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='qq_sme', feed_name='QQ SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='kugou_theorchard', feed_name='Kugou TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='kugou_sme', feed_name='Kugou SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='kuwo_theorchard', feed_name='Kuwo TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='kuwo_sme', feed_name='Kuwo SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='mrc', feed_name='MRC', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='snapchat_creations', feed_name='snapchat creations', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='snapchat_views', feed_name='snapchat views', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), # Chartmetric IngestionFeed(feed_id='chartmetric_charts', feed_name='Chartmetric Charts', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='chartmetric_participants', feed_name='Chartmetric Participants', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='chartmetric_socials', feed_name='Chartmetric Socials', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='chartmetric_tracks', feed_name='Chartmetric Tracks', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='chartmetric_playlists_share', feed_name='Chartmetric Playlists Share', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='chartmetric_socials_backfill', feed_name='Chartmetric Socials Backfill', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_claim_theorchard', feed_name='YouTube TheOrchard Claims', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_claim_sme', feed_name='YouTube SME Claims', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), # Weekly fact_analytics feeds IngestionFeed(feed_id='youtube_weekly', feed_name='YouTube Weekly', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY), IngestionFeed(feed_id='gfk', feed_name='GfK', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY), IngestionFeed(feed_id='tiktok_weekly', feed_name='TikTok Weekly', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY), IngestionFeed(feed_id='sme_latam', feed_name='SME LATAM', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY), IngestionFeed(feed_id='ritmogestion', feed_name='Ritmogestion', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY), # Spotify Marquee IngestionFeed( feed_id='spotify_marquee_theorchard_segment_level', feed_name='Spotify Marquee The orchard Segment Level', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY, ), IngestionFeed( feed_id='spotify_marquee_theorchard_campaign_level', feed_name='Spotify Marquee The orchard Campaign Level', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY, ), IngestionFeed( feed_id='spotify_marketshare_week_ending', feed_name='Spotify marketshare week ending', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY, ), IngestionFeed( feed_id='spotify_track_metrics_playlist_atc', feed_name='Spotify Track Metrics Playlist ATC', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='spotify_track_metrics_track_passion', feed_name='Spotify Track Metrics Track Passion', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='spotify_track_metrics_track_conversion', feed_name='Spotify Track Metrics Track Conversion', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='netease', feed_name='NetEase', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='ticketek_event_booking', feed_name='Ticketek Event Booking', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='seated_opt_ins', feed_name='Seated Opt Ins', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='bandsintown_artist_optins', feed_name='BandsInTown Artist OptIns', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='bandsintown_rsvp', feed_name='BandsInTown RSVP', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='bandsintown_sony_optins', feed_name='BandsInTown Sony OptIns', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='bandsintown_ticketclick', feed_name='BandsInTown TicketClick', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='bandsintown_tracking', feed_name='BandsInTown Tracking', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='gfk_physical', feed_name='GfK Physical', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='gfk_streaming', feed_name='GfK Streaming', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='meta_daily_production_theorchard', feed_name='Meta Daily Production TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='meta_daily_production_sme', feed_name='Meta Daily Production SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='meta_daily_consumption_theorchard', feed_name='Meta Daily Consumption TheOrchard', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='meta_daily_consumption_sme', feed_name='Meta Daily Consumption SME', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), ] for feed_name in flows.Tadas.FEEDS: active_feeds.append( IngestionFeed( feed_id=feed_name, feed_name=feed_name, status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), ) for feed_name in flows.AmazonDatapulse.FEEDS: active_feeds.append( IngestionFeed( feed_id=feed_name, feed_name=' '.join(s.capitalize() for s in feed_name.split('_')), status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), ) # YouTube Bulk TheOrchard Reports active_youtube_bulk_feeds = [ IngestionFeed(feed_id='youtube_asset_theorchard', feed_name='YouTube Asset', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_video_theorchard', feed_name='YouTube Video', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), # youtube_facts flow IngestionFeed(feed_id='youtube_facts_theorchard_asset', feed_name='YouTube Facts Asset', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_facts_theorchard_video', feed_name='YouTube Facts Video', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_asset_conflict', feed_name='YouTube Asset Conflict', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY), ] for report in flows.YouTubeBulkReports.REPORTS_THEORCHARD: feed_name = f'Youtube Bulk {report}' if report in flows.YouTubeBulkReports.FACT_ASSET_REPORTS: feed_name += ' (FA)' if report in flows.YouTubeBulkReports.FACT_VIDEO_REPORTS: feed_name += ' (FV)' active_youtube_bulk_feeds.append( IngestionFeed( feed_id=f'youtube_bulk_reports_theorchard_{report}', feed_name=feed_name, status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), ) # YouTube Bulk SME Reports active_youtube_bulk_sme_feeds = [ IngestionFeed(feed_id='youtube_asset_sme', feed_name='YouTube Asset', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_WEEKLY), IngestionFeed(feed_id='youtube_video_sme', feed_name='YouTube Video', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), # youtube_facts flow IngestionFeed(feed_id='youtube_facts_sme_asset', feed_name='YouTube Facts Asset', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), IngestionFeed(feed_id='youtube_facts_sme_video', feed_name='YouTube Facts Video', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY), ] for report in flows.YouTubeBulkReports.REPORTS_SME: feed_name = f'Youtube Bulk {report}' if report in flows.YouTubeBulkReports.FACT_ASSET_REPORTS: feed_name += ' (FA)' if report in flows.YouTubeBulkReports.FACT_VIDEO_REPORTS: feed_name += ' (FV)' else: feed_type = INGESTION_TYPE_DAILY active_youtube_bulk_sme_feeds.append( IngestionFeed( feed_id=f'youtube_bulk_reports_sme_{report}', feed_name=feed_name, status_source_db='feed_status', update_freq=1, feed_type=feed_type, ), ) active_marketshare_feeds = [ IngestionFeed( feed_id='amazon_prime_marketshare', feed_name='Amazon Prime Marketshare', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='amazon_unlimited_marketshare', feed_name='Amazon Unlimited Marketshare', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='deezer_marketshare', feed_name='Deezer Marketshare', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='itunes_marketshare', feed_name='iTunes Marketshare', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='spotify_marketshare', feed_name='Spotify Marketshare', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), IngestionFeed( feed_id='youtube_red_marketshare', feed_name='Youtube Red Marketshare', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_DAILY, ), ] active_monthly_feeds = [ IngestionFeed( feed_id='tiktok_openescrow', feed_name='TikTok Open Escrow', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_MONTHLY, ), IngestionFeed( feed_id='tiktok_fraudulent_streams_report', feed_name='Tiktok Fraudulent Streams Report', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_MONTHLY ), IngestionFeed( feed_id='insta_reels_creations', feed_name='Insta reels (creations)', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_MONTHLY, ), IngestionFeed( feed_id='insta_reels_views', feed_name='Insta reels (views)', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_MONTHLY, ), IngestionFeed( feed_id='spotify_artificial_streams', feed_name='Spotify Artificial Streams', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_MONTHLY, ), IngestionFeed( feed_id='apple_financial', feed_name='Apple Financial', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_MONTHLY, ), ] for report in flows.YOUTUBE_MONTHLY_REPORTS: active_monthly_feeds.append( IngestionFeed( feed_id=f'youtube_monthly_{report}', feed_name=f'Youtube Monthly {report}', status_source_db='feed_status', update_freq=1, feed_type=INGESTION_TYPE_MONTHLY, ), ) def get_active_feeds(): """Retrieve active feeds. Returns: list: array of IngestionFeed configuration instances. """ return active_feeds def get_active_youtube_bulk_feeds(): """Retrieve youtube bulk feeds. Returns: list: array of IngestionFeed configuration instances. """ return active_youtube_bulk_feeds def get_active_youtube_bulk_sme_feeds(): """Retrieve youtube bulk sme feeds. Returns: list: array of IngestionFeed configuration instances. """ return active_youtube_bulk_sme_feeds def get_active_marketshare_feeds(): """Retrieve marketshare feeds. Returns: list: array of IngestionFeed configuration instances. """ return active_marketshare_feeds def get_active_monthly_feeds(): """Retrieve monthly feeds. Returns: list: array of IngestionFeed configuration instances. """ return active_monthly_feeds def strip_date(date: str) -> str: """Format provided date string replacing dash with space. Args: date: the date to format. Returns: string: String date with dashes replaced with space. """ return date.replace('-', '') # will evolve, but for now feed_id = store_id def get_feed_ingestion_history(feed, from_date, to_date): """Retrieve ingestion history for feed within a date range. DEPRECATED: use dynamo_feed_status.get_ingestion_history and build_status_from_dynamo_items. Args: feed (IngestionFeed): IngestionFeed instance. from_date (date): start date of history. to_date (date): end date of history. Returns: dict: map of dates to feed status. """ status = util.generate_default_date_dict_range(from_date, to_date) if feed.status_source_db == 'feed_status': temp_status = {} feed_history_list = feed_status.get_ingestion_history( feed.feed_id, from_date.isoformat(), to_date.isoformat() ) for feed_history in feed_history_list: if feed_history.get('status'): ingestion_date = feed_history['date'].split('_').pop() # deal with apple music date format bullshit temp_status[ingestion_date] = feed_history['status'].lower() elif 'ingested_files_status' in feed_history: ingestion_date = feed_history['date'].split('_').pop() if feed_history['ingested_files_status']: temp_status[ingestion_date] = INGESTION_STATUS_INGESTED for dt in status.keys(): if dt in temp_status.keys(): status[dt] = temp_status[dt] return status def get_monthly_feed_ingestion_history(feed, from_date, to_date): """Retrieve ingestion history for monthly feed within a date range. Args: feed (IngestionFeed): IngestionFeed instance. from_date (date): start date of history. to_date (date): end date of history. Returns: dict: map of dates to feed status. """ status = util.generate_default_month_date_dict_range(from_date, to_date) if feed.status_source_db == 'feed_status': temp_status = {} feed_history_list = feed_status.get_ingestion_history( feed.feed_id, from_date.isoformat(), to_date.isoformat() ) for feed_history in feed_history_list: if feed_history.get('status'): ingestion_date = feed_history['date'].split('_').pop() # deal with apple music date format bullshit temp_status[ingestion_date] = feed_history['status'].lower() elif 'ingested_files_status' in feed_history: ingestion_date = feed_history['date'].split('_').pop() if feed_history['ingested_files_status']: temp_status[ingestion_date] = INGESTION_STATUS_INGESTED temp_status = collections.OrderedDict( sorted( temp_status.items(), reverse=True, ), ) for dt in temp_status.keys(): status[dt] = temp_status[dt] status = collections.OrderedDict(sorted(status.items(), reverse=True)) return status