from sqlalchemy import ( SmallInteger, Column, Enum, String, Integer, ForeignKey) from sqlalchemy.orm import relationship, backref from feed_status.connectors import mysql from feed_status.models.orm import feed_ingestion_history from feed_status.models.orm import feed_status from feed_status import util class IngestionFeed(object): def __init__(self, feed_id, feed_name, status_source_db, update_freq, strip_date=False): self.feed_id = feed_id self.feed_name = feed_name self.status_source_db = status_source_db self.update_freq = update_freq self.strip_date=strip_date def format_date(self, date): if self.strip_date: date = date.replace('-', '') return date # todo: move to table active_feeds = [ IngestionFeed(feed_id=11, feed_name='24/7', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=187, feed_name='Amazon Digital Services Inc.', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=7, feed_name='eMusic', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=496, feed_name='Google Music', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=49602, feed_name='Google Play', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=392, feed_name='Hulu', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=487, feed_name='X-Box', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=4, feed_name='Rhapsody', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=286, feed_name='Spotify', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=45302, feed_name='YouTube Claims', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id=45305, feed_name='YouTube Weekly', status_source_db='etl_logger', update_freq=1), IngestionFeed(feed_id='iTunes', feed_name='iTunes', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='vudu_feed_ingestion', feed_name='Vudu', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='Apple Music', feed_name='Apple Music', status_source_db='feed_status', update_freq=1, strip_date=True), IngestionFeed(feed_id='iTunes Fanclub', feed_name='iTunes Fanclub', status_source_db='feed_status', update_freq=1, strip_date=True), IngestionFeed(feed_id='Amazon Instant Video Daily', feed_name='Amazon Instant Video Daily', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='deezer_daily', feed_name='Deezer Daily', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='apple_music_streams', feed_name='Apple Music Streams', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='youtube_vod_weekly', feed_name='YouTube Movies', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='proper_daily_shortages', feed_name='Proper Shortages', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='proper_daily_stock', feed_name='Proper Stock', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='proper_daily_goodsin', feed_name='Proper Goodsin', status_source_db='feed_status', update_freq=1), IngestionFeed(feed_id='proper_daily_sales', feed_name='Proper Sales', status_source_db='feed_status', update_freq=1), ] def get_active_feeds(): return active_feeds # will evolve, but for now feed_id = store_id # this is too slow, need to pass a list of feeds and make one mysql query def get_feed_ingestion_history(feed, from_date, to_date): session = mysql.art_relations_session() # @todo(jpenner): make date specific method status = util.generate_default_date_dict(120, 1) if feed.status_source_db == 'etl_logger': feed_history_list_query = session.query( feed_ingestion_history.EtlLoggerHistory).\ filter( feed_ingestion_history.EtlLoggerHistory.storeid == feed.feed_id, feed_ingestion_history.EtlLoggerHistory.downloaddate >= from_date, feed_ingestion_history.EtlLoggerHistory.downloaddate <= to_date) feed_history_list = feed_history_list_query.all() session.close() for feed_history in feed_history_list: status[str(feed_history.downloaddate)] = feed_history.status elif feed.status_source_db == 'feed_status': temp_status = {} feed_history_list = feed_status.get_ingestion_history( feed.feed_id, feed.format_date(str(from_date)), feed.format_date(str(to_date))) # deal with apple music date format bullshit for feed_history in feed_history_list: if feed_history['status']: temp_status[feed_history['date']] = feed_history['status'].lower() for dt in status.keys(): formatted_dt = feed.format_date(dt) if formatted_dt in temp_status: status[dt]=temp_status[formatted_dt] return status # currently just for testing def get_feed(feed_name): return [IngestionFeed(feed_id='iTunes', feed_name='iTunes', status_source_db='feed_status', update_freq=1)]