""" TIKTOK FRAUDULENT STREAMS REPORT Ingestion Workflow. Ingest data from TIKTOK FRAUDULENT STREAMS REPORT feed. Files downloaded from theorchard s3. """ from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.tiktok_fraudulent_streams_report import config from feed_ingestion.flows.tiktok_fraudulent_streams_report import tasks from feed_ingestion.tasks import date_tasks from feed_ingestion.tasks import overall_status_tasks from feed_ingestion.tasks import validate_raw_data_tasks_sf class Flow(base.FlowBase, base.FlowConfigMixin, base.FlowLoadRawMixinSF): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(config.feed_name, version='1.0') def decider(self, schedule): """Activity decider.""" get_first_day_of_month = schedule( 'get_first_day_of_month', self.get_first_day_of_month) bootstrap = schedule('bootstrap', self.bootstrap_activity, requires=[get_first_day_of_month]) grab_drop_files = schedule( 'grab_drop_files', self.grab_drop_files, requires=[bootstrap] ) check_latest_report_downloaded = schedule( 'check_latest_report_downloaded', self.check_latest_report_downloaded, requires=[grab_drop_files], ) # if files unavailable, let's stop here if check_latest_report_downloaded.result.get( 'check_latest_report_downloaded.stop' ): return # Status updated to DOWNLOADED if files are present set_status_to_downloaded = schedule( 'set_status_to_downloaded', self.set_status_to_downloaded, requires=[check_latest_report_downloaded], ) create_temp_staging_raw_table = schedule( 'create_temp_staging_raw_table', self.create_temp_staging_raw_table, requires=[set_status_to_downloaded], ) load_temp_staging_raw_table = schedule( 'load_temp_staging_raw_table', self.load_temp_staging_raw_table, requires=[create_temp_staging_raw_table], ) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[load_temp_staging_raw_table], ) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[load_staging_raw_table], ) @property def get_first_day_of_month(self): """Get the first day of the month of date.""" return self.create( name='get_first_day_of_month', tasks=base.SyncRunner( date_tasks.get_first_day_of_month.fill( namespace='get_first_day_of_month', date='context_date'))) @property def bootstrap_activity(self): """First activity for the workflow.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='get_first_day_of_month.date', ) ), ) @property def grab_drop_files(self): """Grab files and upload to archive bucket on S3.""" return self.create( name='grab_drop_files', generators=[self.grab_drop_files_generator], tasks=base.SyncRunner( tasks.grab_drop_files.fill( namespace='grab_drop_files', date='bootstrap.date', s3_drop_file_path='bootstrap.s3_drop_file_path', s3_archive_file_path='bootstrap.s3_archive_file_path', ) ), ) def grab_drop_files_generator(self, context): """Generate parameters for grab_drop_files activity. Args: context (dict): The current context. Yields: dict: Dictionary with report_name and feed name, which will be used for writing statuses in DynamoDB. """ yield dict( feed_name='feed_name', date='bootstrap.date', source_file_name='source_file_name', new_file_name='new_file_name', drop_path='drop_path', archive_path='archive_path', ) @property def check_latest_report_downloaded(self): """Check reports which were uploaded on s3.""" return self.create( name='check_latest_report_downloaded', schedule_to_start=48000, tasks=base.SyncRunner( tasks.check_latest_report_downloaded.fill( namespace='check_latest_report_downloaded', date='bootstrap.date', feed_name='bootstrap.feed_name', filename='bootstrap.new_file_name', ) ), ) @property def set_status_to_downloaded(self): """Set overall feed status to DOWNLOADED.""" return self.create( name='set_status_to_downloaded', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_downloaded', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam(garcon_feed_status.STATUS_DOWNLOADED), ) ), ) @property def create_temp_staging_raw_table(self): """Create temp staging raw table.""" return self.create( name='create_temp_staging_raw_table', tasks=base.SyncRunner( tasks.create_temp_staging_raw_table.fill( namespace='create_temp_staging_raw_table', date='bootstrap.date', feed_name='bootstrap.feed_name', temp_staging_raw_table='bootstrap.temp_staging_raw_table', sfdb_params='sfdb_params', secrets_path=StaticParam(config.secrets_path), kwargs='kwargs', ) ), ) @property def load_temp_staging_raw_table(self): """Load temp staging raw table.""" return self.create( name='load_temp_staging_raw_table', tasks=base.SyncRunner( validate_raw_data_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_staging_raw_table', feed_name='bootstrap.feed_name', sfdb_params='sfdb_params', date='bootstrap.date', key_dir='bootstrap.key_dir', temp_staging_raw_table='bootstrap.temp_staging_raw_table', secrets_path='bootstrap.secrets_path', error_limit=( StaticParam(config.snowflake_error_limit)), kwargs='kwargs', ) ), ) @property def set_status_to_ingested(self): """Set overall feed status to to_ingested.""" return self.create( name='set_status_to_ingested', tasks=base.SyncRunner( tasks.set_status_to_ingested.fill( namespace='set_status_to_ingested', date='bootstrap.date', feed_name='bootstrap.feed_name', filename='bootstrap.new_file_name', ) ), )