"""Meta Daily Ingestion Workflow. Ingests Meta (Facebook) daily consumption and production reports into Snowflake staging_raw tables. Files are delivered by Meta to the sme-ca-prod-partners S3 bucket for both SME and Orchard licensors. Each report type (consumption, production) runs as a separate SWF workflow execution. Within each workflow, both licensor files are tracked individually so that a file available on day N is loaded immediately while the other licensor's file is picked up on a later run when it arrives. """ import datetime from feed_ingestion.flows import base from feed_ingestion.flows.meta_daily import config, tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Meta Daily ingestion workflow.""" def __init__(self): """Initialise the flow.""" super().__init__(config.feed_name, version='1.0') def contextified_feed_name(self, context): """Return feed name scoped to the report type. Used by the exec layer to derive a unique workflow ID per report. Args: context (dict): Workflow context containing 'report'. Returns: str: e.g. 'meta_daily-consumption' """ return f'{config.feed_name}-{context["report"]}' def workflow_id(self, initial_context): """Generate a unique SWF workflow ID. Args: initial_context (dict): Initial workflow context. Returns: str: e.g. 'meta_daily-consumption-2026-04-17' """ if 'context_date' not in initial_context: date = datetime.datetime.today().strftime('%Y-%m-%d') else: date = initial_context['context_date'] return f'{self.contextified_feed_name(initial_context)}-{date}' def decider(self, schedule): """Define the activity DAG. Args: schedule (callable): Garcon scheduler. """ bootstrap = schedule('bootstrap', self.bootstrap_activity) if bootstrap.result.get('bootstrap.stop') is True: return grab_files = schedule( 'grab_available_files', self.grab_available_files_activity, requires=[bootstrap], ) if grab_files.result.get('grab_available_files.stop'): return load_raw = schedule( 'load_staging_raw', self.load_staging_raw_activity, requires=[grab_files], ) schedule( 'set_overall_status_if_complete', self.set_overall_status_activity, requires=[load_raw], ) @property def bootstrap_activity(self): """Bootstrap activity — checks status and builds pending_sources.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', report='report', reload='reload', ) ), ) @property def grab_available_files_activity(self): """Copy pending files from SME S3 to the archive bucket.""" return self.create( name='grab_available_files', tasks=base.AsyncRunner( tasks.grab_available_files.fill( namespace='grab_available_files', pending_sources='bootstrap.pending_sources', ) ), ) @property def load_staging_raw_activity(self): """Load each available source file into the staging_raw table.""" return self.create( name='load_staging_raw', tasks=base.SyncRunner( tasks.load_staging_raw.fill( namespace='load_staging_raw', available_sources='grab_available_files.available_sources', staging_raw_table='bootstrap.staging_raw_table', date='bootstrap.date', report='bootstrap.report', archive_s3_path='bootstrap.archive_s3_path', ) ), ) @property def set_overall_status_activity(self): """Set overall STATUS_INGESTED when all licensor sources are done.""" return self.create( name='set_overall_status_if_complete', tasks=base.SyncRunner( tasks.set_overall_status_if_complete.fill( namespace='set_overall_status_if_complete', report='bootstrap.report', report_feed_name='bootstrap.report_feed_name', date='bootstrap.date', ) ), )