import snowflake.connector import config import traceback from datetime import datetime from typing import List from utils.list_utils import chunks from utils.snowflake.facebook_data.utils import FacebookImporterUtils from utils.reporting.worker_logger import WorkerLogger from workers.facebook_campaigns_worker import FacebookCampaignsWorker from workers.facebook_ad_sets_worker import FacebookAdSetsWorker from utils.json_helpers import json_load from utils.snowflake.facebook_data.queries import Query from utils.snowflake.facebook_data.models import ( FacebookCampaignsQueryResult, FacebookCampaignModel, FacebookAdSetModel, FacebookAdSetsQueryResult, ) snowflake_connector = snowflake.connector.connect( user=config.SNOWFLAKE_USER, password=config.SNOWFLAKE_PASS, account=config.SNOWFLAKE_ACCOUNT, database=config.ADS_SNOWFLAKE_DB, schema=config.ADS_SNOWFLAKE_SCHEMA, warehouse=config.SNOWFLAKE_WAREHOUSE, ) class FacebookCampaignsFetcher: utils = FacebookImporterUtils() def fetch(self) -> List[FacebookCampaignsQueryResult]: with snowflake_connector as con: marketing_accounts = self.utils.marketing_accounts() data = con.cursor().execute(Query.fetch_campaigns_query(marketing_accounts)).fetchall() return [FacebookCampaignsQueryResult._make(result) for result in data] class FacebookAdSetsFetcher: utils = FacebookImporterUtils() def fetch(self): with snowflake_connector as con: marketing_accounts = self.utils.marketing_accounts() data = con.cursor().execute(Query.fetch_campaign_ads_query(marketing_accounts)).fetchall() return [FacebookAdSetsQueryResult._make(result) for result in data] class FacebookCampaignsImporter: campaigns_importer: FacebookCampaignsFetcher logger: WorkerLogger = WorkerLogger() utils: FacebookImporterUtils = FacebookImporterUtils() def import_campaigns(self): self.logger.info("Facebook campaigns import started", f"Started at {datetime.now()}") try: self.__import_data() except Exception as error: self.logger.error("Facebook campaigns import error", f"{error}\n{traceback.format_exc()}") def __import_data(self): self.campaigns_importer = FacebookCampaignsFetcher() self.__import_campaigns() def __import_campaigns(self): results = self.campaigns_importer.fetch() if not results: self.logger.error("Facebook campaigns import error", "No campaigns returned from datasource") for chunk in chunks(results, 500): campaigns = list(map(self.__map_campaign_model_from, chunk)) FacebookCampaignsWorker().perform_async(campaigns) self.logger.success( "Facebook campaigns import finished", f"Import finished at {datetime.now()} count: {len(results)}" ) def __map_campaign_model_from(self, item) -> FacebookCampaignModel: return FacebookCampaignModel( id=item.campaign_id, name=item.campaign_name, account_id=item.marketing_account_id, objective=item.objective, spend=item.spend, start_date=item.start_date, end_date=item.end_date, platforms=json_load(item.platforms), genders=json_load(item.genders), ages=json_load(item.ages), country_codes=json_load(item.country_codes), links=self.utils.map_links(item.urls), artists=self.utils.map_artists_external_ids(item), budget=item.budget, linkfire_links=self.utils.map_linkfire_links(item.linkfire_links), ) class FacebookAdSetsImporter: importer: FacebookAdSetsFetcher logger: WorkerLogger = WorkerLogger() utils: FacebookImporterUtils = FacebookImporterUtils() def import_ads(self): self.logger.info("Facebook ad sets import started", f"Started at {datetime.now()}") try: self.__import_data() except Exception as error: self.logger.error("Facebook ad sets import error", f"{error}\n{traceback.format_exc()}") def __import_data(self): self.importer = FacebookAdSetsFetcher() self.__import_ads() def __import_ads(self): results = self.importer.fetch() if not results: self.logger.error("Facebook ad sets import error", "No ad sets returned from datasource") for chunk in chunks(results, 500): ad_sets = list(map(self.__map_ad_set_model, chunk)) FacebookAdSetsWorker().perform_async(ad_sets) self.logger.success("Facebook ad sets import finished", f"Import {len(results)} ads sets finished at {datetime.now()}") def __map_ad_set_model(self, item) -> FacebookAdSetModel: return FacebookAdSetModel( id=item.adset_id, name=item.adset_name, campaign_id=str(item.campaign_id), account_id=item.marketing_account_id, start_date=item.start_date, end_date=item.end_date, spend=item.spend, platforms=json_load(item.platforms), genders=json_load(item.genders), ages=json_load(item.ages), country_codes=json_load(item.country_codes), links=self.utils.map_links(item.urls), objective=item.objective, budget=item.budget, )