"""Store Specific Spotify Strikes Workflow tasks.""" from garcon import task from snowflake_connector.etl_connector import SnowflakeSQLExecutor from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.flows.spotify_strikes import config from feed_ingestion.flows.spotify_strikes import utils from feed_ingestion.util import datalytics_util from feed_ingestion.util import read_sheet as gdoc from feed_ingestion.util.aws import s3 as s3utils STOP_RESPONSE = {'stop': True} sql_loader = SQLLoader(__file__) @task.decorate(timeout=1000) def bootstrap(activity): """Bootstrap workflow. Args: activity (ActivityWorker): The Garcon activity worker. Returns: dict: Initial context for the workflow. """ return { 'strikes_sheet_id': config.STRIKES_SHEET_ID, 'snowflake_file_format_tsv': config.SNOWFLAKE_FILE_FORMAT_TSV, 's3_path': config.S3_PATH, 'email_list': config.EMAIL_LIST, 'strikes_table': config.STRIKES_TABLE, 'googledoc_id': config.GOOGLEDOC_ID } @task.decorate(timeout=10000) def process_files( activity, strikes_sheet_id, s3_path, file_format_tsv, strikes_table, aws): """Process files. Args: activity (ActivityWorker): The Garcon activity worker. strikes_sheet_id (str): Id of sheet. s3_path (str): s3 path to pucket. file_format_tsv (str): TSV table name. strikes_table (str): Name of strikes table. aws (dict): AWS credentials. Returns: dict: STOP_RESPONSE or confirmation. """ try: df_labels = utils.preprocess_raw( gdoc.sheet_2_df( s_id=strikes_sheet_id, sheet='Sheet1', t_range='A1:E' ) ) except Exception: activity.logger.error('gdoc id %s not found!' % strikes_sheet_id) return STOP_RESPONSE try: df_labels_old = s3utils.read_csv(s3_path, sep='\t') except Exception: activity.logger.error('%s not found!' % s3_path) return STOP_RESPONSE df_labels_old_not_exist = isinstance(df_labels_old, str)\ and 'The specified key does not exist' in df_labels_old if df_labels_old_not_exist \ or not df_labels.reset_index(drop=True).equals( df_labels_old.reset_index(drop=True)): activity.logger.info('New data in the spreadsheet. Processing.') s3utils.to_csv( df=df_labels, s3_path=s3_path, index=False, sep='\t' ) sf_config = get_sf_config(config.secrets_path) SnowflakeSQLExecutor(sf_config).execute_query( sql_loader, 'truncate_table', { 'table_name': strikes_table } ) SnowflakeSQLExecutor(sf_config).execute_query( sql_loader, 'load_from_s3', { 'table_name': strikes_table, 's3_path': s3_path, 'aws_key_id': aws['access_key'], 'aws_secret_key': aws['access_secret'], 'format_name': file_format_tsv } ) return {'files_processed': True} activity.logger.info('No new data in the spreadsheet.') return STOP_RESPONSE @task.decorate(timeout=10000) def send_emails(activity, email_list, googledoc_id): """Process files found on S3. Args: activity (ActivityWorker): The Garcon activity worker. email_list (list): List of email to send notification to. googledoc_id (str): ID of Google Spreadsheet. """ BODY = ("""Spotify Label Strikes up-to-date with: \nhttps://docs.google.com/spreadsheets/d/ {googledoc_id}/edit#gid=0 \nhttps://theorchard.looker.com/x/WZrWgzN""".format( googledoc_id=googledoc_id)) ses_response = datalytics_util.send_emails( to=email_list, sub='Spotify Label Strikes Updated.', body=BODY ) activity.logger.debug('SES response: {}.'.format(ses_response))