"""Config for iTunes Hides Workflow.""" import os feed_name = 'itunes_hides' feed_version = '1.0' s3_feed_name = 'iTunesHides' secrets_path = 'itunes-hides' expected_bucket_owner = os.environ.get('EXPECTED_BUCKET_OWNER', '437795906767') # environment to run in environment = os.environ.get('ENVIRONMENT', 'dev') # bucket for archived data data_bucket = os.environ.get('FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') # bucket for dropped data drop_bucket = os.environ.get('FEED_INGESTION_DROP_BUCKET', 'dev-feed-drop') s3 = { 'archive_path': ( 's3://{data_bucket}/{s3_feed_name}/' 'archives/{{date:%Y-%m-%d}}/'.format( data_bucket=data_bucket, s3_feed_name=s3_feed_name) ), 'preprocessed_path': ( 's3://{data_bucket}/{s3_feed_name}/' 'preprocessed/{{date:%Y-%m-%d}}/preprocessed_hides.csv.gz'.format( data_bucket=data_bucket, s3_feed_name=s3_feed_name) ), 'download_path': ( 's3://{drop_bucket}/filedrop/ops/hides/'.format( drop_bucket=drop_bucket) ) } # PROD.HIDDEN_CONTENT.ITUNES_HIDDEN - source table snowflake_table_names = { 'temp_staging_raw': 'temp_itunes_hidden_{date:%Y%m%d}', 'staging_raw': 'itunes_hidden', 'full_snapshot_path': 'itunes_hidden_master', } file_pattern = r'.*\.zip' snapshots_amount_to_keep = 1 fields = [ 'apple_id', 'upc', 'grid', 'vendor_id', 'artist_name_via_itunes', 'album_name_via_itunes', 'genre', 'track_apple_id', 'track_isrc', 'track_name', 'total_tracks', 'total_discs', 'is_complete', 'content_owner', 'label_name', 'audio_must_be_redelivered_for_itunes_plus', 'reason', 'import_date', 'hide_date', 'ingest_date', 'days_until_hidden', 'original_reason', 'appleid_reason'] bool_mapping = { 'True': 1, 'Yes': 1, 'False': 0, 'No': 0, 'nan': 0, '': 0 } cms_map = { 'Independent Recording Industry Services (IRIS)': 'IRIS', 'The Orchard Enterprises Inc.': 'Orchard' }