"""Config for Amazon Unlimited Marketshare Workflow.""" import os feed_name = 'amazon_unlimited_marketshare' secrets_path = 'amazon_unlimited_marketshare' feed_version = '2.0' s3_feed_name = 'AmazonUnlimitedMarketShare' environment = os.environ.get('Environment', 'dev') data_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') # S3 locations for dropped and archived files s3 = { 'archive': ( 's3://{data_bucket}/{feed_name}/archives/{{date:%Y-%m}}/'.format( data_bucket=data_bucket, feed_name=s3_feed_name)), 'preprocessed': ( 's3://{data_bucket}/{feed_name}/preprocessed/'.format( data_bucket=data_bucket, feed_name=s3_feed_name)), 'preprocessed_filename': 'amazon_unlimited_mkt_share_{date:%Y-%m-%d}.tsv.gz' } snowflake_table_names = { 'staging_raw': 'staging_raw_amazon_unlimited_market_share', 'temp_staging_raw': 'temp_staging_raw_amazon_unlimited_market_share_{date:%Y%m%d}', } file_pattern = '_Monthly_AMU_Summary_{date:%Y%m}' sftp = { 'orchard': { 'host': os.environ.get( 'AMAZON_SFTP_HOST', 'prod.reporting.amazonmusiccatalog.com'), 'username': os.environ.get('AMAZON_ORCHARD_SFTP_USERNAME'), 'password': os.environ.get('AMAZON_ORCHARD_SFTP_PASSWORD'), 'port': int(os.environ.get('AMAZON_SFTP_PORT', 22)) } } sftp_path_template = 'sales/{country_code}/Monthly/Unlimited' new_file_format_countries = ['ROW', 'ROE', 'JP', 'AU'] # since this date all files are in new format new_format_date = '2018-11-01'