"""Config for the Amazon Digital Services Ingestion Workflow.""" import datetime import os environment = os.environ.get('Environment', 'dev') feed_name = 'amazon_digital_services' secrets_path = 'amazon_digital_services' # SELECT feedid from dim_feed where feedname = 'Amazon Downloads' feedid = 16 expected_bucket_owner = os.environ.get('EXPECTED_BUCKET_OWNER', '437795906767') licensors = ['theorchard', 'sme'] # root AWS bucket s3_bucket = os.environ.get( 'FEED_INGESTION_DATA_BUCKET', 'dev-cucumbers') if environment == 'production': s3_bucket = 'cucumbers' s3_archive_path_template = ( 's3://' + s3_bucket + '/AmazonV2/archives/{date:%Y-%m-%d}/{licensor}/') s3_tmp_path_template = ( 's3://' + s3_bucket + '/AmazonV2/tmp/{date:%Y-%m-%d}/{licensor}/') sme_drop_bucket = os.environ.get('SME_FEED_INGESTION_DROP_BUCKET', 'sme-ca-prod-partners') sme_secrets_path = os.environ.get('SME_DROP_BUCKET_SECRETS_PATH', 'swf-sme-ca-prod-partners-s3-access') snowflake_table_names = { 'staging_raw': { 'theorchard': 'staging_raw_amazon_v2', 'sme': 'staging_raw_amazon_v2_sme', }, 'staging_fact_analytics': 'staging_fact_analytics_{feed_name}_{date:%Y%m%d}' } # configuration for available and required vendors and countries # key - licensor # value - array of tuples: # [0] - date since which this configuration applicable. # Dates shall go in ascending order # [1] - dict: key - vendor code, value - dict of countries # dict: keys 'required', 'optional' values - list of countries vendors_countries = { 'theorchard': [ # Note: this is a kind of random date, not actual starting date. (datetime.date(2017, 1, 1), { 'OR': { 'required': 'AT,DE,FR,GB,US'.split(','), 'optional': 'CA,CH,ES,IN,IT,JP'.split(','), }, 'RED': { 'required': [], 'optional': 'AT,CA,CH,DE,ES,FR,GB,IN,IT,JP,US'.split(','), }, }), ], 'sme': [ (datetime.date(2016, 12, 4), { 'PLV1': { 'required': ['US'], 'optional': [], } }), (datetime.date(2018, 1, 1), { 'PLV1': { 'required': ['US'], 'optional': [], }, 'PJ20': { 'required': 'AT,CH,DE,ES,FR,IT,UK'.split(','), 'optional': [], }, 'P748': { 'required': ['US'], 'optional': [], }, }), # one day 2019-08-19: missing P748_D_20190819_20190819_US.txt (datetime.date(2019, 8, 19), { 'PLV1': { 'required': ['US'], 'optional': [], }, 'PJ20': { 'required': 'AT,CH,DE,ES,FR,IT,UK'.split(','), 'optional': [], }, 'P748': { 'required': [], 'optional': ['US'], }, }), (datetime.date(2019, 8, 20), { 'PLV1': { 'required': ['US'], 'optional': [], }, 'PJ20': { 'required': 'AT,CH,DE,ES,FR,IT,UK'.split(','), 'optional': [], }, 'P748': { 'required': ['US'], 'optional': [], }, }), ] } filename_templates = { 'theorchard': { # ZQOR_20200401_Summary_Statement.csv.zip 'summary_zip': 'ZQ{vendor_code}_{date:%Y%m%d}_Summary_Statement.csv.zip', # noqa:E501 # ZQOR_20200401_Summary_Statement_US.csv.zip 'summary_country_zip': 'ZQ{vendor_code}_{date:%Y%m%d}_Summary_Statement_{country}.csv.zip', # noqa:E501 # ZQOR_20210505_Summary_Statement_us.csv.zip 'summary_country_lower_zip': 'ZQ{vendor_code}_{date:%Y%m%d}_Summary_Statement_{country_lower}.csv.zip', # noqa:E501 # ZQOR_AmazonMP3_JP_Daily_20200401.txt 'daily_zip': 'ZQ{vendor_code}_AmazonMP3_{country}_Daily_{date:%Y%m%d}.txt.zip', # noqa:E501 'remote_dir': 'sales/{country}/Daily/', # pattern used by check_files_on_s3 'search_pattern': ( r'.+(ZQ.+_{date:%Y%m%d}_Summary_Statement_\w+\.csv\.zip)|' r'.+(ZQ.+_AmazonMP3_JP_Daily_{date:%Y%m%d}\.txt\.zip)'), }, 'sme': { # filenames using since 2017-07-12 # example P748_D_20180101_20180101_US.txt 'd_country': '{vendor_code}_D_{date:%Y%m%d}_{date:%Y%m%d}_{country}.txt', # noqa:E501 # filenames using before 2017-07-11 (inclusive) # example PLV1_D_20161204_20161204.txt 'd': '{vendor_code}_D_{date:%Y%m%d}_{date:%Y%m%d}.txt', # P748_PO_D_20180101_20180101_US.txt 'po': '{vendor_code}_PO_D_{date:%Y%m%d}_{date:%Y%m%d}_{country}.txt', 'remote_dir': 'amazon/in/download/{vendor_code}/', # pattern used by check_files_on_s3 'search_pattern': ( r'.+(.+_D_{date:%Y%m%d}_{date:%Y%m%d}_.+\.txt)'), } } # Drop location for raw files # before 2022-02-01: 'digital-music-ftp.amazonmp3.com' # after 2022-02-01: 'prod.reporting.amazonmusiccatalog.com' # sftp sftp = { 'theorchard': { 'host': os.environ.get( 'AMAZON_DIGITAL_SERVICES_SFTP_HOST', 'prod.reporting.amazonmusiccatalog.com'), 'username': os.environ.get('AMAZON_DIGITAL_SERVICES_SFTP_USERNAME'), 'password': os.environ.get('AMAZON_DIGITAL_SERVICES_SFTP_PASSWORD'), 'port': int(os.environ.get('AMAZON_DIGITAL_SERVICES_SFTP_PORT', 22)), 'path': os.environ.get('AMAZON_DIGITAL_SERVICES_SFTP_PATH', '')} } ftp_path_template = 'sales/{country_code}/Daily' # List of emails to send notifications about missing files EMAIL_LIST = ['data-squad@theorchard.com', 'DataAnalyticsDept@theorchard.com'] EMAIL_SOURCE = 'datarequests@theorchard.com'