from collections import OrderedDict from pathlib import Path from common import common_config from utils.conf import var FLOW_DIR = Path(__file__).parent ENVIRONMENT = common_config.ENVIRONMENT FEED_NAME = 'deezer_daily' FEED_ID = 9 # SELECT feedid from dim_feed where feedname = 'Deezer' ARCHIVE_S3_BUCKET = common_config.COMMON_ARCHIVE_S3_BUCKET LICENSORS = ('theorchard', 'sme') # file name as ingested from the service DROP_FILE_NAME = { 'theorchard': 'TheOrchard_{date}_{date}.zip', 'sme': 'sony_{date}_{date}.zip'} SPEC_VERSION = { 'theorchard': 2, 'sme': 3, } ZEPHIR_CREDENTIALS = { 'username': var('ZEPHIR_DEEZER_USERNAME', 'fake_deezer_user'), 'password': var('ZEPHIR_DEEZER_PASSWORD', 'FakePassword123'), 'host': var('ZEPHIR_DEEZER_HOST', 'https://zephir.deezer.com'), 'path': var('ZEPHIR_DEEZER_PATH', '/externaldownload?file=daily/') } # TODO: rename _bucket to _full_path S3 = { 'archive_s3_path': 's3://{s3_bucket}/DeezerV{spec_version}/archives/{datestamp}/{licensor}/', # noqa 'temp_staging_raw_s3_path': 's3://{s3_bucket}/DeezerV{spec_version}/temp/{datestamp}/{licensor}/', # noqa } SME_DROP_BUCKET = 'sme-ca-prod-partners' SME_DROP_PATH = 'deezer/in/PG45/' SNOWFLAKE_TABLE_NAMES = { 'staging_raw': { 'theorchard': 'staging_raw_deezer_v2', 'sme': 'staging_raw_deezer_v3' }, } # source files format specs FILE_SUFFIXES = { 'theorchard': ['.txt', '_TB.txt'], 'sme': ['_FR.txt', '_ROW.txt', '_TB.txt'] } # config for the update_dim_tables task DIMENSION_TABLES = { 'tables_to_update': [ 'dim_playlist', 'dim_user', ], 'include_to_report': [ 'dim_playlist', 'dim_user', ]} AWS_CONN_ID = common_config.COMMON_AWS_CONN_ID