"""Application configuration.""" import logging import os from os.path import abspath, dirname, join, pardir from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from dotenv import load_dotenv from owsrequest.constants import headers as owsrequest_headers from secrets_manager.flask_ext import FlaskSecretsManager # Load environment variables from a .env file if present dotenv_path = abspath(join(dirname(__file__), pardir, ".env")) load_dotenv(dotenv_path) # Service information SERVICE_NAME = "ows-analytics-sr" SERVICE_VERSION = "1.0.0" # Swagger documentation path SWAGGER_FILE_PATH = "spec/ows-analytics-sr-1.0.0.yaml" # Environment DEV_ENVIRONMENT = "dev" QA_ENVIRONMENT = "qa" PROD_ENVIRONMENT = "prod" TEST_ENVIRONMENT = "test" ENVIRONMENT = os.environ.get("Environment", DEV_ENVIRONMENT) ONLY_LOG_ACCESS_ERRORS = True secrets_manager_client = FlaskSecretsManager( application_context=False, environment=ENVIRONMENT, service_name=SERVICE_NAME ) # Mock grass headers for local dev DEV_AUTH = { owsrequest_headers.GRASS_ACCOUNT_TYPE: os.environ.get("GRASS_ACCOUNT_TYPE") or None, owsrequest_headers.GRASS_ACCOUNT_ID: ( os.environ.get("GRASS_ACCOUNT_ID") and str(os.environ.get("GRASS_ACCOUNT_ID")) ) or None, owsrequest_headers.ORCHARD_USER_ID: ( os.environ.get("ORCHARD_USER_ID") and str(os.environ.get("ORCHARD_USER_ID")) ) or None, } # Mock grass headers for local dev DEV_PROFILE_AUTH = { owsrequest_headers.ORCHARD_PROFILE_TYPE: os.environ.get("LOCAL_PROFILE_TYPE") or None, owsrequest_headers.ORCHARD_PROFILE_ID: ( os.environ.get("LOCAL_PROFILE_ID") and str(os.environ.get("LOCAL_PROFILE_ID")) ) or None, } # Snowflake SNOWFLAKE_ACCOUNT = os.environ.get("SNOWFLAKE_ACCOUNT") SNOWFLAKE_USER = os.environ.get("SNOWFLAKE_USER") SNOWFLAKE_ROLE = os.environ.get("SNOWFLAKE_ROLE") SNOWFLAKE_WAREHOUSE = os.environ.get("SNOWFLAKE_WAREHOUSE") SNOWFLAKE_SCHEMA = os.environ.get("SNOWFLAKE_SCHEMA", "DEV") SNOWFLAKE_DATABASE = os.environ.get("SNOWFLAKE_DATABASE") SNOWFLAKE_PASSWORD = secrets_manager_client.get_cred("SNOWFLAKE_PASSWORD") SNOWFLAKE_POOL_SIZE = 15 # UWSGI number of threads SNOWFLAKE_POOL_RECYCLE = 4 * 55 * 60 # Snowflake timeout: ~4 hours SNOWFLAKE_POOL_MAX_OVERFLOW = 5 SNOWFLAKE_POOL_PRE_PING = False SNOWFLAKE_POOL_RESET_ON_RETURN = None # Default for local dev SNOWFLAKE_PRIVATE_KEY_PATH = os.environ["HOME"] + "/.ssh/snowflake/rsa_key.p8" try: SNOWFLAKE_PRIVATE_KEY_PATH = ( secrets_manager_client.get_cred("SNOWFLAKE_PRIVATE_KEY_PATH") or SNOWFLAKE_PRIVATE_KEY_PATH ) SNOWFLAKE_KEY_PASSPHRASE = ( secrets_manager_client.get_cred("SNOWFLAKE_KEY_PASSPHRASE") or None ) except Exception: SNOWFLAKE_KEY_PASSPHRASE = None if SNOWFLAKE_KEY_PASSPHRASE: with open(SNOWFLAKE_PRIVATE_KEY_PATH, "rb") as key: p_key = serialization.load_pem_private_key( key.read(), password=SNOWFLAKE_KEY_PASSPHRASE.encode(), backend=default_backend(), ) pkb = p_key.private_bytes( encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption(), ) SNOWFLAKE_CONNECT_ARGS = {"private_key": pkb} else: SNOWFLAKE_CONNECT_ARGS = {} # Redis REDIS_HOST = os.environ.get("REDIS_HOST") REDIS_PORT = 6379 # Sentry try: # TODO add secret SENTRY = secrets_manager_client.get_cred("SENTRY_DSN") except Exception: SENTRY = None # Loggers LOGGER_DSN = os.environ.get("LOGGER_DSN") or None LOGGER_LEVEL = logging.WARNING if ENVIRONMENT == PROD_ENVIRONMENT else logging.INFO LOGGER_NAME = "ows1" # DynamoDB ANALYTICS_METADATA_TABLE = os.environ.get("ANALYTICS_METADATA_TABLE") # Base URL if ENVIRONMENT == QA_ENVIRONMENT: BASE_URL = "https://qa-ows-analytics-sr.theorchard.io" else: BASE_URL = "http://localhost:5000" # Endpoint URLs def _concat(a, b): return f"{a}{b}" def _base(path): return _concat(BASE_URL, path) HEALTH_CHECK_PATH = "/hello/" # we need a trailing slash due to load balancer HEALTH_CHECK_URL = _base(HEALTH_CHECK_PATH) TOP_SOUND_RECORDINGS_PATH = "/top-sound-recordings" TOP_SOUND_RECORDINGS_URL = _base(TOP_SOUND_RECORDINGS_PATH) SOUND_RECORDING_PATH = "/sound-recording/" SOUND_RECORDING_URL = _base(SOUND_RECORDING_PATH) def _sound_recording(path): return _concat(SOUND_RECORDING_PATH, path) SOUND_RECORDING_METADATA_PATH = _sound_recording("/metadata") SOUND_RECORDING_METADATA_URL = _base(SOUND_RECORDING_METADATA_PATH) PLACEMENTS_PATH = _sound_recording("/placements") PLACEMENTS_URL = _base(PLACEMENTS_PATH) PLACEMENT_PATH = _sound_recording("/placement/") PLACEMENT_URL = _base(PLACEMENT_PATH) SOURCE_OF_STREAMS_PATH = _sound_recording("/source-of-streams") SOURCE_OF_STREAMS_URL = _base(SOURCE_OF_STREAMS_PATH) STREAMS_BREAKDOWN_PATH = _sound_recording("/streams-breakdown") STREAMS_BREAKDOWN_URL = _base(STREAMS_BREAKDOWN_PATH) PERFORMANCE_BY_COUNTRY_PATH = _sound_recording("/performance-by-country") PERFORMANCE_BY_COUNTRY_URL = _base(PERFORMANCE_BY_COUNTRY_PATH) STREAMS_BY_PRODUCT_PATH = _sound_recording("/streams-by-product") STREAMS_BY_PRODUCT_URL = _base(STREAMS_BY_PRODUCT_PATH) STREAMS_BY_SOS_PATH = _sound_recording("/streams-by-sos") STREAMS_BY_SOS_URL = _base(STREAMS_BY_SOS_PATH) STREAMS_BY_STORE_PATH = _sound_recording("/streams-by-store") STREAMS_BY_STORE_URL = _base(STREAMS_BY_STORE_PATH) STREAMS_ALL_PATH = _sound_recording("/streams-all") STREAMS_ALL_URL = _base(STREAMS_ALL_PATH) STREAMS_BULK_PATH = "/sound-recording/streams" STREAMS_PATH = _sound_recording("/streams") STREAMS_URL = _base(STREAMS_PATH) AGGREGATE_STREAMS_PATH = "/sound-recording/aggregate-streams" DOWNLOADS_PATH = _sound_recording("/downloads") DOWNLOADS_URL = _base(DOWNLOADS_PATH) DOWNLOADS_ALL_PATH = _sound_recording("/downloads-all") DOWNLOADS_ALL_URL = _base(DOWNLOADS_ALL_PATH) DOWNLOADS_BY_STORE_PATH = _sound_recording("/downloads-by-store") DOWNLOADS_BY_STORE_URL = _base(DOWNLOADS_BY_STORE_PATH) DOWNLOADS_BY_COUNTRY_PATH = _sound_recording("/downloads-by-country") DOWNLOADS_BY_COUNTRY_URL = _base(DOWNLOADS_BY_COUNTRY_PATH) DOWNLOADS_BY_PRODUCT_PATH = _sound_recording("/downloads-by-product") DOWNLOADS_BY_PRODUCT_URL = _base(DOWNLOADS_BY_PRODUCT_PATH) TOP_MARKETS_PATH = _sound_recording("/top-markets") TOP_MARKETS_URL = _base(TOP_MARKETS_PATH) PLAYLISTS_PATH = _sound_recording("/playlists") PLAYLISTS_URL = _base(PLAYLISTS_PATH) TOP_COUNTRIES_STREAMS_PATH = _sound_recording("/top-countries-streams") TOP_COUNTRIES_STREAMS_URL = _base(TOP_COUNTRIES_STREAMS_PATH) TOP_COUNTRIES_DOWNLOADS_PATH = _sound_recording("/top-countries-downloads") TOP_COUNTRIES_DOWNLOADS_URL = _base(TOP_COUNTRIES_DOWNLOADS_PATH) DEMOGRAPHICS_PATH = _sound_recording("/demographics") DEMOGRAPHICS_URL = _base(DEMOGRAPHICS_PATH) FEEDS = { 1: { "storename": "Spotify", "storeid": 286, "distributors": ["theorchard", "sme"], "name": "Spotify Streams", }, 3: { "storename": "Pandora", "storeid": 708, "distributors": ["theorchard", "sme"], "name": "Pandora", }, 4: { "storename": "Apple Music", "storeid": 1, "distributors": ["theorchard", "sme"], "name": "Apple Music", }, 5: { "storename": "Apple Music", "storeid": 1, "distributors": ["theorchard", "sme"], "name": "Apple Music Non-Royalty Bearing Streams", }, 6: { "storename": "YouTube", "storeid": 453, "distributors": ["theorchard"], "name": "YouTube Weekly", }, 7: { "storename": "Spotify", "storeid": 286, "distributors": ["theorchard", "sme"], "name": "Spotify Aggregated Streams", }, 9: { "storename": "Deezer", "storeid": 348, "distributors": ["theorchard"], "name": "Deezer", }, 10: { "storename": "Google Play", "storeid": 496, "distributors": ["theorchard"], "name": "Google Play Music", }, 11: { "storename": "Napster", "storeid": 4, "distributors": ["theorchard", "sme"], "name": "Napster (Rhapsody)", }, 12: { "storename": "YouTube Movies", "storeid": 463, "distributors": ["theorchard"], "name": "YouTube VOD Weekly", }, 13: { "storename": "Apple Music", "storeid": 1, "distributors": ["theorchard", "sme"], "name": "iTunes", }, 14: { "storename": "YouTube", "storeid": 453, "distributors": ["theorchard"], "name": "YouTube Claim", }, 15: { "storename": "TDC Play", "storeid": 11, "distributors": ["theorchard"], "name": "24/7 (Juke)", }, # 16 storename Amazon Music Feature flagged 16: { "storename": "Amazon Music", "storeid": 187, "distributors": ["theorchard", "sme"], "name": "Amazon Downloads", }, 17: { "storename": "Amazon Prime Video", "storeid": 376, "distributors": ["theorchard"], "name": "Amazon (Prime) Instant Video", }, 18: { "storename": "Google Play", "storeid": 496, "distributors": ["theorchard"], "name": "Google Play Movies & TV", }, 19: { "storename": "VUDU", "storeid": 489, "distributors": ["theorchard"], "name": "VUDU", }, 20: { "storename": "Microsoft Xbox", "storeid": 487, "distributors": ["theorchard"], "name": "Xbox", }, 21: { "storename": "FandangoNOW", "storeid": 624, "distributors": ["theorchard"], "name": "FandangoNOW", }, 22: { "storename": "PlayStation Video", "storeid": 478, "distributors": ["theorchard"], "name": "PlayStation Video", }, 23: { "storename": "", "storeid": None, "distributors": ["theorchard"], "name": "Shazam", }, 24: { "storename": "", "storeid": None, "distributors": ["theorchard"], "name": "GFK", }, 25: { "storename": "eMusic", "storeid": 7, "distributors": ["theorchard"], "name": "eMusic", }, 26: { "storename": "MixRadio", "storeid": 27, "distributors": ["theorchard"], "name": "MixRadio", }, 27: { "storename": "Verizon", "storeid": 36, "distributors": ["theorchard"], "name": "Verizon", }, 28: { "storename": "MusicLoad", "storeid": 153, "distributors": ["theorchard"], "name": "MusicLoad", }, 30: { "storename": "Myxer", "storeid": 355, "distributors": ["theorchard"], "name": "Myxer", }, 31: { "storename": "Hulu", "storeid": 392, "distributors": ["theorchard"], "name": "Hulu", }, 32: { "storename": "Rdio", "storeid": 424, "distributors": ["theorchard"], "name": "Rdio", }, 33: { "storename": "Simfy GmbH", "storeid": 440, "distributors": ["theorchard"], "name": "Simfy", }, 34: { "storename": "Cricket Communications, Inc.", "storeid": 497, "distributors": ["theorchard"], "name": "Cricket Communications, Inc. (Muve)", }, # 35 - 37 Amazon Music Feature flagged 35: { "storename": "Amazon Music", "storeid": 187, "distributors": ["theorchard", "sme"], "name": "Amazon Prime", }, 36: { "storename": "Amazon Music", "storeid": 187, "distributors": ["theorchard", "sme"], "name": "Amazon AdSupported", }, 37: { "storename": "Amazon Music", "storeid": 187, "distributors": ["theorchard", "sme"], "name": "Amazon Unlimited", }, 38: { "storename": "YouTube", "storeid": 453, "distributors": ["theorchard", "sme"], "name": "YouTube Bulk Reports", }, 39: { "storename": "VKontakte", "storeid": 1405, "distributors": ["theorchard"], "name": "VKontakte", }, 40: { "storename": "Odnoklassniki", "storeid": 1406, "distributors": ["theorchard"], "name": "Odnoklassniki", }, 42: { "storename": "LINE Japan", "storeid": 1503, "distributors": ["theorchard"], "name": "LINE Japan", }, 43: { "storename": "SoundCloud", "storeid": 1505, "distributors": ["theorchard"], "name": "SoundCloud", } } FEED_IDS = list(FEEDS.keys()) SLOW_DYNAMO_TIME = float(os.environ.get("SLOW_DYNAMO_TIME", 0.5))