"""Script configs.""" import os import uuid from datetime import date, datetime from constants import common as consts def get_url(field_name: str) -> str: """Get URL in correct format. Args: field_name: Environ field name. Returns: URL. """ url = os.environ[field_name] if not url.startswith("https://") and not url.startswith("http://"): url = f"https://{url}" return url.rstrip("/") ENVIRONMENT = os.environ.get("ENVIRONMENT", "prod").lower() class MySQL: HOST = os.environ["MYSQL_HOST"] PORT = int(os.environ.get("MYSQL_PORT", 3306)) USER = os.environ["MYSQL_USER"] PASSWORD = os.environ["MYSQL_PASSWORD"] DATABASE = os.environ["MYSQL_DATABASE"] POOL_RECYCLE = int(os.environ.get("MYSQL_POOL_RECYCLE", 7200)) POOL_SIZE = int(os.environ.get("MYSQL_POOL_SIZE", 1)) class DspAPI: HOST = get_url("DSPAPI_HOSTNAME") APP_KEY = os.environ["DSPAPI_APPKEY"] class Redis: HOST = os.environ.get("REDIS_HOST") PORT = int(os.environ.get("REDIS_PORT", 6379)) LOCK_ENABLED = bool(os.environ.get("REDIS_LOCK_ENABLED", True)) LOCK_TTL = int(os.environ.get("REDIS_LOCK_TTL", 86400)) assert not Redis.LOCK_ENABLED or Redis.HOST # S3 bucket name S3_BUCKET = os.environ["S3_BUCKET"] # AWS region AWS_DEFAULT_REGION = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") # unique run identifier CURRENT_RUN_ID = os.environ.get("CURRENT_RUN_ID", str(uuid.uuid4())[:8]) # possible date range to update (min of these dates and min of max streams/followers date) DATE_TO = ( datetime.strptime(os.environ["DATE_TO"], "%Y-%m-%d") if "DATE_TO" in os.environ else datetime.utcnow() ).date() DATE_FROM = ( datetime.strptime(os.environ["DATE_FROM"], "%Y-%m-%d").date() if "DATE_FROM" in os.environ else date(2020, 1, 1) ) assert DATE_TO >= DATE_FROM # max possible retry count RETRY_COUNT = int(os.environ.get("RETRY_COUNT", 10)) # retry wait multiplier RETRY_WAIT_RATE = int(os.environ.get("RETRY_WAIT_RATE", 3)) # athena DB name ATHENA_DATABASE = os.environ.get("ATHENA_DATABASE", "dev_apollo") # try create athena DB or not ATHENA_TRY_CREATE_DATABASE = bool(os.environ.get("ATHENA_TRY_CREATE_DATABASE", "")) # enable aggregate missing feature AGGREGATE_MISSING_PLAYLISTS = bool(os.environ.get("AGGREGATE_MISSING_PLAYLISTS", True)) # enable update aggregated data feature UPDATE_AGGREGATED_DATA = bool(os.environ.get("UPDATE_AGGREGATED_DATA", True)) # enable update playlists cut tables UPDATE_PLAYLIST_TABLE = bool(os.environ.get("UPDATE_PLAYLIST_TABLE", True)) # update aggregated playlists only via aggregating UPDATE_BY_AGGREGATING = bool(os.environ.get("UPDATE_BY_AGGREGATING", "")) # playlist metrics count filter PLAYLISTS_CATEGORIZED_MIN_METRIC_COUNT = { consts.Table.STREAMS: int(os.environ.get("PLAYLISTS_CATEGORIZED_MIN_STREAMS_COUNT", 100)), consts.Table.FOLLOWERS: int(os.environ.get("PLAYLISTS_CATEGORIZED_MIN_FOLLOWERS_COUNT", 100)), } PLAYLISTS_NON_CATEGORIZED_MIN_METRIC_COUNT = { consts.Table.STREAMS: int(os.environ.get("PLAYLISTS_NON_CATEGORIZED_MIN_STREAMS_COUNT", 10000)), consts.Table.FOLLOWERS: int(os.environ.get("PLAYLISTS_NON_CATEGORIZED_MIN_FOLLOWERS_COUNT", 10000)), } # playlist chunk size to aggregate PLAYLIST_AGGREGATE_METRIC_CHUNK_SIZE = { consts.Table.STREAMS: int(os.environ.get("PLAYLIST_AGGREGATE_STREAMS_CHUNK_SIZE", 2000)), consts.Table.FOLLOWERS: int(os.environ.get("PLAYLIST_AGGREGATE_FOLLOWERS_CHUNK_SIZE", 5000)), } # playlist chunk size to update PLAYLIST_UPDATE_METRIC_CHUNK_SIZE = { consts.Table.STREAMS: int(os.environ.get("PLAYLIST_UPDATE_STREAMS_CHUNK_SIZE", 100)), consts.Table.FOLLOWERS: int(os.environ.get("PLAYLIST_UPDATE_FOLLOWERS_CHUNK_SIZE", 3000)), } LOG_THROTTLE_UPDATE_CHUNK_COUNT = { consts.Table.STREAMS: int(os.environ.get("LOG_THROTTLE_STREAMS_UPDATE_CHUNK_COUNT", 30)), consts.Table.FOLLOWERS: int(os.environ.get("LOG_THROTTLE_FOLLOWERS_UPDATE_CHUNK_COUNT", 1)), } PLAYLIST_LIST_FOLLOWERS_DATE = os.environ.get("PLAYLIST_LIST_FOLLOWERS_DATE", "") if PLAYLIST_LIST_FOLLOWERS_DATE: PLAYLIST_LIST_FOLLOWERS_DATE = (datetime.strptime(PLAYLIST_LIST_FOLLOWERS_DATE, "%Y-%m-%d")).date() # we have two aggregated hot hits tables, specify which should be updated UPDATE_TABLE_LIST = os.environ.get("UPDATE_TABLE_LIST", "streams,followers").strip().split(",") assert UPDATE_TABLE_LIST and not (set(UPDATE_TABLE_LIST) - {consts.Table.STREAMS, consts.Table.FOLLOWERS}) SENTRY_DSN = os.environ.get("SENTRY_DSN")