import logging import os from os.path import abspath from os.path import join from os.path import dirname from os.path import pardir from dotenv import load_dotenv import pyraml.parser from secrets_manager.flask_ext import FlaskSecretsManager from sqlalchemy.dialects.mysql import LONGBLOB from sqlalchemy.pool import NullPool, StaticPool from sqlalchemy.sql.sqltypes import LargeBinary path = os.path.dirname(os.path.dirname(__file__)) # Load environment variables from a .env file dotenv_path = abspath(join(dirname(__file__), pardir, '.env')) load_dotenv(dotenv_path) # Service information SERVICE_NAME = 'ows-masters-registry' SERVICE_VERSION = '0.0.1' # Environments PROD_ENVIRONMENT = 'prod' DEV_ENVIRONMENT = 'dev' QA_ENVIRONMENT = 'qa' TEST_ENVIRONMENT = 'test' ENVIRONMENT = os.environ.get('Environment', DEV_ENVIRONMENT) secrets_manager_client = FlaskSecretsManager(application_context=False, environment=ENVIRONMENT, service_name=SERVICE_NAME) # Errors and loggers SENTRY_DSN = secrets_manager_client.get_cred('SENTRY_DSN') or None LOGGER_LEVEL = logging.INFO LOGGER_NAME = 'ows1' logging.getLogger('ddtrace').setLevel(logging.WARNING) if ENVIRONMENT in [QA_ENVIRONMENT, PROD_ENVIRONMENT]: if not SENTRY_DSN: raise Exception(f'Sentry is not configured in {ENVIRONMENT}') LOGGER_DSN = os.environ.get('LOGGER_DSN') CELERY_LOGGER_LEVEL = int(os.environ.get('CELERY_LOGGER_LEVEL', logging.WARN)) # Database config if ENVIRONMENT == TEST_ENVIRONMENT: BLOB_TYPE = LargeBinary AR_DB_URL = 'sqlite://' BULK_STATUSES_DB_URL = 'sqlite://' POOLCLASS = StaticPool ISNULL_FUNC = 'is_null' else: BLOB_TYPE = LONGBLOB AR_DB_URL = secrets_manager_client.get_cred('AR_DB_URL') BULK_STATUSES_DB_URL = secrets_manager_client.get_cred( 'BULK_STATUSES_DB_URL') if not AR_DB_URL and not BULK_STATUSES_DB_URL: AR_DB_URL = os.environ.get('AR_DB_URL', None) BULK_STATUSES_DB_URL = os.environ.get('BULK_STATUSES_DB_URL', None) POOLCLASS = NullPool ISNULL_FUNC = 'isnull' assert AR_DB_URL, 'AR_DB_URL is missing from the environment' assert BULK_STATUSES_DB_URL, 'BULK_STATUSES_DB_URL is missing from the ' \ 'environment ' RAML_FILE = os.path.join(path, 'doc', 'api', 'ows-masters-registry-{}.raml'.format( SERVICE_VERSION)) API_DEFINITION = pyraml.parser.load(RAML_FILE) DYNAMODB_MASTERS_ACTIVE = '{}-masters_active'.format(ENVIRONMENT) DYNAMODB_MASTERS_AUDIT = '{}-masters_audit'.format(ENVIRONMENT) DYNAMODB_MASTERS_AUDIT_NEW = '{}-masters_audit_new'.format(ENVIRONMENT) DYNAMODB_MAX_KEYS = 100 # default AWS region to us-east-1 if not os.environ.get('AWS_DEFAULT_REGION'): os.environ['AWS_DEFAULT_REGION'] = 'us-east-1' # SQS QUEUE_NAME_YT_OWNERSHIP = '{}-yt-ownership-1'.format(ENVIRONMENT) # Generic endpoints HEALTH_CHECK = '/hello/' # Amazon S3 configuration REPORTS_BUCKET_NAME = '{}-orcdbucket'.format(ENVIRONMENT) REPORTS_FILE_KEY_PREFIX = 'masters-registry-reports/bulk-action' # App settings MAX_UPCS_COUNT = 10000 MAX_ISRCS_COUNT = 2000 MAX_REASON_LENGTH = 256 DEFAULT_BULK_TASKS_NUM_RECORDS = 30 TASK_STATUS_DEFAULT_ORDER_BY = 'create_datetime' TASK_STATUS_DEFAULT_ORDER_DIRECTION = 'desc' # Celery CELERY_ALWAYS_EAGER = False CELERY_EAGER_PROPAGATES_EXCEPTIONS = False CELERY_WORKER_PREFETCH_MULTIPLIER = int( os.environ.get('PREFETCH_MULTIPLIER', 1)) CELERY_TASK_ACKS_LATE = bool(os.environ.get('TASK_ACKS_LATE', 1)) CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND') BROKER_URL = os.environ.get('BROKER_URL') CELERY_IMPORTS = ['masters_registry.tasks.bulk'] BROKER_TRANSPORT_OPTIONS = { 'queue_name_prefix': '{}-ows-masters-'.format(ENVIRONMENT), 'visibility_timeout': 43200} CELERY_SERIALIZER = 'pickle' if ENVIRONMENT != TEST_ENVIRONMENT: assert CELERY_RESULT_BACKEND, 'CELERY_RESULT_BACKEND env var is missing' assert BROKER_URL, 'BROKER_URL env var is missing' if ENVIRONMENT == TEST_ENVIRONMENT: CELERY_ALWAYS_EAGER = True CELERY_EAGER_PROPAGATES_EXCEPTIONS = True