"""Dimension Refresh flow configuration.""" from importlib.resources import files import logging import os import re from garcon.param import StaticParam import yaml logger = logging.getLogger('dim_refresh_etl') YAML_ENVVAR_RGX = re.compile(r'^ENV\[(.*)\]$') def _parse_envvar(token): """Look if YAML scalar contains the "ENV[foo]". Args: token: a YAML scalar. """ return YAML_ENVVAR_RGX.match(token).groups()[0] def _yaml_envvar_constructor(loader, node): """Get YAML environment variables with constructor. Looks to see if YAML node contains ENV keyword and, if so, will replace it with the os environment var's value. Args: loader: a YAML Loader. node: a YAML Node. """ token = loader.construct_scalar(node) env_var = _parse_envvar(token) if env_var in os.environ: return os.environ[env_var] return None yaml.add_implicit_resolver('!env_var', YAML_ENVVAR_RGX) yaml.add_constructor('!env_var', _yaml_envvar_constructor) def getconf(config_name, ext='yml', package=None): """Convert a config file into a dictionary of Garcon StaticParms. Note will replace YAML scalars whose value is ENV[foo] with the value in stored by os environment var foo. Args: config_name (str): name of the YAML config file minus extension. ext (str): extension of the YAML config file. package (str): package where config file is located (defaults to __name__). Returns: Object: a Python object representing the YAML config. """ if not package: package = __name__ filename = '.'.join((config_name, ext)) logger.debug('loading config file %s', filename) resource = files(package).joinpath(filename) with resource.open('rb') as conf_stream: return yaml.load(conf_stream, Loader=yaml.FullLoader) def get_static_dict(config_name, sub_key=None): """Convert values in config file into a dictionary of Garcon StaticParms. Args: config_name (str): name of the YAML config file. sub_key (str): if passed only converts this config var in the file to a dictionary of Garcon StaticParms. Returns: dict: dictionary of StaticParam vars. """ if sub_key: config_dict = getconf(config_name)[sub_key] else: config_dict = getconf(config_name) static_param_dict(config_dict) return config_dict def static_param_dict(config_dict): """Convert dictionary values into Garcon StaticParams. Args: config_dict (dict): dictionary. Returns: dict: dictionary now has each value as a StaticParam. """ for key, value in config_dict.items(): config_dict[key] = StaticParam(value)