import itertools from collections import OrderedDict from datetime import timedelta from enum import Enum from typing import Dict, Iterable, List, Tuple, Union import structlog from delphi_api.bigtable import BigTableModel from delphi_api.core.constants import UTF8 from delphi_api.errors import Codes, InvalidInputError from delphi_api.utils import DateUtils from delphi_api.v3.constants import ENABLE_GROUP_TYPE_PREFIX from delphi_api.v3.data_models.bigtable.row_key_constants import ( CHARTMETRIC_ROW_KEYS, CHART_ROW_KEYS, PLAYLIST_ROW_KEYS, STREAMS_ROW_KEYS, ) from delphi_api.v3.enums import RowKeyAgg, RowKeyPrefix, RowKeySegment, SubsetParam from delphi_api.v3.view_models.params import Params from delphi_api.v3.view_models.query_builder import QueryBuilder LOG = structlog.get_logger(__name__) class RowKeyGroupType(Enum): """ See Also: :mod:`delphi_api.v3.data_models.bigtable.row_key_constants` """ STREAMS = STREAMS_ROW_KEYS CHART_POSITIONS = CHART_ROW_KEYS PLAYLIST_POSITIONS = PLAYLIST_ROW_KEYS CHARTMETRIC = CHARTMETRIC_ROW_KEYS class RowKeyBuilder: """ Container for the logic surrounding which key prefix to use for BigTable row scans Consumers should typically use use :meth:`RowKeyBuilder.get_row_keys` """ @classmethod def get_row_keys(cls, params: Params, key_group_type: RowKeyGroupType, agg: RowKeyAgg = RowKeyAgg.DAY) -> List[Tuple[bytes, bytes]]: """ Primary method to get row keys for any given set of params Get a list of row key prefixes with/without start_date/end_date - Providing date=None will get the prefixes without the date segments (for ranges) Returns: A list of tuples in format (start_key, end_key) """ try: prefix = cls.get_logical_prefix(params, key_group_type) return cls._get_row_keys_range_sets(params, prefix, key_group_type, agg=agg) except KeyError as e: # re-route KeyErrors, let the others bubble normally raise NotImplementedError(e) @classmethod def get_logical_prefix(cls, params: Params, key_group_type: RowKeyGroupType) -> RowKeyPrefix: if key_group_type == RowKeyGroupType.STREAMS: prefix = cls._get_logical_prefix_streams(params) elif key_group_type == RowKeyGroupType.CHART_POSITIONS: prefix = cls._get_logical_prefix_chart_positions(params) elif key_group_type == RowKeyGroupType.PLAYLIST_POSITIONS: prefix = cls._get_logical_prefix_playlist_positions(params) elif key_group_type == RowKeyGroupType.CHARTMETRIC: prefix = cls._get_logical_prefix_chartmetric(params) else: raise NotImplementedError( 'get_logical_prefix not implemented for type: %s' % key_group_type) return prefix @classmethod def _get_row_keys_range_sets(cls, params: Params, prefix: RowKeyPrefix, key_group_type: RowKeyGroupType, agg: RowKeyAgg = RowKeyAgg.DAY) -> List[Tuple[bytes, bytes]]: """Checks we have config to build sets of range keys, then builds them. Raises: KeyError: Missing row key format structure for prefix Returns: List of row keys in as tuples of row key segments in bytes """ row_key_vars = key_group_type.value.get(prefix.value) if not row_key_vars: raise KeyError('Missing row key format structure for prefix: %s' % prefix.value) full_prefix = cls.get_full_prefix_string(prefix, agg, key_group_type) return cls._build_key_range_sets(params, full_prefix, row_key_vars) @classmethod def _build_key_range_sets(cls, params: Params, full_prefix: str, row_key_vars: List[str]) -> List[Tuple[bytes, bytes]]: """Builds a list of start_key, end_key permutations based on params and row key vars Args: params: full_prefix: complete prefix string before key segments. See :mod:`delphi_api.v3.data_models.bigtable.row_key_constants` row_key_vars: row key segments – ex from CHART_TRACK_POSITIONS_ROW_KEYS values Raises: AttributeError: Missing start_date and/or end_date for range key set AssertionError: If pre-building encountered an error Returns: List of row keys in as tuples of row key segments in bytes """ cls.validate_date_range(params.start_date, params.end_date) # build ordered dictionaries with lists of values of each param in prefix vars start_dict, end_dict = OrderedDict(), OrderedDict() for key_var in row_key_vars: if key_var == RowKeySegment.DATE.value: start_dict[key_var] = [params.start_date] # If our end key is a prefix, add one day to end date (avoids determining end key) end_date = DateUtils.as_str(DateUtils.from_str(params.end_date) + timedelta(days=1)) end_dict[key_var] = [end_date] continue param_val = getattr(params, key_var, None) if not param_val: LOG.debug('No param value provided for row key segment: %s' % key_var) continue # alter any params provided by client that require mutation for bigtable queries final_param_val = RowKeyBuilder.mutate_segment(key_var, param_val) # make all the params a list for simplified logic here start_dict[key_var] = QueryBuilder.param_as_list(final_param_val) end_dict[key_var] = QueryBuilder.param_as_list(final_param_val) # convert dictionaries to a list of dictionaries of parameter permutations start_dicts = RowKeyBuilder.flatmap_dict_of_lists(start_dict) end_dicts = RowKeyBuilder.flatmap_dict_of_lists(end_dict) if not len(start_dicts) == len(end_dicts): raise AssertionError('Pre-building param lists was interrupted.') key_sets = [] for idx in range(len(start_dicts)): if full_prefix != '': start_keys, end_keys = [full_prefix], [full_prefix] else: start_keys, end_keys = [], [] start_keys.extend(start_dicts[idx].values()) end_keys.extend(end_dicts[idx].values()) start_key = cls._get_bigtable_row_key(start_keys) end_key = cls._get_bigtable_row_key(end_keys) # ensure we don't pass invalid range to big table if start_key == end_key: raise AssertionError('Start key equals end key: %s' % start_key.decode(UTF8)) elif start_key < end_key: key_sets.append((start_key, end_key,)) else: LOG.warning('Potential processing issue, start_key > end key. ' 'Client may have requested invalid range. Attempting swap to proceed.') key_sets.append((end_key, start_key,)) return key_sets @classmethod def validate_date_range(cls, start_date, end_date): """Check the client provided valid dates before continuing""" err = None if not (start_date and end_date): err = 'Missing start_date and/or end_date for range key set' if start_date > end_date: err = 'start_date is greater than end_date' if err: raise InvalidInputError({ 'code': Codes.invalid_input.value, 'description': err, }) @staticmethod def mutate_segment(key_var: str, value: Union[Iterable[str], str]): """Row key segments are run through this filter-like method. Currently it reverses the ISRC. Args: key_var: segment name (ex: ``'isrc'``) value: segment value (ex: ``AAZZZ0123456``) Returns: a mutated ``value`` if mutation is required, else the unmodified ``value`` """ if key_var == RowKeySegment.ISRC.value: if isinstance(value, list): return [elem[::-1] for elem in value] return value[::-1] return value @staticmethod def flatmap_dict_of_lists(dl: Dict[str, List[str]]) -> List[Dict[str, str]]: """Creates a flat list of the permutation of our dictionaries values""" return [dict(zip(dl, v)) for v in itertools.product(*dl.values())] @classmethod def _get_bigtable_row_key(cls, keys: Iterable[str]) -> bytes: """Use existing code to join keys and encode""" return BigTableModel.get_row_key(keys) @classmethod def _get_group_type_prefix(cls, key_group_type: RowKeyGroupType) -> str: """If setting enabled, adds prefix row keys with group (ex: streams_*) Decision was to omit the additional row prefix, so this is now DEPRECATED """ if not ENABLE_GROUP_TYPE_PREFIX: return '' names = { RowKeyGroupType.CHART_POSITIONS: 'charts', RowKeyGroupType.STREAMS: 'streams', RowKeyGroupType.PLAYLIST_POSITIONS: 'playlists', } return names.get(key_group_type, '') @classmethod def get_full_prefix_string(cls, prefix: RowKeyPrefix, agg: RowKeyAgg, key_group_type: RowKeyGroupType) -> str: """Combines our RowKeyPrefix with a RowKeyAgg to get the full prefix""" prefix_str = f'{prefix.value}_{agg.value}' group_type_prefix = cls._get_group_type_prefix(key_group_type) return f'{group_type_prefix}_{prefix_str}' if group_type_prefix else prefix_str @classmethod def _get_logical_prefix_streams(cls, params: Params) -> RowKeyPrefix: """Assumes RowKeyGroupType.STREAMS See :meth:`delphi_api.v3.view_models.stream.StreamsViewModel.validate_params` for how params are filtered before here Notes: - if we have product_id, find by product prefix - if we have playlist_id, find by playlist prefix - if we have chart_id, find by chart prefix - if we have no artist_id and have track_id or isrc, find by track prefix - if we have artist_id and no track_id or isrc, find by artist prefix - if we have artist_id and track_id or isrc, find by artist prefix Raises: KeyError: Params object missing essential field(s) """ if params.product_id: return cls._get_product_prefix(params) elif params.playlist_id: return cls._get_playlist_prefix(params) elif params.chart_id: return cls._get_chart_prefix(params) elif params.track_id or params.isrc: return cls._get_track_prefix(params) elif params.artist_id: return cls._get_artist_prefix(params) else: raise KeyError('Missing minimum provided parameters to get prefix') @classmethod def _get_logical_prefix_playlist_positions(cls, params: Params) -> RowKeyPrefix: """Assumes RowKeyGroupType.PLAYLIST_TRACK_POSITIONS See :meth:`delphi_api.v3.view_models.track_position.TrackPositionsViewModel.validate_params` for how params are filtered before here Notes: - if we have playlist_id, find by playlist prefix Raises: KeyError: Params object missing essential field(s) """ if params.playlist_id and params.isrc: return RowKeyPrefix.playlist_isrc_date elif params.playlist_id: return RowKeyPrefix.playlist_date_isrc elif params.isrc and params.dsp: return RowKeyPrefix.dsp_isrc_date_playlist elif params.isrc: return RowKeyPrefix.isrc_date_playlist elif params.dsp: # This is not allowed in param validation return RowKeyPrefix.dsp_isrc_date_playlist else: raise KeyError('Missing minimum provided parameters to get prefix') @classmethod def _get_logical_prefix_chart_positions(cls, params: Params) -> RowKeyPrefix: """Assumes RowKeyGroupType.CHART_TRACK_POSITIONS See :meth:`delphi_api.v3.view_models.track_position.TrackPositionsViewModel.validate_params` for how params are filtered before here Notes: - if we have chart_id, find by :meth:`_get_chart_prefix`, else - if we have dsp_id and isrc, find by dsp and isrc, else - if we have isrc, find by isrc, else - if we have dsp, find all by dsp (not currently enabled in param validation) Raises: KeyError: Params object missing essential field(s) """ if params.chart_id and params.isrc: return RowKeyPrefix.chart_isrc_date elif params.chart_id: return RowKeyPrefix.chart_date_isrc elif params.isrc and params.dsp: return RowKeyPrefix.dsp_isrc_date_chart elif params.isrc: return RowKeyPrefix.isrc_date_chart elif params.dsp: return RowKeyPrefix.dsp_date_isrc_chart else: raise KeyError('Missing minimum provided parameters to get prefix') @classmethod def _get_logical_prefix_chartmetric(cls, params: Params) -> RowKeyPrefix: """Assumes RowKeyGroupType.CHARTMETRIC (we should have ``artist_id``) See :meth:`delphi_api.v3.view_models.track_position.TrackPositionsViewModel.validate_params` for how params are filtered before here Notes: - if we have artist_id, find by artist_id Raises: KeyError: Params object missing essential field(s) """ if params.artist_id: return RowKeyPrefix.artist else: raise KeyError('Missing minimum provided parameters to get prefix') @classmethod def _get_artist_prefix(cls, params: Params) -> RowKeyPrefix: """Assumes we have ``artist_id``""" if params.track_id: return RowKeyPrefix.artist_track_date elif params.isrc: return RowKeyPrefix.artist_isrc_date elif params.playlist_id: return RowKeyPrefix.artist_playlist_date else: return RowKeyPrefix.artist_date @classmethod def _get_track_prefix(cls, params: Params) -> RowKeyPrefix: """Assumes we have ``track_id`` or ``isrc``""" if params.isrc: if params.playlist_id: return RowKeyPrefix.playlist_isrc_date if params.subset == SubsetParam.PLAYLISTS.value: return RowKeyPrefix.isrc_date_playlist if params.chart_id: return RowKeyPrefix.chart_isrc_date return RowKeyPrefix.isrc_date if params.track_id: if params.playlist_id: return RowKeyPrefix.playlist_track_date if params.chart_id: return RowKeyPrefix.chart_track_date return RowKeyPrefix.track_date raise KeyError('Missing minimum provided parameters to get prefix') @classmethod def _get_product_prefix(cls, params: Params) -> RowKeyPrefix: """Assumes we have product_id""" return RowKeyPrefix.product_date @classmethod def _get_playlist_prefix(cls, params: Params) -> RowKeyPrefix: """Assumes we have ``playlist_id`` Raises: KeyError: missing track_id or isrc """ if params.track_id: return RowKeyPrefix.playlist_track_date elif params.isrc: return RowKeyPrefix.playlist_isrc_date elif params.artist_id: return RowKeyPrefix.artist_playlist_date else: return RowKeyPrefix.playlist_date @classmethod def _get_chart_prefix(cls, params: Params) -> RowKeyPrefix: """Assumes we have ``chart_id``""" if params.isrc: return RowKeyPrefix.chart_isrc_date else: return RowKeyPrefix.chart_date_isrc # ----------------------------------- # # --- Test related helper methods --- # @classmethod def get_row_keys_single(cls, params: Params, key_group_type: RowKeyGroupType, date: str, agg: RowKeyAgg = RowKeyAgg.DAY) -> List[bytes]: # pragma: no cover """TEST ONLY: Get a list of single row keys (currently only used for dynamic test seeds)""" prefix = cls.get_logical_prefix(params, key_group_type) return cls._get_row_keys_single_sets(params, prefix, key_group_type, date, agg=agg) @classmethod def _get_row_keys_single_sets(cls, params: Params, prefix: RowKeyPrefix, key_group_type: RowKeyGroupType, date: str, agg: RowKeyAgg = RowKeyAgg.DAY ) -> List[bytes]: # pragma: no cover """TEST ONLY: Get a list of single row keys (currently only used for dynamic test seeds)""" row_key_vars = key_group_type.value.get(prefix.value) if not row_key_vars: raise KeyError('Missing row key format structure for prefix: %s' % prefix.value) full_prefix = cls.get_full_prefix_string(prefix, agg, key_group_type) return cls._build_single_key_sets(params, full_prefix, row_key_vars, date) @classmethod def _build_single_key_sets(cls, params: Params, full_prefix: str, row_key_vars: List[str], date: str ) -> List[bytes]: # pragma: no cover """TEST ONLY: Builds a list of single row keys from params (currently only used for dynamic test seeds) Args: params: full_prefix: date: string of a single date row_key_vars: row key segments – ex from CHART_TRACK_POSITIONS_ROW_KEYS values Returns: A list of single keys as bytes """ # build ordered dictionaries with lists of values of each param in prefix vars key_dict = OrderedDict() for key_var in row_key_vars: if key_var == RowKeySegment.DATE.value: key_dict[key_var] = [date] continue param_val = getattr(params, key_var, None) if not param_val: LOG.warning('No param value provided for row key segment: %s' % key_var) continue # alter any params provided by client that require mutation for bigtable queries final_param_val = RowKeyBuilder.mutate_segment(key_var, param_val) # make all the params a list for simplified logic here key_dict[key_var] = QueryBuilder.param_as_list(final_param_val) # convert dictionaries to a list of dictionaries of parameter permutations key_dicts = RowKeyBuilder.flatmap_dict_of_lists(key_dict) key_sets = [] for idx in range(len(key_dicts)): keys = [full_prefix] keys.extend(key_dicts[idx].values()) key_sets.append(cls._get_bigtable_row_key(keys)) return key_sets # --- end: Test related helper methods --- # # ---------------------------------------- #