"""Different small util functions.""" import datetime from datetime import timedelta from garcon.contrib.dynamo_feed_status import \ feed_status_ingestion as feed_status def list_of_dates(start_date, end_date, reversed=False): """Generate list of dates between start_date and end_date. Args: start_date (str): Start date in 'YYYY-MM-DD' format. end_date (str): End date in 'YYYY-MM-DD' format. Yields: str: String date in 'YYYY-MM-DD' format. """ start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d') end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d') if reversed: for day_num in range(0, (end_date - start_date).days + 1): yield (end_date - timedelta(days=day_num)).strftime('%Y-%m-%d') else: for day_num in range(0, (end_date - start_date).days + 1): yield (start_date + timedelta(days=day_num)).strftime('%Y-%m-%d') def exit_message(message): """Create dict-message which will be processed by decider. Args: message (str): Error message. Returns: dict: Dict that will be added to the SWF context. """ return {'stop': True, 'message': message} def get_bool_from_flag(flag): """Validate if str or bool flag is positive. Args: flag(str|bool): flag to check Returns: bool: python value of the flag """ return ( isinstance(flag, str) and flag.lower() == 'true' or isinstance(flag, bool) and flag ) class UnprocessableFlowParamsException(Exception): """Exception of incorrect int list format.""" pass def format_id_list(value): """Format passed value to the list of integers. Args: value (str | list | int): Value for transformation Returns: list[int]: List of integers Raises: InvalidIntListException """ if value is not None: try: if isinstance(value, str): return list(map(int, value.split(','))) if isinstance(value, list): return list(map(int, value)) if isinstance(value, int): return [value] except (ValueError, TypeError): raise UnprocessableFlowParamsException raise UnprocessableFlowParamsException return [] def sos_labelid_filter(labelids, table_alias=None): """Generate filter clause and prepare values for it. Args: labelids (list): List of label ids. table_alias (str): Table alias in sql statement. Returns: tuple(str, tuple): str: Filter clause. tuple: label ids. """ labelid_clause_template = 'AND {}labelid in %(labelids)s' if labelids: labelid_clause = labelid_clause_template.format( table_alias + '.' if table_alias else '') return labelid_clause, tuple(labelids) return '', None def validate_date_range_reload(context_date_range, reload): """Validate combination of reload flag and date range value. Args: context_date_range (str): Passed to the flow context date range value. reload (bool): Passed to the flow context reload flag value. Returns: Bool: True if params are valid. Raises: UnprocessableFlowParamsException: If passed params can't be processed. """ if not context_date_range and reload: raise UnprocessableFlowParamsException( "'reload' flag passed but date range is not specified.") if context_date_range and not reload: raise UnprocessableFlowParamsException( "'explicit_date_range' specified but 'reload' flag omitted.") return True def find_flow_date_range( feed_name, source_feed_name, context_date_range, days_back): """Find processing date range for flow. If context_date_range is not None returns split start and end dates, otherwise look for unprocessed dates. Args: feed_name (str): Flow name. source_feed_name (str): Source data feed name. context_date_range (str): Passed to the Garcon date range value in `YYYY-MM-DD_YYYY-MM-DD` format. days_back (int): Number of last days for check. """ if context_date_range: return context_date_range.split('_') range_result = _find_range_to_process( feed_name, source_feed_name, days_back) if not range_result['found_days']: raise UnprocessableFlowParamsException( 'There is no data to aggregate for {} range'.format( range_result['week_range'])) return range_result['start_date'], range_result['end_date'] def _find_range_to_process(feed_name, source_feed_name, days_back): """Determine range of dates that need processing within past week. Args: feed_name (str): Name of the feed. Returns: dict: { 'found_days': Boolean, 'week_range': String date range for past week, 'start_date': First date that needs processing, 'end_date': Last date that needs processing } """ result = {'found_days': False} today = datetime.date.today().strftime('%Y-%m-%d') range_start_date = ( datetime.date.today() - timedelta(days=days_back)).strftime('%Y-%m-%d') result['week_range'] = range_start_date + '_' + today day_range = [] for day in list_of_dates(range_start_date, today): source_feed_ingested = is_feed_ingested(source_feed_name, day) feed_ingested = is_feed_ingested(feed_name, day) if source_feed_ingested and not feed_ingested: result['found_days'] = True day_range.append(day) if result['found_days']: result['start_date'] = day_range[0] result['end_date'] = day_range[-1] return result def is_feed_ingested(feed_name, date): """Check if apple music streams flow ingested for given date. Args: feed_name (str): Feed name. date (str): Date in sting format 'YYYY-MM-DD'. Returns: bool: True if data ingested for all given date. """ status = feed_status.get_overall_status(feed_name, date) return status == feed_status.STATUS_INGESTED def sos_labelid_clause(labelids, table_alias=None): """Generate filter clause. Args: labelids (list): List of label ids. table_alias (str): Table alias in sql statement. Returns: str: Filter clause. """ labelid_clause_template = 'AND {}labelid in (%(labelids)s)' if labelids: labelid_clause = labelid_clause_template.format( table_alias + '.' if table_alias else '') return labelid_clause return ''