import datetime import logging import dateparser import numpy as np from dateparser_data.settings import default_parsers from dateutil.relativedelta import relativedelta from utils.base_attributes_generation import get_fan_attribute_values from .enrichment import Enrichment component_logger = logging.getLogger().getChild("enrichment.enrich_age") # Optimize and remove some un-necessary dateparsers. See https://dateparser.readthedocs.io/en/v1.0.0/settings.html date_parsers = [x for x in default_parsers if x not in ["relative-time", "no-spaces-time", "custom-formats"]] class EnrichAge(Enrichment): """ Gets age from date of birth. If not date of birth but age is preset in source data, returns age. """ source_attribute_ids = [ 9, # 'userDob' 10, # 'userAge' ] result_attribute_ids = [97] # 'enrAge' data_gathering_rule = "strict" def __init__(self, collection_id: int, schema: str, user_id: str, default_value: str = "unknown", **kwargs): super().__init__(collection_id, schema, user_id, default_value) def _run(self): """ Main enrichment method """ source_df = self.available_data["source"] # Drop columns with all nans source_df = source_df.dropna(axis=1, how="all") if "9" in source_df.columns: if hasattr(source_df, "parallel_apply"): # Parallel apply to speed up this enrichment source_df["enrAge"] = source_df.parallel_apply(self._get_age_from_dob, axis=1, args=["9"]) else: source_df["enrAge"] = source_df.apply(self._get_age_from_dob, axis=1, args=["9"]) elif "10" in source_df.columns: source_df["enrAge"] = source_df["10"] """Moving row_id field to index as we require this data to be preserved for connecting with source""" source_df.set_index("row_id", inplace=True) source_df = source_df[["fan_id", "enrAge"]] self.result_df = source_df @staticmethod def _get_age_from_dob(row, col): """ Get date object out of string. We use dateparser to conveniently make date object out of almost any string representation of date - this is pretty fool-proof and can work with stuff like "20 November 2020" as well as "22/11/2016" """ date_string = row[col] if date_string is None: # If we don't return None, we get "0" for missing dates (and in analytics falsely put to 0-17 age group) return date_string global date_parsers try: # Adding "en" language makes parsing of 10k dates ~60% faster dob_date_object = dateparser.parse( str(date_string), languages=["en"], settings={"PREFER_DATES_FROM": "past", "PARSERS": date_parsers} ) now_date_object = datetime.datetime.now() age_in_years_now = relativedelta(now_date_object, dob_date_object).years except Exception as e: age_in_years_now = np.nan return age_in_years_now """ In age enrich we need to join with fan_id and row_id because the output is row based """ def _generate_fan_attributes(self): """ Getting a list of attributes that are present in this enrichment""" condition = self.system_fields["a_id"].isin(self.result_attribute_ids) attributes_df = self.system_fields[condition][["a_system_name", "a_id"]] attributes_df.rename(columns={"a_system_name": "name", "a_id": "attribute_id"}, inplace=True) fan_attributes_df = get_fan_attribute_values( source_df=self.result_df, attributes_df=attributes_df, collection_id=self.enrichment_collection_id ) self.fan_attribute = fan_attributes_df