from datetime import datetime import logging import pandas as pd component_logger = logging.getLogger().getChild("tasks.base_attributes_gen") def get_fan_attribute_values(source_df: pd.DataFrame, attributes_df: pd.DataFrame, collection_id: int) -> pd.DataFrame: """Returns fan_attribute table """ """ We create a row_id """ source_df['row_id'] = source_df.index """ We transpose attributes """ fan_attribute_base = pd.DataFrame(source_df.set_index(['fan_id', 'row_id']).stack(), columns=['value']) fan_attribute_base.index.names = ['fan_id', 'row_id', 'name'] fan_attribute_base = fan_attribute_base.reset_index(level=[0, 1]) """ Left join and map attributes by name to get ids for attributes """ fan_attribute_result = pd.merge(fan_attribute_base, attributes_df, left_on='name', right_on='name', how='left') # Limit the columns we want in fan_attribute table fan_attribute_result = fan_attribute_result[['fan_id', 'attribute_id', 'value', 'row_id']] # Add the field that holds collection_id fan_attribute_result['collection_id'] = collection_id """ Adding timestamp field to successfully stage the file during upload to RDS""" fan_attribute_result['timestamp'] = str(datetime.utcnow()) return fan_attribute_result