"""Util functions for Projections ETL.""" from datetime import datetime, timedelta import re from flows import art_relations from flows import datastore from flows.projections import queries def get_upc_projection_anchor_date(upc, projection_type): """Get the anchor date of a UPC from the datastore records. Args: upc (str): UPC to lookup. projection_type (str): Type of projection file we are processing. Returns: date: UPC's anchor date. """ if projection_type == 'original': results = art_relations.query( queries.ORIGINAL_PROJECTIONS_ANCHOR_DATE, {'upc': upc}) else: results = datastore.query( queries.PROJECTIONS_ANCHOR_DATE, {'upc': upc}) row = results.fetchone() return row[0] def transform(rows, anchor_date, upc, transaction_types): """Transform projections csv data to db table format. Args: rows (list(list)): list of rows from csv. anchor_date (str): anchor date in ISO format: YYYY-MM-DD. upc (str): release UPC. transaction_types (dict): transaction type mapping. Yields: tuple: row in db table format (upc, date, transaction_type, amount). """ start_sate = datetime.strptime(anchor_date, '%Y-%m-%d').date() dates = [ (start_sate + timedelta(days=i*7)).isoformat() for i in range(len(rows[0])-1)] for row in rows: if not transaction_types.get(row[0].upper()): continue for cell, date in zip(row[1:], dates): amount = None if cell: amount = cell.replace('$', '').replace(',', '') amount = round(float(amount), -1) yield upc, date, transaction_types.get(row[0].upper()), amount def parse_file_name(s3_path): """Parse projection CSV file. Args: s3_path (str): S3 key path. Returns: (str, str): UPC value, timestamp from file name """ match = re.match( '(?P[0-9]+)-(?P[0-9]{14}).csv', s3_path.split('/')[-1]) return match.group('upc'), match.group('date')