""" Monthly statement generation script. ==================================== Generate AVRO files for all labels / subaccounts monthly. Each run will generate AVRO files for the specified month or the quarter of the specified month for all labels and subaccounts. Example call: accounting_statement_export -p 207 -i month accounting_statement_export -p 206 -i quarter """ import argparse import importlib import json import time import boto.swf.layer2 as swf from processing_accounting import flows from processing_accounting.flows.accounting_statement_export import setting from processing_accounting.flows.accounting_statement_export.sql_generator \ import SqlGenerator from processing_accounting.util import db as db_util from processing_accounting.util import logging from processing_accounting.util import swf as swf_util def _execute_flow( flow, period_ids, user_type, payment_interval, options=[]): params = dict( period_ids=str(period_ids), user_type=user_type, payment_interval=payment_interval ) start_kwargs = dict( workflow_id=flow.workflow_id(params), input=json.dumps(params, sort_keys=True), execution_start_to_close_timeout=str(setting.START_TO_CLOSE_TIMEOUT)) return swf.WorkflowType( name=flow.name, domain=flow.domain, version=flow.version, task_list=flow.name).start(**start_kwargs) def _get_quarter_period_ids(period_id): """Get period ids for the year quarter in which the passed in period_id resides Args: period_id (int): accounting period id Returns: str: comma delimited period ids """ db_result = db_util.snowflake_query( SqlGenerator.get_quarter_year_period_ids(period_id)) return [periods.get('PERIOD_IDS') for periods in db_result].pop() def _wait_for_dynamodb(table): current_status = table.describe().get('Table').get('TableStatus') while current_status != 'ACTIVE': current_status = table.describe().get('Table').get('TableStatus') time.sleep(2) # wait 10 seconds before checking again logging.logger.info('Waiting dynamodb....') def _update_dynamodb_throughput(table, write_cap, read_cap=None): table_throughput = table.describe().get('Table').get( 'ProvisionedThroughput') reads = table_throughput.get('ReadCapacityUnits') writes = table_throughput.get('WriteCapacityUnits') if ((write_cap and write_cap != writes) or ( read_cap and read_cap != reads)): table.update( throughput=dict(write=write_cap, read=read_cap)) def run(*args): """Execute workflows for accounting statement export @todo(pkuong) research on a better way to trigger workflows and updating status for all workflows on dynamodb Args: args (list): arguments passed into this cli script. """ parser = argparse.ArgumentParser(description='Garcon command line util') parser.add_argument( '-p', '--period_id', required=True, type=int, help=( 'Desired accounting period id' 'Example: accounting_statement_export -p \'204\'')) parser.add_argument( '-i', '--payment_interval', required=False, help=( 'Optionally specify payment interval. Either: "month" or "quarter"' 'Example: accounting_statement_export -p \'204\' -i month'), choices=['month', 'quarter']) args = parser.parse_args(args) if args else parser.parse_args() args = args or '{}' payment_interval = args.payment_interval if hasattr(args, 'period_id'): flow = importlib.import_module( '.accounting_statement_export.flow', flows.__name__) flow_class = getattr(flow, 'Flow', None) quarter_periods = _get_quarter_period_ids(args.period_id) try: workflows = [ {'payment_interval': 'quarter', 'account_type': 'label'}, {'payment_interval': 'quarter', 'account_type': 'subaccount'}, {'payment_interval': 'month', 'account_type': 'label'}, {'payment_interval': 'month', 'account_type': 'subaccount'} ] # Begin workflows. workflow_executions = [] for workflow in workflows: payment_interval_match = payment_interval == workflow.get( 'payment_interval') if payment_interval is None or payment_interval_match: periods = quarter_periods if workflow.get('payment_interval') == 'month': periods = str(args.period_id) workflow_executions.append(_execute_flow( flow_class(), periods, workflow.get('account_type'), workflow.get('payment_interval'))) # Wait for all workflows to be done first before lowering the # write capacity. swf_util.wait_for_workflow_to_complete( workflow_executions, setting.START_TO_CLOSE_TIMEOUT) except Exception as err: # Turn down write capacity. logging.logger.info(err) raise Exception(err)