"""Fetch contracts task method.""" import logging from lib import config from lib import constants from lib.utils import aws from lib.utils import ows from lib.utils.helpers import chunk_collection_by_size from tasks.accounting_run_calculate import helpers def snapshot_contracts(dag_run, *args, **kwargs): """Fetch contracts from ows-royalties.""" event = helpers.get_event_from_params(dag_run, **kwargs) accounting_period, accounting_run = helpers.get_event_records(event.target_id) snapshot_directory = helpers.build_snapshot_prefix_from_accounting_run( accounting_period, accounting_run, constants.DIRECTORY_MODULE_MATCH_CONTRACTS ) contract_ids = ows.get_contracts_by_accounting_run(event.target_id) contracts_count = len(contract_ids) logging.info(f'FOUND {contracts_count} contracts') contracts_detail = [] contract_terms_detail = [] for contract_ids_chunk in chunk_collection_by_size( contract_ids, config.SNAPSHOT_CONTRACTS_BATCH_SIZE ): contracts = ows.get_contract_details_by_contract_ids( contract_ids_chunk ).decode().splitlines() if contracts: if not contracts_detail: contracts_detail = contracts else: contracts_detail += contracts[1:] # skip header contract_terms = ows.get_contract_term_details_by_contract_ids( contract_ids_chunk ).decode().splitlines() if contract_terms: if not contract_terms_detail: contract_terms_detail = contract_terms else: contract_terms_detail += contract_terms[1:] # skip header contract_detail_count = len(contracts_detail) - 1 logging.info(f'Writing to s3 {contract_detail_count} contracts details') _write_contracts_to_s3( '\n'.join(contracts_detail).encode('utf-8'), snapshot_directory ) contract_terms_count = len(contract_terms_detail) - 1 if contract_terms_count > 0: logging.info(f'Writing to s3 {contract_terms_count} contract_terms') _write_contract_terms_to_s3( '\n'.join(contract_terms_detail).encode('utf-8'), snapshot_directory ) def _write_contracts_to_s3(contracts, snapshot_directory): """Write contract data to s3.""" contract_url = aws.location( snapshot_directory, constants.FILE_NAME_CONTRACTS_CSV ) aws.write_file(contract_url.url, contracts) def _write_contract_terms_to_s3(contract_terms, snapshot_directory): """Write contract term data to s3.""" contract_term_url = aws.location( snapshot_directory, constants.FILE_NAME_CONTRACT_TERMS_CSV ) aws.write_file(contract_term_url.url, contract_terms)