"""Main application logic.""" import os import sys from ioda import config from ioda.connectors import youtube from ioda import const from ioda import csv_dump from ioda import db from ioda import queries from ioda.connectors.youtube import ServiceMapping OUTPUT_DIR = 'output' def generate_ioda_csv_files(): """Load the data from RedShift and generate CSV files.""" try: con = db.get_connection() if not con: sys.exit(1) print('Getting total number of assets...') count = db.get_total_records_count(con) print('Number of assets: {}'.format(count)) if count % config.QUERY_LIMIT == 0: amount_of_csvs = count // config.QUERY_LIMIT else: amount_of_csvs = count // config.QUERY_LIMIT + 1 if config.BATCH_NUMBER > amount_of_csvs: print('An error happend. BATCH_NUMBER out of range') return if not os.path.isdir(OUTPUT_DIR): os.mkdir(OUTPUT_DIR) generated_files = 0 start_batch = config.BATCH_NUMBER - 1 for i in range(start_batch, amount_of_csvs): csv_number = i + 1 print('Get batch {}'.format(csv_number)) data = db.get_batch(con, i*config.QUERY_LIMIT, config.QUERY_LIMIT) file_name = generate_file_name(csv_number, amount_of_csvs) print('Generating csv {}'.format(file_name)) csv_dump.save_csv(data=data, file_name=file_name) print('{} generated'.format(file_name)) generated_files += 1 return generated_files except Exception as ex: print('An error happened. {}'.format(ex)) finally: if con: con.close() def generate_file_name(number, total_amount): """Generate file_name by template. Template is XXXX_of_YYYY.csv Args: number: number of current batch total_amount: total number of batches Returns: str: file name """ return '{dir}/{number}_of_{total}.csv'.format( dir=OUTPUT_DIR, number=str(number).zfill(4), total=total_amount) def save_youtube_metadata(): """Load the data from RedShift, query YouTube and generate CSV files.""" try: con = db.get_connection() if not con: sys.exit(1) print('Getting total number of assets...') count = db.get_total_records_count(con) print('Number of assets: {}'.format(count)) # YouTube allows us to query up to 50 assets at once query_limit = 50 if count % query_limit == 0: amount_of_csvs = count // query_limit else: amount_of_csvs = count // query_limit + 1 if config.BATCH_NUMBER > amount_of_csvs: print('An error happend. BATCH_NUMBER out of range') return if not os.path.isdir(OUTPUT_DIR): os.mkdir(OUTPUT_DIR) google_client = ServiceMapping(config.GOOGLE_API_KEY_FILE).youtube_partner generated_files = 0 start_batch = config.BATCH_NUMBER - 1 for i in range(start_batch, amount_of_csvs): csv_number = i + 1 print('Get batch {}'.format(csv_number)) data = db.get_batch( con, i*query_limit, query_limit, queries.ASSET_ID_FIELD) asset_ids = ','.join([r[const.ASSET_ID].strip() for r in data]) print(asset_ids) data = youtube.get_assets(google_client, asset_ids) file_name = generate_file_name(csv_number, amount_of_csvs) print('Generating csv {}'.format(file_name)) csv_dump.save_csv( data=data, file_name=file_name, fields=csv_dump.YOUTUBE_CSV_FIELDS, row_mapper=csv_dump.create_csv_row_from_asset, write_header=False) print('{} generated'.format(file_name)) generated_files += 1 return generated_files except Exception as ex: print('An error happened. {}'.format(ex)) finally: if con: con.close()