"""Generates Elasticsearch JSON output file from gzipped CSV file.""" import csv from datetime import datetime import gzip import json from yt_conflict_elasticsearch.flows.elasticsearch_export.models \ import ows_territories INT_FIELDS = ( 'conflict_id', 'tuid', 'vendor_id', 'subaccount_id', 'daily_average_views', 'product_id', 'views_in_conflict') DATE_FIELDS = ( 'conflict_date',) DATETIME_FIELDS = ( 'resolved_datetime',) NULL_VALUES = ('NULL', '\\N', '\\\\N') def process_csv_file(input_gzipped_csv_file_path, output_fp): """Process CSV file and output JSON to file. Args: input_gzipped_csv_file_path (string): Input file path output_fp (io.IOBase subclass): Output file pointer Results: int: Number of conflict entries created. """ columns = None result = {} prev_unique_key = (None, None, None) num_results = 0 territories = ows_territories.get_territories() if not territories: return territories territories_map = _make_territory_map(territories.message['items']) for row in _read_gzip_csv(input_gzipped_csv_file_path): if columns is None: columns = [name.lower() for name in row] continue data = _convert_row_to_dict(row, columns) curr_unique_key = ( data['conflicting_owner'], data['tuid'], data['conflict_date']) territory_code = data['territory'] child_data = { 'conflict_id': data['conflict_id'], 'code': territory_code, 'continent_name': territories_map[territory_code]['continent'], 'name': territories_map[territory_code]['territory_name'] } del data['conflict_id'] del data['territory'] if result: if prev_unique_key == curr_unique_key: result['territories'].append(child_data) else: _write_json_to_file(result, output_fp) num_results += 1 result = None if not result: result = data data['territories'] = [child_data] prev_unique_key = curr_unique_key _write_json_to_file(result, output_fp) num_results += 1 return num_results def _write_json_to_file(json_entry, output_fp): """Write JSON to output file.""" json.dump(json_entry, output_fp) output_fp.write('\n') def _read_gzip_csv(gzipped_file_path): """Read gzipped CSV file.""" with gzip.open(gzipped_file_path, 'rt') as f: reader = csv.reader(f) for row in reader: yield row def _convert_row_to_dict(data, columns): """Convert a row of CSV data to dict.""" result = {} # Quick Read and store vals for i in range(0, len(columns)): result[columns[i]] = data[i] # Val type conversion for field in INT_FIELDS: result[field] = _to_int(result[field]) for field in DATE_FIELDS: result[field] = _to_date(result[field]) for field in DATETIME_FIELDS: result[field] = _to_datetime(result[field]) artist_names = result['artist_names'] if artist_names: artist_names = artist_names.split('|||') else: artist_names = [] if result['subaccount_name'] in NULL_VALUES: result['subaccount_name'] = '' del result['artist_names'] result['track_artists'] = artist_names if result['version'] and result['version'] not in NULL_VALUES: result['track_name'] = '{} ({})'.format( result['track_name'], result['version']) del result['version'] if (result['product_delivered_version'] and result['product_delivered_version'] not in NULL_VALUES): result['product_name'] = '{} ({})'.format( result['product_name'], result['product_delivered_version']) del result['product_delivered_version'] if result['es_id'] in NULL_VALUES: result['es_id'] = '' return result def _to_int(value): if value in NULL_VALUES: return None else: return int(value) def _to_date(value): if value in NULL_VALUES: return None else: return value def _to_datetime(value): if value in NULL_VALUES: return None else: return datetime.strptime( value[:-2], '%a, %d %b %Y %H:%M:%S').isoformat() def _make_territory_map(territories): """Make map of territories with `territory_code_a2`.""" territories_map = {} for territory in territories: territories_map[territory['territory_code_a2']] = territory return territories_map