import click from click import secho from structlog import get_logger from delphi_es_utils.constants import AVAILABLE_INDICES from delphi_es_utils.core.utils import Stopwatch from delphi_es_utils.services import DataManager, IndexManager from delphi_es_utils.settings import DEBUG LOG = get_logger(__name__) def bulk_index(index_name, context=None, event=None): LOG.info('Running bulk index command...', index=index_name, context=context, lambda_event=event) secho(f'Running bulk index command... index={index_name}', fg='yellow') if index_name not in AVAILABLE_INDICES: raise NotImplementedError('Unsupported index alias provided') stopwatch = Stopwatch() # Data export LOG.debug( 'Starting data export from the database...', index=index_name, context=context, lambda_event=event ) secho(f'Starting data export from the database... index={index_name}', fg='yellow') data_manager = DataManager(index_name) data = data_manager.get_data() # Data export logging end_data_dump = stopwatch.get_end() LOG.info( 'Completed data export from the database.', num_rows=len(data), export_duration=stopwatch.get_end_seconds(end_data_dump), context=context ) secho( f'Completed data export from the database. items={len(data)}; elapsed={end_data_dump}', fg='magenta' ) # Bulk index LOG.debug( 'Starting bulk index to Elasticsearch...', index=index_name, context=context, lambda_event=event ) secho(f'Starting bulk index to Elasticsearch... index={index_name}', fg='yellow') index_manager = IndexManager(index_name) result = index_manager.bulk_index(data, stats_only=not DEBUG) # Bulk index logging end_total = stopwatch.get_end() LOG.info( 'Ran the bulk indexing job for index', index=index_name, result=result, bulk_duration=stopwatch.get_end_seconds(end_total), context=context, lambda_event=event ) secho( f'Completed the bulk indexing process in elapsed={end_total}; ' f'index={index_name}; result={result}', fg='magenta' ) return result @click.command('bulk-index', short_help='Bulk index from a remote data source.') @click.option('--name', required=True, help='Name of an index alias. example: "linkfire-link"') def bulk_index_cmd(name, context=None, event=None): """Run a bulk indexing job from a remote data source. Examples: bulk-index --name linkfire-link """ return bulk_index(name, context=context, event=event)