from abc import abstractmethod from typing import Any, Dict from app_types import S3Path from ... import BaseStep from ...mixins import CSVMixin, ESMixin, S3Mixin __all__ = ["SearchLoadIndexBase"] class SearchLoadIndexBase(ESMixin, S3Mixin, CSVMixin, BaseStep): priority = 4 @property @abstractmethod def data_path(self) -> S3Path: pass @property @abstractmethod def base_index_name(self) -> str: pass @property @abstractmethod def index_mapping_params(self) -> dict: pass @property def index_name(self) -> str: return f"{self.base_index_name}_{self.timestamp}" def process(self) -> Dict[str, Any]: total = 0 self.es_client.index_create(index=self.index_name, param=self.index_mapping_params) for path in self.get_s3_keys(self.data_path): records = self.read_csv(self.get_s3_object(path)) total += self.es_client.index(index=self.index_name, records=records) self.logger.info(f"{total} records upserted") return {"inserted_records": total}