import csv import json import re from datetime import date from io import StringIO from typing import Any, Callable, Collection, Dict, Iterator, List, Type, Union from constants import CSV_DELIMITER, CSV_NULL_VALUE, CSV_QUOTE_CHAR from utils.perf_counter import timeit __all__ = ["CSVMixin"] class JSONEncoder(json.JSONEncoder): def default(self, o: Any) -> Any: if isinstance(o, date): return o.isoformat() return super().default(o) class CSVMixin: csv_delimiter: str = CSV_DELIMITER csv_quote_char: str = CSV_QUOTE_CHAR csv_null_value: str = CSV_NULL_VALUE date_pattern: re.Pattern = re.compile(r"\d{4}-\d{2}-\d{2}") @property def csv_normalizers(self) -> Dict[str, Callable[[Any], Any]]: return {} @staticmethod def format_json(value: Union[List, Dict], cls: Type[json.JSONEncoder] = JSONEncoder) -> str: return json.dumps(value, cls=cls) @classmethod def _is_date(cls, value: Any) -> bool: return isinstance(value, str) and bool(re.match(cls.date_pattern, value)) @classmethod def _is_json(cls, value: Any) -> bool: return ( isinstance(value, str) and len(value) >= 2 and ((value[0] == "[" and value[-1] == "]") or (value[0] == "{" and value[-1] == "}")) ) @staticmethod def _parse_date(value: str) -> Any: try: return date.fromisoformat(value) except ValueError: pass return value @classmethod def _parse_json(cls, value: str) -> Any: try: value = json.loads(value) if isinstance(value, list) and value and cls._is_date(value[0]): value = [cls._parse_date(v) for v in value] except ValueError: pass return value @classmethod def _default_normalizer(cls, value: Any) -> Any: if not isinstance(value, str): return value # if value looks like null/na/nan elif value == cls.csv_null_value: value = None # if value looks like numeric elif value.lstrip("-").isdigit(): try: value = int(value) except ValueError: pass # if value looks like json elif cls._is_json(value): value = cls._parse_json(value) # if value looks like date elif cls._is_date(value): value = cls._parse_date(value) return value def _normalize_record(self, record: Dict[str, Any]) -> Dict[str, Any]: normalized_record = {} for key, value in record.items(): key = key.lower() normalizer = self.csv_normalizers.get(key, self._default_normalizer) normalized_record[key] = normalizer(value) return normalized_record @timeit() def read_csv(self, data: Collection[str]) -> Iterator[Dict[str, Any]]: for record in csv.DictReader(data, delimiter=self.csv_delimiter, quotechar=self.csv_quote_char): yield self._normalize_record(record) @timeit() def write_csv(self, data: Collection[Dict[str, Any]], header: Collection[str]) -> bytes: with StringIO() as buf: writer = csv.DictWriter(buf, header, delimiter=CSV_DELIMITER, quotechar=CSV_QUOTE_CHAR) writer.writeheader() writer.writerows(data) result = buf.getvalue().encode() return result