"""Database utility functions.""" from collections.abc import Callable, Sequence DEFAULT_SAFETY_MARGIN_PCT = 0.9 """Fraction of max_query_bytes to utilize when estimating query size.""" def calculate_max_batch_size( max_query_bytes: int, bytes_per_entry: int, base_query_bytes: int, safety_margin_pct: float = DEFAULT_SAFETY_MARGIN_PCT, ) -> int: """Calculate maximum batch size that fits within a query byte limit. Estimates how many entries (e.g., contract IDs in an IN clause) can fit in a query without exceeding max_query_bytes. Accounts for 1 byte of separator overhead (comma) per entry. Args: max_query_bytes: Maximum query size in bytes (e.g., max_allowed_packet). bytes_per_entry: Estimated byte size of a single entry. base_query_bytes: Byte size of the fixed query template. safety_margin_pct: Fraction of max_query_bytes to utilize (0.0 to 1.0). Returns: Maximum number of entries, or 0 if not even one fits. Raises: ValueError: If inputs are invalid. """ if max_query_bytes < 1: raise ValueError( f'max_query_bytes must be positive. Received: {max_query_bytes}' ) if bytes_per_entry < 1: raise ValueError( f'bytes_per_entry must be positive. Received: {bytes_per_entry}' ) if base_query_bytes < 0: raise ValueError( f'base_query_bytes must be nonnegative. Received: {base_query_bytes}' ) if not 0 < safety_margin_pct <= 1: raise ValueError( f'safety_margin_pct must be within (0, 1]. Received: {safety_margin_pct}' ) # Calculate safe available bytes available = int(max_query_bytes * safety_margin_pct) - base_query_bytes # Calculate batch size # Adds 1 byte to bytes_per_entry for the comma separator: 1,2,3,... # However, the last item has no comma: ...7,8,9 # 1 byte is added to available to adjust for the extra comma batch_size = (available + 1) // (bytes_per_entry + 1) return max(0, batch_size) def _ffd_place[K]( batches: list[list[tuple[K, int]]], batch_remaining: list[int], item_key: K, item_size: int, max_batch_size: int, ) -> None: """Place a single item into batches using First Fit. Finds the first batch with enough remaining capacity, or creates a new one. """ for i, remaining in enumerate(batch_remaining): if remaining >= item_size: batches[i].append((item_key, item_size)) batch_remaining[i] -= item_size return batches.append([(item_key, item_size)]) batch_remaining.append(max_batch_size - item_size) def build_batches[T, K]( items: Sequence[T], max_batch_size: int, *, key: Callable[[T], K], size: Callable[[T], int], ) -> list[list[tuple[K, int]]]: """Pack items into batches under max_batch_size using First Fit Decreasing. Sorts items by size descending, then places each into the first existing batch with enough remaining capacity. Each batch entry is a (key, size) pair. Oversized items (size > max_batch_size) get their own single-item batch containing the full-portion (size minus remainder). The remainder is FFD-packed with regular items. Args: items: Sequence of items to batch. max_batch_size: Maximum total size per batch. key: Callable to extract the grouping key from an item. size: Callable to extract the size/weight from an item. Returns: List of batches, each a list of (key, size) pairs. Oversized batches appear first, then FFD-packed regular batches. """ regular: list[list[tuple[K, int]]] = [] batch_remaining: list[int] = [] oversized: list[list[tuple[K, int]]] = [] sorted_items = sorted(items, key=size, reverse=True) for item in sorted_items: item_key = key(item) item_size = size(item) if item_size <= max_batch_size: _ffd_place(regular, batch_remaining, item_key, item_size, max_batch_size) continue remainder = item_size % max_batch_size oversized.append([(item_key, item_size - remainder)]) if remainder > 0: _ffd_place(regular, batch_remaining, item_key, remainder, max_batch_size) # Oversized batches must be processed first so their adjustments are # committed before regular batches query the same contracts' remainders. return oversized + regular