"""Utility functions for concurrency operations.""" from collections.abc import Mapping from itertools import islice from time import sleep from typing import Any, Dict, Generator, List from connectors.logging import log from utils.jitter import jitter from config import LOG_FREQUENCY, TASK_DELAY def init_async_process(data, task_id: int = None) -> int: """Initialize the async runner. Args: data (list|dict): The data to process. task_id (int, optional): The task ID. Defaults to None. Returns: int: The log frequency. """ msg = f'Started. Processing {len(data)} items.' log_frequency = max(len(data) // LOG_FREQUENCY, 1) # Anti-hammer if TASK_DELAY: sleep_delay = TASK_DELAY + jitter(do_sleep=False) msg = msg + f' (Sleeping {sleep_delay}s).' # Task Id if task_id: msg = f'Task {task_id}: {msg}' log.info(msg) log.info( f'Task {task_id}: Logging every {log_frequency} items.') # Rest before the work begins sleep(sleep_delay) return log_frequency def merge_dicts_old(d1: dict, d2: dict) -> dict: """Recursively merge two dictionaries, extending lists. Args: d1 (dict): The first dictionary. d2 (dict): The second dictionary. Returns: dict: The merged dictionary.""" for key, value in d2.items(): if key in d1: if isinstance(d1[key], dict) and isinstance(value, dict): merge_dicts(d1[key], value) elif isinstance(d1[key], list) and isinstance(value, list): d1[key].extend(value) else: log.warning( f"Conflicting types for key '{key}': {type(d1[key])} vs " f"{type(value)}") else: d1[key] = value return d1 def chunk_data( data: Dict | List, chunk_size: int) -> Generator[Dict | List, None, None]: """Yield successive chunks from iterable data. This method will correctly yield chunks for both dictionaries and lists. Args: data (Dict | List): The data to chunk. chunk_size (int): The size of the chunks to yield. Yields: Generator[Dict | List, None, None]: The chunked data. """ if isinstance(data, dict): data_iter = iter(data.items()) while True: chunk = dict(islice(data_iter, chunk_size)) if not chunk: break yield chunk elif isinstance(data, list): for i in range(0, len(data), chunk_size): yield data[i:i + chunk_size] else: raise TypeError( "Unsupported data type. Only dict and list are supported.") def prepare_errors_dict( errors: dict, category: str, variable: str, optional: str = None) -> dict: """Prepare an error dictionary that has the required structure. Args: errors (dict): The error dictionary. category (str): The top-level category to check or add. This is always a string constant like 'tuid' or 'vendor'. variable (str): A variable value to check or add. This could be a string constant like 'subaccount', or a value like `1432` or 'hey'. optional (str, optional): An optional value to check or add. This is always a value. Defaults to None. """ if not errors.get(category): errors[category] = {} if not errors[category].get(variable): if optional is not None: errors[category][variable] = {} errors[category][variable][optional] = [] else: errors[category][variable] = [] if isinstance(errors[category][variable], dict) and optional is not None: if not errors[category][variable].get(optional): errors[category][variable][optional] = [] return errors def merge_values(val1: Any, val2: Any) -> Any: """ Merges two values intelligently. - If both are dictionaries, merge recursively. - If both are lists, merge with unique elements. - If both are sets, merge. - If they are the same scalar values, keep one. - If scalar values conflict, return the first by default. """ if isinstance(val1, Mapping) and isinstance(val2, Mapping): return merge_dicts(val1, val2) if isinstance(val1, list) and isinstance(val2, list): return val1 + [item for item in val2 if item not in val1] if isinstance(val1, set) and isinstance(val2, set): return val1 | val2 if val1 == val2: return val1 # Identical values, return one # Conflict resolution for non-collections; raise an error raise ValueError( f"Conflicting scalar values: {val1} vs {val2} while attempting to " 'merge') def merge_dicts( dict1: Dict[Any, Any], dict2: Dict[Any, Any]) -> Dict[Any, Any]: """ Recursively merges two dictionaries: - Merges nested dictionaries. - Merges lists and sets to avoid duplicates. - Keeps identical values. - Resolves conflicting scalar values by keeping the first (default behavior). """ merged = {**dict1} # Start with the first dictionary for key, val2 in dict2.items(): if key in merged: merged[key] = merge_values(merged[key], val2) else: merged[key] = val2 # Key is unique to dict2, add it directly return merged def freeze(obj): """Recursively convert a mutable object into an immutable one. Args: obj: The object to freeze (can be dict, list, or other). Returns: An immutable representation of the object. """ if isinstance(obj, dict): # Convert dictionary to a tuple of sorted (key, frozen_value) pairs. return tuple(sorted((k, freeze(v)) for k, v in obj.items())) elif isinstance(obj, list): # Convert list to a tuple of frozen elements. return tuple(freeze(item) for item in obj) return obj # For other immutable types def process_response_data(collected_data, collected_errors): """Process the collected data and errors. Args: collected_data (list): The collected data. collected_errors (list): The collected errors. Returns: None """ # Merge the items in collected_data into a single dictionary final_data = [] seen = set() for data in collected_data: data_tuple = freeze(data) if data_tuple not in seen: seen.add(data_tuple) final_data.append(data) final_errors = {} for error in collected_errors: final_errors = merge_dicts(final_errors, error) return final_data, final_errors def check_vendor_subaccount_match(contributors): # confirm that the vendor_id is the same on participations, performers, # and publishers publisher_vendor_id = contributors['publishers']['vendor_id'] performer_vendor_id = contributors['performers']['vendor_id'] participation_vendor_id = \ contributors['participations']['vendor_id'] # vendor_id = participation_vendor_id # They should all be the same publisher_subaccount_id = \ contributors['publishers']['subaccount_id'] performer_subaccount_id = \ contributors['performers']['subaccount_id'] participation_subaccount_id = \ contributors['participations']['subaccount_id'] # subaccount_id = participation_subaccount_id # ditto vendors_match = all([ publisher_vendor_id == performer_vendor_id, publisher_vendor_id == participation_vendor_id, ]) subaccounts_match = all([ publisher_subaccount_id == performer_subaccount_id, publisher_subaccount_id == participation_subaccount_id, ]) return vendors_match, subaccounts_match