import json import time import logging import os from botocore.exceptions import ClientError logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) TERMINAL = {'SUCCEEDED', 'FAILED', 'TIMED_OUT', 'ABORTED'} POLL_INTERVAL = 2.0 POLL_TIMEOUT = 900 class TestResult: """Container for test execution results.""" def __init__(self, review_queue_id, status, **kwargs): self.review_queue_id = review_queue_id self.status = status # Store any additional attributes from kwargs for key, value in kwargs.items(): setattr(self, key, value) def load_test_fixtures(fixture_file): """ Fixture format: [ { "review_queue_id": 123, "is_integration_test": true, ..., "expected": { "validation": "pass" | "fail" } }, ... ] """ fixture_path = os.path.join(os.path.dirname(__file__), fixture_file) with open(fixture_path, 'r', encoding='utf-8') as f: raw = json.load(f) if not isinstance(raw, list): raise ValueError(f'{fixture_file} must be a JSON array') for i, item in enumerate(raw): if not isinstance(item, dict): raise ValueError(f'{fixture_file}[{i}] must be an object') if 'review_queue_id' not in item: raise ValueError(f'{fixture_file}[{i}] is missing required key "review_queue_id"') return raw def start_stepfunction_execution(sfn_client, stepfunction_arn, input_data): """ Start a Step Function execution. """ execution_input = { 'data': input_data, 'is_integration_test': True } rid = input_data.get('review_queue_id', 'unknown') timestamp = int(time.time()) try: resp = sfn_client.start_execution( stateMachineArn=stepfunction_arn, name=f'int-test-execution-{rid}-{timestamp}', input=json.dumps(execution_input) ) return resp['executionArn'] except ClientError as e: error_code = e.response.get('Error', {}).get('Code', 'Unknown') http_code = e.response['ResponseMetadata']['HTTPStatusCode'] logger.error( f'AWS error starting execution for review_queue_id={rid}: ' f'{error_code} (HTTP {http_code}) - {e.response["Error"]["Message"]}' ) raise def poll_execution(sfn_client, execution_arn, **kwargs): """ Poll a Step Function execution until completion. """ start = time.time() while True: elapsed = time.time() - start if elapsed > POLL_TIMEOUT: raise TimeoutError(f'Timed out after {POLL_TIMEOUT}s: {execution_arn}') desc = sfn_client.describe_execution(executionArn=execution_arn) status = desc['status'] if status in TERMINAL: out_str = desc.get('output') if out_str: try: output = json.loads(out_str) except json.JSONDecodeError: output = out_str else: output = None return status, output, elapsed time.sleep(POLL_INTERVAL) def run_tests(sfn_client, stepfunction_arn, fixtures, **kwargs): """Run all fixtures through the step function.""" results = [] for fixture in fixtures: expected = fixture.get('expected', {}) input_data = {k: v for k, v in fixture.items() if k != 'expected'} review_queue_id = input_data.get('review_queue_id', 'test') try: execution_arn = start_stepfunction_execution(sfn_client, stepfunction_arn, input_data) status, output, elapsed = poll_execution(sfn_client, execution_arn) # Extract actual validation result from SFN output actual_validation = None if output and isinstance(output, dict): try: actual_validation = output['ProductValidationTaskResponse']['Payload']['validations'] except (KeyError, TypeError) as e: logger.warning(f'Could not extract validation from output for review_queue_id={review_queue_id}: {e}') results.append(TestResult( review_queue_id=review_queue_id, status=status, output=output, elapsed=elapsed, expected=expected, actual_validation=actual_validation, )) except Exception as e: logger.error(f'Error for review_queue_id={review_queue_id}: {e}', exc_info=True) results.append(TestResult( review_queue_id=review_queue_id, status='ERROR', expected=expected, error=e, error_type=type(e).__name__, error_message=str(e), )) return results def generate_test_report(results, stepfunction_arn, **kwargs): """ Generate test report summary. Tracks: - Successful step function executions - Actual vs expected validation pass/fail counts - Validation mismatches """ total = len(results) execution_failure_count = 0 execution_failure_details = [] test_fail_details = [] for r in results: expected_validation = getattr(r, 'expected', {}).get('validation') actual_validation = getattr(r, 'actual_validation', None) if r.status != 'SUCCEEDED': execution_failure_count += 1 execution_failure_details.append({ 'review_queue_id': r.review_queue_id, 'status': r.status, 'error_type': getattr(r, 'error_type', None), 'error_message': getattr(r, 'error_message', None), }) if actual_validation != expected_validation: test_fail_details.append({ 'review_queue_id': r.review_queue_id, 'expected': expected_validation, 'actual': actual_validation, }) test_pass = total - len(test_fail_details) return { 'total': total, 'sfn_execution_success': total - execution_failure_count, 'sfn_execution_failure': execution_failure_count, 'sfn_execution_failure_details': execution_failure_details, 'test_pass_count': test_pass, 'test_fail_count': total - test_pass, 'test_fail_details': test_fail_details, }