"""Unit tests for the evaluate_rules logic module.""" import json import re from unittest.mock import patch from src.logic import evaluate_rules as norm ISO_UTC = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$") def _wrap(inner: dict) -> dict: """Wrap an inner validation dict in an Anthropic API envelope.""" return {"content": [{"type": "text", "text": json.dumps(inner)}]} def test_bucket_passed_rule_ids_groups_by_confidence() -> None: """Passed rule IDs are grouped into HIGH/MEDIUM/LOW buckets.""" buckets = norm.bucket_passed_rule_ids(["1.5", "1.2", "1.1", "3.3"]) assert buckets == { "HIGH": ["1.5", "3.3"], "MEDIUM": ["1.2"], "LOW": ["1.1"], } def test_bucket_passed_rule_ids_skips_unknown() -> None: """Unknown rule IDs are silently skipped from buckets.""" buckets = norm.bucket_passed_rule_ids(["999.99", "1.1"]) assert buckets == {"HIGH": [], "MEDIUM": [], "LOW": ["1.1"]} def test_confidence_score_weighted_pass_rate() -> None: """Score is passing weight / total evaluated weight * 100.""" assert norm.confidence_score(["1.5", "3.3"], ["2.1"]) == 83 def test_confidence_score_all_passed() -> None: """Score is 100 when there are no issues.""" assert norm.confidence_score(["1.5", "1.2", "1.1"], []) == 100 def test_confidence_score_all_issues() -> None: """Score is 0 when everything is in issues.""" assert norm.confidence_score([], ["1.5", "1.2", "1.1"]) == 0 def test_confidence_score_zero_when_no_known_rules() -> None: """Score is 0 when no recognized rules appear on either side.""" assert norm.confidence_score([], []) == 0 assert norm.confidence_score(["999.99"], ["888.88"]) == 0 def test_evaluation_coverage_full() -> None: """Coverage is 100 when all expected rules are accounted for.""" from src.logic.rule_confidence import get_expected_rule_ids all_rules = get_expected_rule_ids() assert norm.evaluation_coverage(all_rules, set()) == 100 def test_evaluation_coverage_empty() -> None: """Coverage is 0 when no rules were evaluated.""" assert norm.evaluation_coverage(set(), set()) == 0 def test_evaluation_coverage_not_applicable_counts() -> None: """N/A rules count toward coverage the same as returned rules.""" from src.logic.rule_confidence import get_expected_rule_ids all_rules = get_expected_rule_ids() half = set(list(all_rules)[: len(all_rules) // 2]) other_half = all_rules - half assert norm.evaluation_coverage(half, other_half) == 100 def test_evaluation_coverage_unknown_rules_ignored() -> None: """Unknown rule IDs do not contribute to coverage.""" assert norm.evaluation_coverage({"999.99"}, set()) == 0 def test_evaluation_coverage_weighted_by_confidence() -> None: """A single HIGH rule contributes more coverage than a single LOW rule.""" high_only = norm.evaluation_coverage({"1.5"}, set()) # HIGH=100 low_only = norm.evaluation_coverage({"1.1"}, set()) # LOW=40 assert high_only > low_only def test_evaluation_coverage_returns_zero_when_lookup_is_empty() -> None: """Coverage is 0 when the rule lookup table has no entries.""" with patch("src.logic.evaluate_rules.get_lookup", return_value={}): assert norm.evaluation_coverage({"1.5"}, set()) == 0 def test_derive_validation_status_fail_takes_precedence() -> None: """FAIL wins over warnings even when both are present.""" assert ( norm.derive_validation_status( [ {"ruleId": "1.5", "status": "FAIL"}, {"ruleId": "2.1", "status": "WARNING"}, ] ) == "FAIL" ) def test_derive_validation_status_pass_with_warnings() -> None: """Warnings without failures yield PASS_WITH_WARNINGS.""" assert ( norm.derive_validation_status([{"ruleId": "2.1", "status": "WARNING"}]) == "PASS_WITH_WARNINGS" ) def test_derive_validation_status_pass() -> None: """No issues yield PASS.""" assert norm.derive_validation_status([]) == "PASS" def test_unevaluated_rule_ids_diffs_against_expected() -> None: """Expected rule IDs missing from the LLM response are returned sorted.""" returned = {"1.1", "1.2"} result = norm.unevaluated_rule_ids(returned, set()) assert "1.1" not in result assert "1.5" in result assert result == sorted(result) def test_unevaluated_rule_ids_excludes_not_applicable() -> None: """Rules marked N/A are excluded from unevaluated even if not in returned.""" result = norm.unevaluated_rule_ids({"1.1"}, {"1.5"}) assert "1.5" not in result def test_evaluate_full_response_shape(mock_llm_response: dict) -> None: """End-to-end evaluate output has all required fields.""" result = norm.evaluate(mock_llm_response, release_id=2427859) assert result["validationStatus"] == "FAIL" assert result["releaseId"] == 2427859 assert result["llmScore"] == 50 assert result["llmRationale"] == "Test rationale." # pass_weight=510 (1.1=40,1.2=70,1.10=100,2.7=100,3.3=100,3.4=100) # issue_weight=140 (1.5=HIGH=100, 2.1=LOW=40) → round(510/650*100)=78 assert result["confidenceScore"] == 78 # 8 rules evaluated out of 85 expected (weighted: 650/5530) → 12 assert result["evaluationCoverage"] == 12 assert result["passedRuleIds"] == { "HIGH": ["1.10", "2.7", "3.3", "3.4"], "MEDIUM": ["1.2"], "LOW": ["1.1"], } assert result["notApplicableRuleIds"] == [] assert result["summary"] == { "totalRulesChecked": 8, "passed": 6, "warnings": 1, "failures": 1, "notApplicable": 0, "unevaluated": len(result["unevaluatedRuleIds"]), } issue_rule_ids = {i["ruleId"] for i in result["issues"]} assert issue_rule_ids == {"1.5", "2.1"} for issue in result["issues"]: assert "confidence" in issue assert "1.5" not in result["unevaluatedRuleIds"] assert "2.1" not in result["unevaluatedRuleIds"] assert "1.1" not in result["unevaluatedRuleIds"] assert ISO_UTC.match(result["validationTimestamp"]) def test_evaluate_enriches_issues_with_confidence() -> None: """Each issue gets a confidence field populated from the rule lookup.""" llm = _wrap( { "passedRuleIds": [], "notApplicableRuleIds": [], "issues": [ { "ruleId": "1.5", "status": "FAIL", "scope": "release", "fieldName": "x", "affectedInstances": [], "message": "m", }, { "ruleId": "2.1", "status": "WARNING", "scope": "release", "fieldName": "y", "affectedInstances": [], "message": "m", }, ], "score": 0, "rationale": "", } ) result = norm.evaluate(llm) confidences = {i["ruleId"]: i["confidence"] for i in result["issues"]} assert confidences["1.5"] == "HIGH" assert confidences["2.1"] == "LOW" def test_evaluate_not_applicable_excluded_from_unevaluated() -> None: """Non Applicable Rule ids are not counted as unevaluated.""" llm = _wrap( { "passedRuleIds": [], "notApplicableRuleIds": ["1.5", "2.7"], "issues": [], "score": 0, "rationale": "", } ) result = norm.evaluate(llm) assert "1.5" not in result["unevaluatedRuleIds"] assert "2.7" not in result["unevaluatedRuleIds"] assert result["summary"]["notApplicable"] == 2 def test_evaluate_handles_empty_input() -> None: """Missing arrays default to empty without errors.""" result = norm.evaluate(_wrap({})) assert result["validationStatus"] == "PASS" assert result["summary"]["passed"] == 0 assert result["summary"]["failures"] == 0 assert result["summary"]["warnings"] == 0 assert result["confidenceScore"] == 0 assert len(result["unevaluatedRuleIds"]) > 0 def test_evaluate_skips_non_dict_issues() -> None: """Non-dict entries in the issues list are silently ignored.""" llm = _wrap( { "passedRuleIds": [], "notApplicableRuleIds": [], "issues": [ "not a dict", None, { "ruleId": "1.5", "status": "FAIL", "scope": "release", "fieldName": "x", "affectedInstances": [], "message": "m", }, ], "score": 0, "rationale": "", } ) result = norm.evaluate(llm) assert len(result["issues"]) == 1 assert result["issues"][0]["ruleId"] == "1.5" def test_evaluate_issue_without_rule_id_has_null_confidence() -> None: """Issues with no ruleId are kept but get confidence=None.""" llm = _wrap( { "passedRuleIds": [], "notApplicableRuleIds": [], "issues": [ { "status": "WARNING", "scope": "release", "fieldName": "x", "affectedInstances": [], "message": "no ruleId here", }, ], "score": 0, "rationale": "", } ) result = norm.evaluate(llm) assert len(result["issues"]) == 1 assert result["issues"][0]["confidence"] is None def test_derive_validation_status_ignores_non_dict_items() -> None: """Non-dict issues are skipped; valid dicts still decide status.""" assert ( norm.derive_validation_status( [ "not a dict", None, {"ruleId": "2.1", "status": "WARNING"}, ] ) == "PASS_WITH_WARNINGS" ) assert norm.derive_validation_status(["not a dict", None]) == "PASS"