#!/usr/bin/env python3 """ Script to compare account query results against accountsxPP: - orchardLabels vs accountsxPP (for no search term / first 10 accounts) - orchardLabelSearch vs accountsxPP (for search terms) """ import json import os import subprocess import sys from pathlib import Path from typing import Dict, Any, List, Tuple, Optional from dotenv import load_dotenv from search_terms_to_test import SEARCH_TERMS from queries import ORCHARD_LABEL_SEARCH_QUERY, ACCOUNTS_XPP_QUERY, ORCHARD_LABELS_QUERY # Load environment variables load_dotenv() # Configuration GRAPHQL_ENDPOINT = "https://qa-graphql-router.theorchard.io/graphql" DEFAULT_QUERY_LIMIT = 10 # Default number of results to fetch from queries SKIP_QUERIES = False # Set to True to skip querying and use existing results/query.json LIMIT_TERMS = None # Set to a number to limit search terms for testing, or None for all RESULTS_DIR = Path("results") QUERY_RESULTS_FILE = RESULTS_DIR / "query.json" COMPARISON_RESULTS_FILE = RESULTS_DIR / "comparison.json" # Required headers from environment REQUIRED_ENV_VARS = [ "ORCHARD_PROFILE_TYPE", "ORCHARD_PROFILE_ID", "ORCHARD_IDENTITY_ID", "APOLLOGRAPHQL_CLIENT_NAME", "ORCHARD_PROFILE_UUID", "AUTHORIZATION", "ORCHARD_ROLES" ] def make_graphql_request(query: str, variables: Dict[str, Any]) -> Dict[str, Any]: """Make a GraphQL request using curl.""" payload = {"query": query, "variables": variables} curl_command = [ "curl", "-X", "POST", GRAPHQL_ENDPOINT, "-H", "Content-Type: application/json", "-H", f"Orchard-Profile-Type: {os.getenv('ORCHARD_PROFILE_TYPE')}", "-H", f"Orchard-Profile-Id: {os.getenv('ORCHARD_PROFILE_ID')}", "-H", f"Orchard-Identity-Id: {os.getenv('ORCHARD_IDENTITY_ID')}", "-H", f"apollographql-client-name: {os.getenv('APOLLOGRAPHQL_CLIENT_NAME')}", "-H", f"Orchard-profile-uuid: {os.getenv('ORCHARD_PROFILE_UUID')}", "-H", f"Authorization: {os.getenv('AUTHORIZATION')}", "-H", f"Orchard-roles: {os.getenv('ORCHARD_ROLES')}", "-d", json.dumps(payload) ] try: result = subprocess.run(curl_command, capture_output=True, text=True, check=True) return json.loads(result.stdout) except subprocess.CalledProcessError as e: print(f"Error making request: {e}\nstderr: {e.stderr}") return {} except json.JSONDecodeError as e: print(f"Error parsing response: {e}") return {} def query_orchard_label_search(term: str, limit: int = DEFAULT_QUERY_LIMIT) -> Dict[str, Any]: """Query using orchardLabelSearch.""" return make_graphql_request( ORCHARD_LABEL_SEARCH_QUERY, {"term": term, "limit": limit} ) def query_accounts_xpp(term: str, limit: int = DEFAULT_QUERY_LIMIT) -> Dict[str, Any]: """Query using accountsxPP.""" return make_graphql_request( ACCOUNTS_XPP_QUERY, { "limit": limit, "scope": {"action": "bulk_create", "resourceType": "digital_audio"}, "filter": {"labelType": "VENDOR", "searchTerm": term, "status": ["SIGNED"]} } ) def query_accounts_no_term(limit: int = DEFAULT_QUERY_LIMIT) -> Dict[str, Any]: """ Query accounts with no search term. Returns results from both orchardLabels and accountsxPP (empty string). """ return { "term": "", "orchardLabels": make_graphql_request(ORCHARD_LABELS_QUERY, {}), "accountsxPP": query_accounts_xpp("", limit) } def extract_results(response: Dict[str, Any], query_type: str) -> List[Tuple[Optional[int], Optional[str]]]: """ Extract (vendorId, name) tuples from query response. Args: response: GraphQL response query_type: 'orchard' (orchardLabelSearch or orchardLabels) or 'xpp' Returns: List of (vendorId, name) tuples """ # Check for GraphQL errors (but continue processing if data is present) if "errors" in response: print(f"WARNING: GraphQL query returned errors: {response['errors']}") results = [] try: data = response.get("data") or {} if query_type == "orchard": items = data.get("orchardLabelSearch") or data.get("orchardLabels") or [] for item in items: if item is None: results.append((None, None)) else: results.append(( (item.get("id") or {}).get("vendorId"), item.get("name") )) elif query_type == "xpp": accounts_data = data.get("accountsxPP") or {} items = accounts_data.get("items") or [] for item_wrapper in items: if item_wrapper is None: results.append((None, None)) else: item = item_wrapper.get("item") or {} results.append(( item.get("vendorId"), item.get("name") )) except Exception as e: print(f"WARNING: Error extracting {query_type} results: {e}") return [] return results def compare_results(term: str, orchard_response: Dict[str, Any], xpp_response: Dict[str, Any], orchard_query_type: str) -> Dict[str, Any]: """ Compare results from both queries. Args: term: Search term orchard_response: Response from orchardLabels or orchardLabelSearch xpp_response: Response from accountsxPP orchard_query_type: Either 'orchardLabels' or 'orchardLabelSearch' """ orchard_results = extract_results(orchard_response, "orchard") xpp_results = extract_results(xpp_response, "xpp") # Check if first result matches first_result_same = ( bool(orchard_results and xpp_results) and orchard_results[0] == xpp_results[0] ) # Count matching accounts matching_count = len(set(orchard_results) & set(xpp_results)) # Use appropriate field name based on query type orchard_count_field = f"{orchard_query_type}ResultsCount" return { "searchTerm": term, "firstResultSame": first_result_same, "matchingAccountsCount": matching_count, orchard_count_field: len(orchard_results), "accountsxPPResultsCount": len(xpp_results) } def load_query_results() -> List[Dict[str, Any]]: """Load existing query results from file.""" try: results = json.loads(QUERY_RESULTS_FILE.read_text()) print(f"āœ“ Loaded {len(results)} query results from {QUERY_RESULTS_FILE}\n") return results except FileNotFoundError: print(f"Error: {QUERY_RESULTS_FILE} not found. Run with SKIP_QUERIES=False first.") sys.exit(1) def check_environment_variables() -> None: """Check that all required environment variables are set.""" missing_vars = [var for var in REQUIRED_ENV_VARS if not os.getenv(var)] if missing_vars: print("Error: Missing required environment variables:") for var in missing_vars: print(f" - {var}") print("\nPlease create a .env file based on env.shadow and fill in the values.") sys.exit(1) def run_queries() -> List[Dict[str, Any]]: """Run queries for all search terms.""" check_environment_variables() # First, query first 10 accounts with no search term print("Querying first 10 accounts with no search term...") results = [query_accounts_no_term()] print("āœ“ Completed\n") # Query search terms (LIMIT_TERMS controls total results, so subtract 1 for the no-term query) terms = SEARCH_TERMS if LIMIT_TERMS is None else SEARCH_TERMS[:max(0, LIMIT_TERMS - 1)] if terms: print(f"Querying {len(terms)} search terms") print(f"Endpoint: {GRAPHQL_ENDPOINT}\n") for i, term in enumerate(terms, 1): print(f"[{i}/{len(terms)}] Testing: {term}") results.append({ "term": term, "orchardLabelSearch": query_orchard_label_search(term), "accountsxPP": query_accounts_xpp(term) }) # Create results directory if it doesn't exist RESULTS_DIR.mkdir(exist_ok=True) # Save results QUERY_RESULTS_FILE.write_text(json.dumps(results, indent=2)) print(f"\nāœ“ Query results saved to: {QUERY_RESULTS_FILE}") return results def run_comparison(query_results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Run comparison on query results.""" comparisons = [] for result in query_results: term = result.get("term", "") # Determine which orchard query was used orchard_query_type = "orchardLabels" if "orchardLabels" in result else "orchardLabelSearch" orchard_response = result.get(orchard_query_type) xpp_response = result.get("accountsxPP") # Validate required data exists if not orchard_response: raise ValueError( f"Missing orchard query response for term '{term}'. " f"Expected 'orchardLabelSearch' or 'orchardLabels' in result." ) if not xpp_response: raise ValueError( f"Missing accountsxPP response for term '{term}'. " f"Expected 'accountsxPP' in result." ) comparisons.append( compare_results(term, orchard_response, xpp_response, orchard_query_type) ) return comparisons def print_summary(comparison_results: List[Dict[str, Any]]) -> None: """Print comparison summary statistics.""" total = len(comparison_results) if total == 0: print("No results to compare") return def percentage(count: int) -> str: return f"{count/total*100:.1f}%" # Calculate statistics first_match_count = sum(1 for r in comparison_results if r["firstResultSame"]) perfect_match_count = sum( 1 for r in comparison_results if (r["firstResultSame"] and r["matchingAccountsCount"] == r.get("orchardLabelSearchResultsCount", r.get("orchardLabelsResultsCount")) == r["accountsxPPResultsCount"]) ) thresholds = {3: 0, 5: 0, 7: 0} for r in comparison_results: for threshold in thresholds: if r["matchingAccountsCount"] >= threshold: thresholds[threshold] += 1 # Print summary print(f"āœ“ Comparison results saved to: {COMPARISON_RESULTS_FILE}") print(f"\n{'='*60}") print(f"SUMMARY") print(f"{'='*60}") print(f"Total search terms: {total}") print(f"First result matches: {first_match_count} ({percentage(first_match_count)})") print(f"Perfect matches (all results same): {perfect_match_count} ({percentage(perfect_match_count)})") print(f"\nMatching results thresholds:") for threshold, count in sorted(thresholds.items()): print(f" {threshold}+ matches: {count} ({percentage(count)})") def main(): """Main execution function.""" # Load or run queries query_results = load_query_results() if SKIP_QUERIES else run_queries() # Run comparison print("\nRunning comparison...") comparison_results = run_comparison(query_results) # Create results directory if it doesn't exist RESULTS_DIR.mkdir(exist_ok=True) # Save comparison results COMPARISON_RESULTS_FILE.write_text(json.dumps(comparison_results, indent=2)) # Print summary print_summary(comparison_results) if __name__ == "__main__": main()