""" Heavy Rotation POC - Streamlit Application This application identifies "Heavy Rotation" listeners using Spotify API data by finding fans who have specific artists in their Spotify Top Artists list. """ import streamlit as st import pandas as pd import json import time from snowflake.snowpark.context import get_active_session from lib.spotify_auth import SpotifyAuth from lib.spotify_client import SpotifyClient from lib.wave_processor import WaveProcessor, WaveConfig from lib.dynamodb_client import ( load_aws_credentials, create_dynamodb_client, fetch_refresh_tokens ) # Page configuration # Theme is configured via .streamlit/config.toml st.set_page_config( page_title="Heavy Rotation POC", page_icon="đŸŽĩ", layout="wide", initial_sidebar_state="expanded" ) def get_spotify_credentials(): """ Retrieve Spotify API credentials from Streamlit secrets In Container Runtime, secrets are provided via secrets.toml file and accessed using st.secrets API. Returns: tuple: (client_id, client_secret) or (None, None) if unavailable """ try: # Access secrets via st.secrets API client_id = st.secrets.get("SPOTIFY_CLIENT_ID") client_secret = st.secrets.get("SPOTIFY_CLIENT_SECRET") if not client_id or not client_secret: st.error("❌ Spotify credentials not found in secrets") st.info(""" **Setup secrets.toml file:** Create a `secrets.toml` file with your Spotify credentials: ```toml SPOTIFY_CLIENT_ID = "your_client_id_here" SPOTIFY_CLIENT_SECRET = "your_client_secret_here" ``` See [Streamlit Secrets documentation](https://docs.streamlit.io/develop/api-reference/connections/secrets.toml) for details. """) return None, None # Show success message only once per session if not st.session_state.get("credentials_loaded", False): st.success("✅ Spotify credentials loaded successfully") st.session_state["credentials_loaded"] = True return client_id, client_secret except Exception as e: st.error(f"❌ Failed to retrieve Spotify credentials: {e}") st.info(""" **Troubleshooting:** 1. Create a `secrets.toml` file in your app directory 2. Add your Spotify API credentials to the file 3. Ensure the file is uploaded to your Snowflake stage 4. Restart the Streamlit app **secrets.toml format:** ```toml SPOTIFY_CLIENT_ID = "your_client_id_here" SPOTIFY_CLIENT_SECRET = "your_client_secret_here" ``` """) return None, None def fetch_presave_campaigns(): """ Fetch presave campaigns from ROTATION_PRESAVE_LOOKUP view Returns: pandas.DataFrame: Presave campaigns with metadata """ try: session = get_active_session() query = """ SELECT PRESAVE_ID, PRESAVE_NAME, ARTIST_NAME, ARTIST_ID, SPOTIFY_ARTIST_ID, PAGE_URL, LABEL_NAME, RELEASE_DATE_TIME, PRESAVE_VERSION FROM ROTATION_PRESAVE_LOOKUP ORDER BY RELEASE_DATE_TIME DESC LIMIT 100 """ df = session.sql(query).to_pandas() return df except Exception as e: st.error(f"Error fetching presave campaigns: {str(e)}") return None def parse_artist_id(artist_input: str) -> str: """ Parse artist ID from Spotify URL or return as-is if already an ID Args: artist_input: Spotify artist URL or artist ID Returns: str: Artist ID Examples: 'https://open.spotify.com/artist/1Xyo4u8uXC1ZmMpatF05PJ' -> '1Xyo4u8uXC1ZmMpatF05PJ' '1Xyo4u8uXC1ZmMpatF05PJ' -> '1Xyo4u8uXC1ZmMpatF05PJ' """ if not artist_input: return None # If it's a URL, extract the ID if 'spotify.com/artist/' in artist_input: # Extract ID from URL parts = artist_input.split('artist/') if len(parts) > 1: # Remove any query parameters artist_id = parts[1].split('?')[0] return artist_id # Otherwise assume it's already an ID return artist_input.strip() def create_time_range_selector(key_suffix=''): """ Create a reusable time range selector widget Args: key_suffix: Optional suffix for the widget key to avoid conflicts Returns: str: Selected time range ('short_term', 'medium_term', or 'long_term') """ return st.selectbox( "Time Range", options=['short_term', 'medium_term', 'long_term'], format_func=lambda x: { 'short_term': '🕐 4 Weeks', 'medium_term': '📅 6 Months', 'long_term': '📆 1 Year' }[x], key=f'time_range{key_suffix}' ) def render_metric_card(label, value, delta=None, icon="📊"): """Render a metric using native Streamlit component""" st.metric(label=f"{icon} {label}", value=value, delta=delta) def render_section_header(title, icon=""): """Render a section header using native Streamlit component""" st.subheader(f"{icon} {title}" if icon else title, divider="green") def render_artist_info(artist_metadata): """Render artist information card""" col1, col2 = st.columns([1, 3]) with col1: if artist_metadata.get('images') and len(artist_metadata['images']) > 0: st.image(artist_metadata['images'][0]['url'], width=150) with col2: st.subheader(f"đŸŽ¯ {artist_metadata['name']}") st.text(f"Spotify ID: {artist_metadata['id']}") st.text(f"Genres: {', '.join(artist_metadata.get('genres', ['N/A'])[:3])}") st.text(f"Popularity: {artist_metadata.get('popularity', 'N/A')}/100") # Initialize session state if 'album_id' not in st.session_state: st.session_state.album_id = '6644715' # Default album ID if 'batch_results' not in st.session_state: st.session_state.batch_results = [] # Results from batch processing if 'batch_processed' not in st.session_state: st.session_state.batch_processed = False if 'processed_time_range' not in st.session_state: st.session_state.processed_time_range = None # Time range used for current results if 'target_artist_id' not in st.session_state: st.session_state.target_artist_id = None # Artist ID to match against if 'target_artist_metadata' not in st.session_state: st.session_state.target_artist_metadata = None # Cached artist metadata if 'presave_campaigns' not in st.session_state: st.session_state.presave_campaigns = None # Cached presave campaigns # Header st.title("đŸŽĩ Heavy Rotation Analytics") st.caption("Identify and analyze fans with artists in their Spotify Top Artists list") # Sidebar configuration with st.sidebar: st.header("âš™ī¸ Configuration") # === PRESAVE CAMPAIGN SELECTION === st.subheader("1ī¸âƒŖ Presave Campaign") # Fetch presave campaigns on first load or when explicitly requested if st.session_state.presave_campaigns is None or st.button("🔄 Refresh Campaigns", use_container_width=True): with st.spinner("Fetching presave campaigns..."): st.session_state.presave_campaigns = fetch_presave_campaigns() # Display presave dropdown or fallback to manual input if st.session_state.presave_campaigns is not None and not st.session_state.presave_campaigns.empty: presaves_df = st.session_state.presave_campaigns # Create display labels for dropdown presaves_df['display_label'] = ( presaves_df['PRESAVE_NAME'] + ' - ' + presaves_df['ARTIST_NAME'] + ' (' + presaves_df['RELEASE_DATE_TIME'].astype(str) + ')' ) # Presave dropdown selected_presave_idx = st.selectbox( "Select Campaign", options=range(len(presaves_df)), format_func=lambda i: presaves_df.iloc[i]['display_label'], help="Select a presave campaign to analyze" ) # Get selected presave details selected_presave = presaves_df.iloc[selected_presave_idx] album_id = str(selected_presave['PRESAVE_ID']) presave_version = str(selected_presave['PRESAVE_VERSION']).strip('*').upper() # Display presave metadata with st.expander("📋 Campaign Details"): st.write(f"**Artist:** {selected_presave['ARTIST_NAME']}") st.write(f"**Label:** {selected_presave['LABEL_NAME']}") st.write(f"**Release Date:** {selected_presave['RELEASE_DATE_TIME']}") st.write(f"**Version:** {selected_presave['PRESAVE_VERSION']}") if selected_presave['PAGE_URL']: st.write(f"**URL:** {selected_presave['PAGE_URL']}") # Auto-populate artist ID from presave auto_artist_id = selected_presave['SPOTIFY_ARTIST_ID'] if selected_presave.get('SPOTIFY_ARTIST_ID') else None else: # Fallback to manual campaign ID input st.warning("âš ī¸ Using manual input") album_id = st.text_input( "Campaign ID", value=st.session_state.album_id, help="Enter the presave campaign ID to fetch tokens from DynamoDB" ) presave_version = 'V1' auto_artist_id = None st.divider() # === ARTIST MATCHING === st.subheader("2ī¸âƒŖ Artist Matching") artist_input = st.text_input( "Target Artist ID or URL", value=auto_artist_id if auto_artist_id else "", help="Enter Spotify artist ID or URL to filter fans by Heavy Rotation", placeholder="e.g., 1Xyo4u8uXC1ZmMpatF05PJ" ) # Parse and validate artist ID if artist_input: target_artist_id = parse_artist_id(artist_input) st.session_state.target_artist_id = target_artist_id # Fetch artist metadata for confirmation if target_artist_id and ( st.session_state.target_artist_metadata is None or st.session_state.target_artist_metadata.get('id') != target_artist_id ): try: # Get Spotify credentials for artist lookup client_id, client_secret = get_spotify_credentials() if client_id and client_secret: with st.spinner("Fetching artist metadata..."): # Use a temporary auth with no refresh token temp_auth = SpotifyAuth( refresh_token=None, client_id=client_id, client_secret=client_secret ) # Get client credentials token temp_auth.get_client_credentials_token() temp_client = SpotifyClient(temp_auth) artist_metadata = temp_client.get_artist(target_artist_id) st.session_state.target_artist_metadata = artist_metadata # Display artist confirmation st.success(f"✅ {artist_metadata['name']}") if artist_metadata.get('images') and len(artist_metadata['images']) > 0: st.image(artist_metadata['images'][0]['url'], width=120) except Exception as e: with st.expander("❌ Error fetching artist"): st.error(str(e)) st.session_state.target_artist_metadata = None else: st.session_state.target_artist_id = None st.session_state.target_artist_metadata = None st.info("â„šī¸ No artist specified") st.divider() # === BATCH PROCESSING SETTINGS === st.subheader("3ī¸âƒŖ Batch Settings") # Max tokens max_tokens = st.number_input( "Max Fans", min_value=1, max_value=2000, value=50, help="Maximum number of fans to analyze (wave processing enabled for large batches)" ) # Time range selector time_range = create_time_range_selector('_batch') # Advanced Settings for wave processing with st.expander("âš™ī¸ Advanced Settings"): wave_size = st.slider( "Fans per Wave", min_value=5, max_value=20, value=10, help="Number of fans processed in each wave" ) concurrency = st.slider( "Concurrent Workers", min_value=1, max_value=5, value=3, help="Parallel API requests within each wave" ) inter_wave_delay = st.slider( "Inter-Wave Delay (ms)", min_value=200, max_value=1000, value=500, step=100, help="Pause between waves for rate limit safety" ) # Consolidated button: Fetch & Process if st.button("🚀 Start Analysis", type="primary", use_container_width=True): if not album_id: st.error("❌ Please enter an album ID") st.stop() # Save album ID st.session_state.album_id = album_id # Clear sidebar, move processing to main panel st.session_state.processing_started = True st.session_state.batch_processed = False # Reset processing flag st.session_state.processing_config = { 'album_id': album_id, 'presave_version': presave_version, 'max_tokens': max_tokens, 'time_range': time_range, 'client_id': None, 'client_secret': None, 'wave_size': wave_size, 'concurrency': concurrency, 'inter_wave_delay': inter_wave_delay } # Force immediate rerun to show progress st.rerun() # Main content area - mutually exclusive states # Priority: Processing > Results > Welcome # STATE 1: Processing in progress - show progress UI at top if st.session_state.get('processing_started', False) and not st.session_state.get('batch_processed', False): config = st.session_state.processing_config album_id = config['album_id'] presave_version = config.get('presave_version', 'V1') max_tokens = config['max_tokens'] time_range = config['time_range'] # ===== PROGRESS UI AT TOP - ALWAYS VISIBLE ===== st.divider() st.subheader("⚡ Processing in Progress", divider="green") # Progress bar and status - created FIRST so they appear at top progress_bar = st.progress(0) status_text = st.empty() # Show initial status status_text.info(f"🚀 Starting Analysis... Preparing to process {max_tokens} fans") # Phase status placeholder (updates as we progress) phase_status = st.empty() # ==================== PHASE 1: Fetch Tokens ==================== phase_status.info("đŸ“Ĩ **Phase 1: Fetching Tokens** - Retrieving fan data from DynamoDB...") # Load AWS credentials with st.spinner("Loading AWS credentials..."): access_key, secret_key, region, session_token = load_aws_credentials() if not access_key or not secret_key: st.error("❌ Cannot proceed without AWS credentials") st.info("Add AWS credentials to secrets.toml and restart the app") st.session_state.processing_started = False st.stop() # Create DynamoDB client with st.spinner("Connecting to DynamoDB..."): dynamodb_client = create_dynamodb_client( access_key, secret_key, region, session_token ) if not dynamodb_client: st.error("❌ Failed to create DynamoDB client") st.session_state.processing_started = False st.stop() # Fetch tokens try: with st.spinner(f"Fetching tokens for campaign {album_id}..."): tokens, raw_items, error = fetch_refresh_tokens( dynamodb_client, album_id, max_tokens, presave_version=presave_version ) if error: st.error(f"❌ Error fetching tokens: {error}") st.session_state.processing_started = False st.stop() elif not tokens: st.warning(f"âš ī¸ No tokens found for campaign {album_id}") st.session_state.processing_started = False st.stop() st.success(f"✅ Fetched {len(tokens)} tokens for campaign {album_id}") except Exception as e: st.error(f"❌ Error fetching tokens: {str(e)}") st.session_state.processing_started = False st.stop() # ==================== PHASE 2: Process Batch ==================== phase_status.info("đŸŽĩ **Phase 2: Processing Fans** - Analyzing listening behavior from Spotify...") # Get Spotify credentials client_id, client_secret = get_spotify_credentials() if not client_id or not client_secret: st.error("❌ Cannot proceed without Spotify credentials") st.session_state.processing_started = False st.stop() # Wave processing configuration # Use WaveConfig defaults if not specified in config wave_config = WaveConfig() if 'wave_size' in config: wave_config.wave_size = config['wave_size'] if 'concurrency' in config: wave_config.concurrency = config['concurrency'] if 'inter_wave_delay' in config: wave_config.inter_wave_delay_ms = config['inter_wave_delay'] total_tokens = len(tokens) total_waves = wave_config.calculate_wave_count(total_tokens) # Wave status display wave_status = st.empty() wave_status.info(f"🌊 Wave Processing: {total_waves} waves, {wave_config.wave_size} fans/wave, {wave_config.concurrency} workers") # Progress callback - updates pre-created UI elements def update_progress(current_fan: int, total_fans: int, current_wave: int, total_waves: int): progress_percent = current_fan / total_fans progress_bar.progress(progress_percent) status_text.info( f"đŸŽĩ Processing Fan {current_fan} of {total_fans} " f"(Wave {current_wave}/{total_waves}, {int(progress_percent * 100)}% Complete)" ) # Create wave processor processor = WaveProcessor( config=wave_config, client_id=client_id, client_secret=client_secret, time_range=time_range, target_artist_id=st.session_state.target_artist_id ) # Process all tokens in waves wave_result = processor.process_all(tokens, progress_callback=update_progress) results = wave_result.results # Complete progress with success message progress_bar.progress(1.0) status_text.success( f"✅ Analysis Complete! Processed {total_tokens} fans in {wave_result.total_time:.1f} seconds " f"({wave_result.wave_count} waves)" ) # Show wave error summary if any errors occurred if wave_result.errors_by_wave: with st.expander("âš ī¸ Wave Error Summary"): for wave_num, error_count in sorted(wave_result.errors_by_wave.items()): st.warning(f"Wave {wave_num}: {error_count} error(s)") # Store results in session state st.session_state.batch_results = results st.session_state.batch_processed = True st.session_state.processed_time_range = time_range st.session_state.processing_started = False # Clear processing flag # Brief pause before showing results time.sleep(1) st.rerun() # STATE 2: Results ready - show results elif st.session_state.batch_processed and st.session_state.batch_results: results = st.session_state.batch_results st.divider() render_section_header("Results Overview", "📊") # Aggregate metrics total_fans = len(results) successful_fans = len([r for r in results if r['success']]) failed_fans = len([r for r in results if not r['success']]) success_rate = (successful_fans / total_fans * 100) if total_fans > 0 else 0 # Heavy Rotation metrics (if artist matching enabled) if st.session_state.target_artist_id and st.session_state.target_artist_metadata: heavy_rotation_fans = [r for r in results if r.get('artist_match') is True] heavy_rotation_count = len(heavy_rotation_fans) heavy_rotation_pct = (heavy_rotation_count / successful_fans * 100) if successful_fans > 0 else 0 # Display target artist info render_artist_info(st.session_state.target_artist_metadata) # Metrics row col1, col2, col3, col4, col5 = st.columns(5) with col1: render_metric_card("Total Fans", total_fans, icon="đŸ‘Ĩ") with col2: render_metric_card( "Heavy Rotation", heavy_rotation_count, delta=f"{heavy_rotation_pct:.1f}%", icon="đŸ”Ĩ" ) with col3: render_metric_card("Success Rate", f"{success_rate:.1f}%", icon="✅") with col4: render_metric_card("Failed", failed_fans, icon="❌") with col5: if heavy_rotation_fans: avg_position = sum(r['artist_position'] for r in heavy_rotation_fans) / len(heavy_rotation_fans) render_metric_card("Avg Position", f"#{avg_position:.1f}", icon="📍") else: render_metric_card("Avg Position", "N/A", icon="📍") else: # No artist matching - show standard metrics col1, col2, col3, col4 = st.columns(4) with col1: render_metric_card("Total Fans", total_fans, icon="đŸ‘Ĩ") with col2: render_metric_card("Successful", successful_fans, delta=f"{success_rate:.1f}%", icon="✅") with col3: render_metric_card("Failed", failed_fans, icon="❌") with col4: total_artists = sum(r['artists_count'] for r in results if r['success']) avg_artists = total_artists / successful_fans if successful_fans > 0 else 0 render_metric_card("Avg Artists/Fan", f"{avg_artists:.1f}", icon="đŸŽĩ") # Detailed results table st.divider() render_section_header("Detailed Results", "📋") # Prepare data for table table_data = [] for result in results: row = { 'Fan #': result['fan_number'], 'Status': '✅ Success' if result['success'] else '❌ Failed', 'Artists': result['artists_count'], 'Top Artist': result['artists'][0]['name'] if result['success'] and result['artists'] else 'N/A', } # Add artist match columns if enabled if st.session_state.target_artist_id: if result.get('artist_match') is True: row['Heavy Rotation'] = 'đŸ”Ĩ Yes' row['Position'] = f"#{result['artist_position']}" elif result.get('artist_match') is False: row['Heavy Rotation'] = '❌ No' row['Position'] = '-' else: row['Heavy Rotation'] = '-' row['Position'] = '-' row['Error'] = result['error'][:50] + '...' if result['error'] and len(result['error']) > 50 else (result['error'] if result['error'] else '-') table_data.append(row) df = pd.DataFrame(table_data) # Display table with styling st.dataframe( df, use_container_width=True, hide_index=True, height=400 ) # Heavy Rotation fans breakdown if st.session_state.target_artist_id and st.session_state.target_artist_metadata: st.divider() render_section_header("Heavy Rotation Analysis", "đŸ”Ĩ") heavy_rotation_results = [r for r in results if r.get('artist_match') is True] if heavy_rotation_results: col1, col2, col3 = st.columns(3) with col1: top_10 = len([r for r in heavy_rotation_results if r['artist_position'] <= 10]) render_metric_card("Top 10", top_10, icon="đŸĨ‡") with col2: top_25 = len([r for r in heavy_rotation_results if r['artist_position'] <= 25]) render_metric_card("Top 25", top_25, icon="đŸĨˆ") with col3: top_50 = len([r for r in heavy_rotation_results if r['artist_position'] <= 50]) render_metric_card("Top 50", top_50, icon="đŸĨ‰") # Show detailed list with st.expander(f"📜 View all {len(heavy_rotation_results)} Heavy Rotation fans"): for result in heavy_rotation_results: st.write(f"**Fan #{result['fan_number']}** - Position: **#{result['artist_position']}**") if result['artists']: top_3 = result['artists'][:3] st.caption(f"Top 3: {', '.join([a['name'] for a in top_3])}") else: st.warning("âš ī¸ **No Heavy Rotation Fans Found** - None of the analyzed fans have this artist in their top 50.") # Export options st.divider() render_section_header("Export Results", "💾") col1, col2 = st.columns(2) with col1: # Export as CSV csv = df.to_csv(index=False) st.download_button( label="đŸ“Ĩ Download CSV", data=csv, file_name=f"heavy_rotation_album_{st.session_state.album_id}.csv", mime="text/csv", use_container_width=True ) with col2: # Export full data as JSON json_data = json.dumps(results, indent=2) st.download_button( label="đŸ“Ĩ Download JSON", data=json_data, file_name=f"heavy_rotation_full_album_{st.session_state.album_id}.json", mime="application/json", use_container_width=True ) # Error analysis section if failed_fans > 0: st.divider() render_section_header("Error Analysis", "âš ī¸") # Categorize errors error_categories = {} failed_results = [r for r in results if not r['success']] for result in failed_results: error_msg = result['error'] or '' if 'token' in error_msg.lower() or 'auth' in error_msg.lower(): category = 'Authentication/Token Issues' elif 'rate' in error_msg.lower() or 'limit' in error_msg.lower(): category = 'Rate Limiting' elif 'network' in error_msg.lower() or 'connection' in error_msg.lower(): category = 'Network/Connection Issues' elif 'expired' in error_msg.lower(): category = 'Expired Credentials' else: category = 'Other Errors' if category not in error_categories: error_categories[category] = [] error_categories[category].append(result) # Display error categories for category, category_results in error_categories.items(): with st.expander(f"❌ {category} ({len(category_results)} fans)"): fan_numbers = ', '.join([f"#{r['fan_number']}" for r in category_results]) st.write(f"**Affected Fans:** {fan_numbers}") st.code(category_results[0]['error'][:300] + '...' if len(category_results[0]['error']) > 300 else category_results[0]['error']) else: # Welcome screen st.info(""" 👋 **Welcome to Heavy Rotation Analytics** Get started by configuring your analysis settings in the sidebar and clicking "Start Analysis". This tool helps you identify which fans have specific artists in their Spotify Top Artists list, providing valuable insights into listening behavior. """) # Feature highlights col1, col2, col3 = st.columns(3) with col1: with st.container(border=True): st.markdown("### đŸŽ¯ Target Analysis") st.caption("Identify fans with specific artists in their Heavy Rotation") with col2: with st.container(border=True): st.markdown("### ⚡ Batch Processing") st.caption("Analyze up to 200 fans at once with real-time progress tracking") with col3: with st.container(border=True): st.markdown("### 📊 Rich Analytics") st.caption("Detailed insights with position tracking and export options") # Footer st.divider() st.caption("Heavy Rotation POC - Listening Behavior Collector Initiative â€ĸ Powered by Spotify API â€ĸ Snowflake Streamlit Container Runtime")