Overview.py
1 """ 2 333 Method Analytics Dashboard - Overview Page 3 4 Multi-page dashboard with separate pages for detailed analysis. 5 Navigate using the sidebar. 6 """ 7 8 import logging 9 import streamlit as st 10 from streamlit_autorefresh import st_autorefresh 11 from dashboard import config 12 from dashboard.utils import database 13 from dashboard.components import metrics, charts, pipeline_widgets 14 from dashboard.utils.logging_config import configure_app_logging, log_exception 15 16 # Configure logging for this app 17 logger = configure_app_logging("dashboard") 18 19 # Page configuration 20 st.set_page_config(page_title="Overview", page_icon="📊", layout="wide") 21 22 # Auto-refresh (no UI, just functionality) 23 st_autorefresh(interval=config.REFRESH_INTERVAL * 1000, key="data_refresh") # 5 minute default 24 25 # Sidebar - Full Refresh Controls 26 with st.sidebar: 27 st.header("🔄 Dashboard Controls") 28 29 # Cache status 30 cache_stats = database.get_cache_stats() 31 if cache_stats['fresh_entries'] > 0: 32 st.success(f"✅ {cache_stats['fresh_entries']} cached metrics") 33 elif cache_stats['total_entries'] > 0: 34 st.warning(f"⚠️ {cache_stats['expired_entries']} expired cache entries") 35 else: 36 st.info("💤 No cache yet") 37 38 st.caption(f"Cache auto-refreshes every {config.CACHE_TTL // 60} minutes") 39 40 # Full Refresh button 41 st.markdown("---") 42 st.markdown("**Manual Refresh**") 43 44 if st.button("🔄 Full Refresh", type="primary", use_container_width=True, help="Clear cache → Run precompute → Refresh dashboard"): 45 with st.spinner("Clearing cache..."): 46 database.clear_all_cache() 47 st.cache_data.clear() 48 st.cache_resource.clear() 49 50 with st.spinner("Pre-computing fresh data..."): 51 import subprocess 52 import os 53 project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 54 result = subprocess.run( 55 ['node', os.path.join(project_root, 'src/cron/precompute-dashboard.js')], 56 capture_output=True, 57 text=True, 58 cwd=project_root 59 ) 60 61 if result.returncode == 0: 62 st.success("✅ Fresh data computed!") 63 else: 64 st.error(f"❌ Precompute failed: {result.stderr}") 65 66 st.rerun() 67 68 st.caption("⚡ Quick refresh: Just reload the page") 69 st.caption("🔥 Full refresh: Clears cache + recomputes") 70 71 st.markdown("---") 72 st.markdown("**Navigation**") 73 st.caption("Use the sidebar to navigate to detailed pages") 74 75 # Main title 76 st.title("📊 Overview") 77 st.caption("Key metrics and pipeline visualization across the system") 78 79 st.markdown("---") 80 81 # ============================================================================ 82 # OVERVIEW PAGE - Key Metrics Summary 83 # ============================================================================ 84 85 st.header("Key Metrics") 86 87 try: 88 # Get pipeline stats 89 funnel = database.get_pipeline_funnel() 90 total_sites = funnel["count"].sum() if not funnel.empty else 0 91 92 # Get outreach stats 93 response_rates = database.get_response_rates() 94 total_responses = ( 95 response_rates["responses"].sum() if not response_rates.empty else 0 96 ) 97 total_sent = ( 98 response_rates["total_sent"].sum() if not response_rates.empty else 0 99 ) 100 overall_response_rate = ( 101 round(100.0 * total_responses / total_sent, 2) if total_sent > 0 else 0 102 ) 103 104 # Get sales stats 105 sales = database.get_sales_data() 106 total_sales = sales["sales_count"].sum() if not sales.empty else 0 107 total_revenue = sales["total_revenue"].sum() if not sales.empty else 0 108 109 # Get error stats 110 errors = database.get_error_breakdown() 111 total_errors = errors["count"].sum() if not errors.empty else 0 112 113 # Display metrics 114 metrics.display_metric_grid( 115 [ 116 {"label": "Total Sites", "value": f"{total_sites:,}"}, 117 {"label": "Outreach Sent", "value": f"{total_sent:,}"}, 118 {"label": "Response Rate", "value": f"{overall_response_rate}%"}, 119 {"label": "Sales", "value": f"{total_sales} (${total_revenue:,.0f})"}, 120 ] 121 ) 122 123 st.markdown("---") 124 125 # === Site Funnel Breakdown === 126 st.subheader("📂 Site Funnel Breakdown") 127 try: 128 funnel_breakdown = database.get_funnel_breakdown() 129 if funnel_breakdown['total'] > 0: 130 col_tree, col_summary = st.columns([3, 1]) 131 132 with col_tree: 133 fig_tree = pipeline_widgets.create_funnel_tree(funnel_breakdown) 134 st.plotly_chart(fig_tree, use_container_width=True) 135 136 with col_summary: 137 st.markdown("#### By Category") 138 total = funnel_breakdown['total'] 139 140 ignored = funnel_breakdown['ignored'] 141 st.metric("🚫 Ignored", f"{ignored:,}", delta=f"{round(100*ignored/total,1)}% of total", delta_color="off") 142 143 failing = funnel_breakdown['failing'] 144 st.metric("❌ Failing", f"{failing:,}", delta=f"{round(100*failing/total,1)}% of total", delta_color="off") 145 146 active = funnel_breakdown['active'] 147 st.metric("✅ Active", f"{active:,}", delta=f"{round(100*active/total,1)}% of total", delta_color="off") 148 149 if funnel_breakdown['outreach_breakdown']: 150 outreach_total = sum(r['count'] for r in funnel_breakdown['outreach_breakdown']) 151 gdpr_blocked = funnel_breakdown.get('outreach_gdpr_blocked', 0) 152 st.markdown("---") 153 st.markdown("#### Outreaches") 154 st.metric("📤 Pending", f"{outreach_total:,}") 155 for row in funnel_breakdown['outreach_breakdown']: 156 ch_pct = round(100 * row['count'] / outreach_total, 0) if outreach_total else 0 157 st.caption(f"**{row['channel'].upper()}**: {row['count']:,} ({ch_pct:.0f}%)") 158 if gdpr_blocked > 0: 159 st.caption(f"🔒 GDPR blocked: {gdpr_blocked:,}") 160 else: 161 st.info("No site data available") 162 except Exception as e: 163 st.error(f"Error loading funnel breakdown: {e}") 164 165 st.markdown("---") 166 167 # === Response Rate === 168 st.subheader("Response Rate by Channel") 169 if not response_rates.empty: 170 fig = charts.create_bar_chart( 171 response_rates, "channel", "response_rate", "Percentage by Outreach Channel" 172 ) 173 st.plotly_chart(fig, use_container_width=True) 174 else: 175 st.info("No outreach data available") 176 177 st.markdown("---") 178 179 # === 24h Activity === 180 st.subheader("📊 Recent Activity (Last 24 Hours)") 181 182 try: 183 new_records = database.get_new_records_24h() 184 185 if not new_records.empty: 186 # Calculate total and breakdown by table 187 total_activity = new_records["count"].sum() 188 189 # Create activity cards 190 activity_cols = st.columns(4) 191 192 # Total activity 193 with activity_cols[0]: 194 st.metric("🔥 Total Activity", f"{total_activity:,}") 195 196 # Sites activity 197 sites_data = new_records[new_records["table_name"] == "sites"] 198 sites_count = sites_data["count"].sum() if not sites_data.empty else 0 199 with activity_cols[1]: 200 st.metric("🌐 New Sites", f"{sites_count:,}") 201 202 # Outreach activity 203 outreach_data = new_records[new_records["table_name"] == "outreaches"] 204 outreach_count = outreach_data["count"].sum() if not outreach_data.empty else 0 205 with activity_cols[2]: 206 st.metric("📤 Outreaches", f"{outreach_count:,}") 207 208 # Conversation activity 209 conv_data = new_records[new_records["table_name"] == "conversations"] 210 conv_count = conv_data["count"].sum() if not conv_data.empty else 0 211 with activity_cols[3]: 212 st.metric("💬 Conversations", f"{conv_count:,}") 213 214 # Horizontal stacked bar chart by status 215 st.markdown("") # Spacing 216 217 # Prepare data for visualization 218 chart_data = new_records.copy() 219 chart_data["display_label"] = chart_data["table_name"].str.capitalize() 220 221 # Create grouped bar chart 222 import plotly.express as px 223 fig_activity = px.bar( 224 chart_data, 225 x="count", 226 y="display_label", 227 color="status", 228 orientation="h", 229 title="Activity Breakdown by Status", 230 labels={"count": "Count", "display_label": "Table", "status": "Status"}, 231 text="count", 232 height=300 233 ) 234 235 fig_activity.update_traces(textposition="inside", texttemplate="%{text:,}") 236 fig_activity.update_layout( 237 showlegend=True, 238 legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), 239 margin=dict(l=20, r=20, t=60, b=20), 240 xaxis_title="Number of Records", 241 yaxis_title="", 242 bargap=0.2 243 ) 244 245 st.plotly_chart(fig_activity, use_container_width=True) 246 247 else: 248 st.info("💤 No new records created in the last 24 hours") 249 250 except Exception as e: 251 log_exception(logger, f"Error loading 24h activity data: {e}") 252 st.error(f"Error loading activity data: {e}") 253 254 st.markdown("---") 255 256 # === Sankey Flow Diagram === 257 st.subheader("Pipeline Flow") 258 if not funnel.empty: 259 try: 260 fig_sankey = pipeline_widgets.create_sankey_flow(funnel) 261 st.plotly_chart(fig_sankey, use_container_width=True) 262 except Exception as e: 263 st.error(f"Error rendering Sankey diagram: {e}") 264 else: 265 st.info("No pipeline data available") 266 267 st.markdown("---") 268 269 # === Enhanced Funnel with Sidebar === 270 st.subheader("Pipeline Progress") 271 if not funnel.empty: 272 try: 273 fig_funnel, sidebar_metrics = pipeline_widgets.create_funnel_with_sidebar(funnel) 274 275 col_funnel, col_metrics = st.columns([3, 1]) 276 277 with col_funnel: 278 st.plotly_chart(fig_funnel, use_container_width=True) 279 280 with col_metrics: 281 st.markdown("#### Filtered Sites") 282 st.metric("Ignored", f"{sidebar_metrics['ignored']:,}") 283 st.caption(f"{sidebar_metrics['ignored_percent']}% of total") 284 285 st.markdown("#### Active Sites") 286 st.metric("In Pipeline", f"{sidebar_metrics['active']:,}") 287 st.caption(f"{sidebar_metrics['active_percent']}% of total") 288 289 st.markdown("#### Conversion") 290 st.metric("Found → Sent", f"{sidebar_metrics['conversion_rate']}%") 291 st.caption("Pipeline efficiency") 292 293 st.markdown("#### Total") 294 st.metric("All Sites", f"{sidebar_metrics['total']:,}") 295 except Exception as e: 296 st.error(f"Error rendering Enhanced Funnel: {e}") 297 else: 298 st.info("No pipeline data available") 299 300 st.markdown("---") 301 302 # === Error Summary === 303 col3, col4 = st.columns(2) 304 305 with col3: 306 st.subheader("Top Errors") 307 if not errors.empty: 308 # Show top 5 errors 309 top_errors = errors.head(5)[["error_message", "count", "stage"]] 310 st.dataframe(top_errors, use_container_width=True, hide_index=True) 311 else: 312 st.success("No errors found") 313 314 with col4: 315 st.subheader("Sales by Channel") 316 if not sales.empty and sales["sales_count"].sum() > 0: 317 fig = charts.create_pie_chart( 318 sales[sales["sales_count"] > 0], 319 "sales_count", 320 "channel", 321 "Sales Distribution", 322 config.CHANNEL_COLORS, 323 ) 324 st.plotly_chart(fig, use_container_width=True) 325 else: 326 st.info("No sales data available yet") 327 328 # === Quick Stats === 329 st.markdown("---") 330 st.subheader("Quick Stats") 331 332 stats_cols = st.columns(4) 333 334 with stats_cols[0]: 335 stuck_sites = database.get_stuck_sites_by_error() 336 st.metric( 337 "Sites Stuck with Errors", 338 len(stuck_sites) if not stuck_sites.empty else 0, 339 ) 340 341 with stats_cols[1]: 342 conversations = database.get_conversation_stats() 343 unread = ( 344 conversations["unread_count"].iloc[0] if not conversations.empty else 0 345 ) 346 st.metric("Unread Messages", unread) 347 348 with stats_cols[2]: 349 optouts = database.get_optout_stats() 350 total_optouts = ( 351 ( 352 optouts["total_email_optouts"].iloc[0] 353 + optouts["total_sms_optouts"].iloc[0] 354 ) 355 if not optouts.empty 356 else 0 357 ) 358 st.metric("Total Opt-outs", total_optouts) 359 360 with stats_cols[3]: 361 st.metric("Active Errors", total_errors) 362 363 except Exception as e: 364 log_exception(logger, f"Error loading dashboard data: {e}") 365 st.error(f"Error loading dashboard data: {e}") 366 st.info("Please ensure the database is accessible and contains data.") 367 368 369 # Footer 370 st.markdown("---") 371 st.caption("333 Method Analytics Dashboard | Navigate to detailed pages using the sidebar")