hybrid_search.py
1 """ 2 Hybrid Search: Combining Dense and Sparse Retrieval 3 4 This example demonstrates hybrid retrieval that combines semantic (dense) 5 search with keyword (sparse) search for improved accuracy. 6 7 RAG Concept: Dense embeddings capture meaning but may miss exact matches. 8 Sparse/keyword search finds exact terms but lacks semantic understanding. 9 Hybrid search combines both for best results. 10 """ 11 12 from praisonaiagents import Agent 13 14 # Sample knowledge base: Technical troubleshooting guides 15 TROUBLESHOOTING_GUIDES = [ 16 { 17 "id": "error_001", 18 "content": """ 19 Error Code: ERR_CONNECTION_REFUSED 20 21 This error occurs when the client cannot establish a connection to the server. 22 23 Common Causes: 24 - Server is not running or has crashed 25 - Firewall blocking the connection port 26 - Incorrect host or port configuration 27 - Network connectivity issues 28 29 Resolution Steps: 30 1. Verify the server process is running: `systemctl status myservice` 31 2. Check firewall rules: `sudo ufw status` 32 3. Test port connectivity: `telnet hostname 8080` 33 4. Review server logs for startup errors 34 """ 35 }, 36 { 37 "id": "error_002", 38 "content": """ 39 Error Code: ERR_OUT_OF_MEMORY 40 41 This error indicates the application has exhausted available memory. 42 43 Common Causes: 44 - Memory leak in application code 45 - Insufficient heap size configuration 46 - Too many concurrent connections 47 - Large data processing without streaming 48 49 Resolution Steps: 50 1. Increase heap size: `-Xmx4g` for Java applications 51 2. Enable memory profiling to identify leaks 52 3. Implement pagination for large datasets 53 4. Add memory limits to container configurations 54 """ 55 }, 56 { 57 "id": "error_003", 58 "content": """ 59 Error Code: ERR_SSL_CERTIFICATE_EXPIRED 60 61 This error means the SSL/TLS certificate has passed its validity date. 62 63 Common Causes: 64 - Certificate not renewed before expiration 65 - Automatic renewal process failed 66 - Wrong certificate installed 67 68 Resolution Steps: 69 1. Check certificate expiry: `openssl x509 -enddate -noout -in cert.pem` 70 2. Renew certificate with your CA or Let's Encrypt 71 3. Update certificate files on the server 72 4. Restart the web server to load new certificate 73 """ 74 }, 75 { 76 "id": "error_004", 77 "content": """ 78 Error Code: ERR_DATABASE_DEADLOCK 79 80 A deadlock occurs when two transactions block each other indefinitely. 81 82 Common Causes: 83 - Transactions acquiring locks in different orders 84 - Long-running transactions holding locks 85 - Missing indexes causing table scans with locks 86 87 Resolution Steps: 88 1. Review transaction isolation levels 89 2. Ensure consistent lock ordering across queries 90 3. Add appropriate indexes to reduce lock duration 91 4. Implement retry logic for deadlock exceptions 92 """ 93 }, 94 { 95 "id": "error_005", 96 "content": """ 97 Error Code: ERR_RATE_LIMIT_EXCEEDED 98 99 The API has rejected requests due to exceeding the rate limit. 100 101 Common Causes: 102 - Too many requests in a short time window 103 - Missing request throttling in client code 104 - Retry loops without backoff 105 106 Resolution Steps: 107 1. Implement exponential backoff for retries 108 2. Cache responses to reduce API calls 109 3. Use bulk endpoints where available 110 4. Request rate limit increase if needed 111 """ 112 } 113 ] 114 115 116 def demonstrate_search_types(): 117 """Show the difference between semantic and keyword search concepts.""" 118 119 print("=" * 60) 120 print("SEARCH TYPES COMPARISON") 121 print("=" * 60) 122 123 print(""" 124 š DENSE (Semantic) Search: 125 - Uses vector embeddings to capture meaning 126 - "memory issues" matches "out of memory" and "RAM exhausted" 127 - Good for natural language queries 128 - May miss exact technical terms 129 130 š SPARSE (Keyword) Search: 131 - Uses term frequency and exact matching 132 - "ERR_SSL_CERTIFICATE_EXPIRED" matches exactly 133 - Good for error codes, IDs, technical terms 134 - Misses synonyms and paraphrases 135 136 š HYBRID Search: 137 - Combines both approaches 138 - Weighted scoring: (Ī± Ć dense_score) + ((1-Ī±) Ć sparse_score) 139 - Best of both worlds for technical documentation 140 """) 141 142 143 def hybrid_rag_example(): 144 """Demonstrate hybrid retrieval with an agent.""" 145 146 # Build context from troubleshooting guides 147 context = "\n\n".join([f"[{g['id']}]\n{g['content']}" for g in TROUBLESHOOTING_GUIDES]) 148 149 # Create agent with context in instructions 150 agent = Agent( 151 name="Tech Support", 152 instructions=f"""You are a technical support specialist. 153 Help users troubleshoot errors using the knowledge base. 154 Always mention the specific error code when relevant. 155 Provide step-by-step resolution guidance. 156 157 TROUBLESHOOTING KNOWLEDGE BASE: 158 {context}""", 159 output="silent" 160 ) 161 162 # Test with different query types 163 queries = [ 164 # Semantic query (natural language) 165 "My application is running out of memory, what should I do?", 166 167 # Exact match query (error code) 168 "ERR_SSL_CERTIFICATE_EXPIRED", 169 170 # Mixed query 171 "Getting connection refused error when connecting to port 8080", 172 173 # Conceptual query 174 "How do I handle API throttling?" 175 ] 176 177 print("\n" + "=" * 60) 178 print("HYBRID RAG IN ACTION") 179 print("=" * 60) 180 181 for query in queries: 182 print(f"\nš Query: {query}") 183 response = agent.chat(query) 184 print(f"š” Answer: {response[:300]}..." if len(str(response)) > 300 else f"š” Answer: {response}") 185 print("-" * 40) 186 187 188 def compare_retrieval_modes(): 189 """Compare results with different retrieval configurations.""" 190 191 print("\n" + "=" * 60) 192 print("RETRIEVAL MODE COMPARISON") 193 print("=" * 60) 194 195 # Build context 196 context = "\n\n".join([f"[{g['id']}]\n{g['content']}" for g in TROUBLESHOOTING_GUIDES]) 197 198 query = "database transaction blocking issue" 199 200 # Agent with context 201 agent = Agent( 202 name="Search Agent", 203 instructions=f"""Answer based on the knowledge provided. 204 205 KNOWLEDGE BASE: 206 {context}""", 207 output="silent" 208 ) 209 210 print(f"\nš Query: {query}") 211 212 print("\nš RAG Response:") 213 response = agent.chat(query) 214 print(f" {response[:200]}...") 215 216 print("\nš” Note: Hybrid search combines semantic + keyword matching for best results.") 217 218 219 def hybrid_search_tuning(): 220 """Explain hybrid search tuning parameters.""" 221 222 print("\n" + "=" * 60) 223 print("HYBRID SEARCH TUNING") 224 print("=" * 60) 225 226 print(""" 227 Key parameters for hybrid search optimization: 228 229 1. **Alpha (Ī±) Weight** - Balance between dense and sparse 230 - Ī± = 1.0: Pure semantic search 231 - Ī± = 0.5: Equal weight (good default) 232 - Ī± = 0.0: Pure keyword search 233 234 2. **Top-K Retrieval** - Number of candidates 235 - Higher K: More context, slower, may dilute relevance 236 - Lower K: Faster, focused, may miss relevant docs 237 - Typical range: 3-10 for RAG 238 239 3. **Reranking** - Second-pass scoring 240 - Cross-encoder reranking improves precision 241 - Adds latency but improves quality 242 243 Example configuration: 244 ```python 245 knowledge_config = { 246 "hybrid": True, 247 "hybrid_alpha": 0.6, # Favor semantic 248 "top_k": 5, 249 "rerank": True 250 } 251 ``` 252 """) 253 254 255 def main(): 256 """Run all hybrid search examples.""" 257 print("\nš PraisonAI Hybrid Search Examples\n") 258 259 # Example 1: Explain search types 260 demonstrate_search_types() 261 262 # Example 2: Hybrid RAG in action 263 hybrid_rag_example() 264 265 # Example 3: Compare retrieval modes 266 compare_retrieval_modes() 267 268 # Example 4: Tuning guidance 269 hybrid_search_tuning() 270 271 print("\nā Hybrid search examples completed!") 272 273 274 if __name__ == "__main__": 275 main()