test_agent_llm.sh
1 #!/bin/bash 2 3 # Test ECHO Agent LLM Integration 4 # Tests if a specific agent can successfully use its configured LLM model 5 # Usage: ./test_agent_llm.sh <agent_name> 6 7 set -e 8 9 # Colors 10 GREEN='\033[0;32m' 11 YELLOW='\033[1;33m' 12 RED='\033[0;31m' 13 BLUE='\033[0;34m' 14 CYAN='\033[0;36m' 15 NC='\033[0m' # No Color 16 17 # Agent to model mapping (matches shared/lib/echo_shared/llm/config.ex) 18 # Format: agent|model 19 AGENT_MODELS=( 20 "ceo|qwen2.5:14b" 21 "cto|deepseek-coder:33b" 22 "chro|llama3.1:8b" 23 "operations_head|mistral:7b" 24 "product_manager|llama3.1:8b" 25 "senior_architect|deepseek-coder:33b" 26 "uiux_engineer|llama3.2-vision:11b" 27 "senior_developer|deepseek-coder:6.7b" 28 "test_lead|codellama:13b" 29 ) 30 31 # Helper function to get model for agent 32 get_model_for_agent() { 33 local agent_name="$1" 34 for entry in "${AGENT_MODELS[@]}"; do 35 local agent=$(echo "$entry" | cut -d'|' -f1) 36 local model=$(echo "$entry" | cut -d'|' -f2) 37 if [ "$agent" = "$agent_name" ]; then 38 echo "$model" 39 return 0 40 fi 41 done 42 return 1 43 } 44 45 # Display help 46 if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then 47 echo -e "${BLUE}Test ECHO Agent LLM Integration${NC}" 48 echo "================================" 49 echo "" 50 echo "Usage: $0 <agent_name>" 51 echo "" 52 echo "Available agents:" 53 for entry in "${AGENT_MODELS[@]}"; do 54 agent=$(echo "$entry" | cut -d'|' -f1) 55 model=$(echo "$entry" | cut -d'|' -f2) 56 printf " %-20s -> %s\n" "$agent" "$model" 57 done 58 echo "" 59 echo "Example:" 60 echo " $0 senior_architect" 61 echo " $0 ceo" 62 exit 0 63 fi 64 65 AGENT_NAME="$1" 66 67 # Normalize agent name (convert dashes to underscores) 68 AGENT_NAME=$(echo "$AGENT_NAME" | tr '-' '_') 69 70 echo -e "${BLUE}🧪 Testing Agent LLM Integration${NC}" 71 echo "==================================" 72 echo "" 73 echo "Agent: $AGENT_NAME" 74 75 # Check if agent is valid and get model 76 MODEL_NAME=$(get_model_for_agent "$AGENT_NAME") 77 if [ $? -ne 0 ]; then 78 echo -e "${RED}✗ Unknown agent: $AGENT_NAME${NC}" 79 echo "" 80 echo "Available agents:" 81 for entry in "${AGENT_MODELS[@]}"; do 82 echo " - $(echo "$entry" | cut -d'|' -f1)" 83 done 84 exit 1 85 fi 86 echo "Model: $MODEL_NAME" 87 echo "" 88 89 # Step 1: Check Ollama 90 echo -e "${BLUE}Step 1: Checking Ollama${NC}" 91 if ! command -v ollama &> /dev/null; then 92 echo -e "${RED}✗ Ollama is not installed${NC}" 93 exit 1 94 fi 95 echo -e "${GREEN}✓ Ollama is installed${NC}" 96 97 # Step 2: Check service 98 echo -e "${BLUE}Step 2: Checking Ollama service${NC}" 99 if ! pgrep -x "ollama" > /dev/null && ! curl -s http://localhost:11434/api/tags > /dev/null 2>&1; then 100 echo -e "${YELLOW}⚠ Ollama service is not running${NC}" 101 echo "Starting Ollama..." 102 ollama serve > /dev/null 2>&1 & 103 sleep 3 104 echo -e "${GREEN}✓ Ollama started${NC}" 105 else 106 echo -e "${GREEN}✓ Ollama service is running${NC}" 107 fi 108 echo "" 109 110 # Step 3: Check model 111 echo -e "${BLUE}Step 3: Checking model availability${NC}" 112 INSTALLED_MODELS=$(ollama list 2>&1 | tail -n +2 | awk '{print $1}' || echo "") 113 114 if ! echo "$INSTALLED_MODELS" | grep -q "^${MODEL_NAME}$"; then 115 echo -e "${RED}✗ Model $MODEL_NAME is not installed${NC}" 116 echo "" 117 echo "Install with:" 118 echo " ./scripts/ollama/download_model_single.sh $MODEL_NAME" 119 exit 1 120 fi 121 echo -e "${GREEN}✓ Model $MODEL_NAME is installed${NC}" 122 echo "" 123 124 # Step 4: Check agent files 125 echo -e "${BLUE}Step 4: Checking agent files${NC}" 126 127 # Find project root (where apps/ directory is) 128 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 129 PROJECT_ROOT="$( cd "$SCRIPT_DIR/../.." && pwd )" 130 AGENT_DIR="$PROJECT_ROOT/apps/$AGENT_NAME" 131 132 if [ ! -d "$AGENT_DIR" ]; then 133 echo -e "${RED}✗ Agent directory not found: $AGENT_DIR${NC}" 134 exit 1 135 fi 136 echo -e "${GREEN}✓ Agent directory exists${NC}" 137 138 AGENT_FILE="$AGENT_DIR/lib/$AGENT_NAME.ex" 139 if [ ! -f "$AGENT_FILE" ]; then 140 echo -e "${RED}✗ Agent file not found: $AGENT_FILE${NC}" 141 exit 1 142 fi 143 echo -e "${GREEN}✓ Agent file exists${NC}" 144 145 # Check if agent has ai_consult tool 146 if grep -q "ai_consult" "$AGENT_FILE"; then 147 echo -e "${GREEN}✓ Agent has ai_consult tool${NC}" 148 else 149 echo -e "${YELLOW}⚠ Agent may not have ai_consult tool${NC}" 150 fi 151 echo "" 152 153 # Step 5: Test LLM configuration 154 echo -e "${BLUE}Step 5: Testing LLM configuration${NC}" 155 156 # Create a temporary Elixir test script 157 TEST_SCRIPT="$PROJECT_ROOT/shared/test_llm_temp.exs" 158 cat > "$TEST_SCRIPT" << 'EOFTEST' 159 # Get agent role from command line 160 agent_role = System.argv() |> List.first() |> String.to_atom() 161 162 # Simple test that doesn't require full app startup 163 Mix.install([ 164 {:req, "~> 0.5"} 165 ]) 166 167 # Define minimal config and client modules inline 168 defmodule SimpleConfig do 169 @agent_models %{ 170 ceo: "qwen2.5:14b", 171 cto: "deepseek-coder:33b", 172 chro: "llama3.1:8b", 173 operations_head: "mistral:7b", 174 product_manager: "llama3.1:8b", 175 senior_architect: "deepseek-coder:33b", 176 uiux_engineer: "llama3.2-vision:11b", 177 senior_developer: "deepseek-coder:6.7b", 178 test_lead: "codellama:13b" 179 } 180 181 @system_prompts %{ 182 ceo: "You are the CEO of an AI-powered software organization. You focus on strategic decisions, organizational direction, and high-level vision.", 183 cto: "You are the CTO, responsible for technology strategy, architecture decisions, and technical leadership.", 184 chro: "You are the CHRO, managing human resources, team dynamics, and organizational culture.", 185 operations_head: "You are the Operations Head, ensuring infrastructure reliability, scalability, and operational excellence.", 186 product_manager: "You are a Product Manager, defining product vision, prioritizing features, and managing the product roadmap.", 187 senior_architect: "You are a Senior Architect, designing system architecture, defining technical specifications, and ensuring best practices.", 188 uiux_engineer: "You are a UI/UX Engineer, creating user interfaces, ensuring excellent user experience, and implementing designs.", 189 senior_developer: "You are a Senior Developer, implementing features, writing high-quality code, and solving technical challenges.", 190 test_lead: "You are a Test Lead, ensuring quality through comprehensive testing, test automation, and QA processes." 191 } 192 193 def get_model(role), do: Map.get(@agent_models, role, "llama3.1:8b") 194 def get_system_prompt(role), do: Map.get(@system_prompts, role, "You are a helpful AI assistant.") 195 end 196 197 defmodule SimpleClient do 198 def health_check do 199 endpoint = System.get_env("OLLAMA_ENDPOINT", "http://localhost:11434") 200 url = "#{endpoint}/api/tags" 201 202 case Req.get(url, receive_timeout: 5_000) do 203 {:ok, %{status: 200, body: body}} -> 204 models = Map.get(body, "models", []) 205 model_names = Enum.map(models, & &1["name"]) 206 {:ok, model_names} 207 208 {:ok, %{status: status}} -> 209 {:error, {:http_error, status}} 210 211 {:error, reason} -> 212 {:error, reason} 213 end 214 rescue 215 e -> {:error, {:exception, e}} 216 end 217 218 def generate(model, prompt, opts \\ %{}) do 219 endpoint = System.get_env("OLLAMA_ENDPOINT", "http://localhost:11434") 220 url = "#{endpoint}/api/generate" 221 222 payload = %{ 223 model: model, 224 prompt: prompt, 225 stream: false, 226 options: %{ 227 temperature: opts[:temperature] || 0.7, 228 num_predict: opts[:max_tokens] || 100 229 } 230 } 231 232 payload = if system = opts[:system] do 233 Map.put(payload, :system, system) 234 else 235 payload 236 end 237 238 case Req.post(url, json: payload, receive_timeout: 120_000) do 239 {:ok, %{status: 200, body: body}} -> 240 response_text = body["response"] || "" 241 {:ok, response_text} 242 243 {:ok, %{status: status, body: body}} -> 244 {:error, {:api_error, status, body}} 245 246 {:error, reason} -> 247 {:error, reason} 248 end 249 rescue 250 e -> {:error, {:exception, e}} 251 end 252 end 253 254 # Run the test 255 model = SimpleConfig.get_model(agent_role) 256 IO.puts("Configured model: #{model}") 257 258 prompt = SimpleConfig.get_system_prompt(agent_role) 259 IO.puts("System prompt length: #{String.length(prompt)} chars") 260 261 # Test health check 262 case SimpleClient.health_check() do 263 {:ok, models} -> 264 IO.puts("Ollama responding: #{length(models)} models available") 265 266 {:error, reason} -> 267 IO.puts("Ollama error: #{inspect(reason)}") 268 System.halt(1) 269 end 270 271 # Quick test generation 272 IO.puts("\nTesting generation...") 273 test_prompt = "In one sentence, what is your role?" 274 275 case SimpleClient.generate(model, test_prompt, %{system: prompt, max_tokens: 100, temperature: 0.7}) do 276 {:ok, response} -> 277 IO.puts("✓ Generation successful") 278 IO.puts("Response: #{String.trim(response)}") 279 280 {:error, reason} -> 281 IO.puts("✗ Generation failed: #{inspect(reason)}") 282 System.halt(1) 283 end 284 EOFTEST 285 286 echo "" 287 echo -e "${CYAN}LLM Test Output:${NC}" 288 echo "---" 289 290 if elixir "$TEST_SCRIPT" "$AGENT_NAME" 2>&1; then 291 echo "---" 292 echo "" 293 echo -e "${GREEN}✓ LLM integration test passed!${NC}" 294 rm -f "$TEST_SCRIPT" 2>/dev/null || true 295 else 296 echo "---" 297 echo "" 298 echo -e "${RED}✗ LLM integration test failed${NC}" 299 rm -f "$TEST_SCRIPT" 2>/dev/null || true 300 exit 1 301 fi 302 303 # Step 6: Summary 304 echo "" 305 echo -e "${BLUE}Summary:${NC}" 306 echo "--------" 307 echo -e "${GREEN}✓ Ollama installed and running${NC}" 308 echo -e "${GREEN}✓ Model $MODEL_NAME is available${NC}" 309 echo -e "${GREEN}✓ Agent $AGENT_NAME has LLM integration${NC}" 310 echo -e "${GREEN}✓ Configuration is correct${NC}" 311 echo -e "${GREEN}✓ Can generate responses${NC}" 312 echo "" 313 echo -e "${GREEN}🎉 Agent LLM integration is working!${NC}" 314 echo "" 315 echo "Next steps:" 316 echo " - Build agent: cd $PROJECT_ROOT/apps/$AGENT_NAME && mix escript.build" 317 echo " - Test interactively: ollama run $MODEL_NAME" 318 echo " - Connect to Claude Desktop as MCP server"