browser_pinchtab_test.go
1 package tools 2 3 import ( 4 "context" 5 "encoding/json" 6 "net/http" 7 "net/http/httptest" 8 "net/url" 9 "strings" 10 "sync" 11 "testing" 12 ) 13 14 // fakePinchtab returns a test server that mimics pinchtab's HTTP API. 15 func fakePinchtab(t *testing.T) *httptest.Server { 16 t.Helper() 17 mux := http.NewServeMux() 18 19 mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { 20 json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) 21 }) 22 23 mux.HandleFunc("/navigate", func(w http.ResponseWriter, r *http.Request) { 24 var req ptNavigateReq 25 json.NewDecoder(r.Body).Decode(&req) 26 json.NewEncoder(w).Encode(ptNavigateResp{ 27 TabID: "tab_test123", 28 URL: req.URL, 29 Title: "Test Page", 30 }) 31 }) 32 33 mux.HandleFunc("/snapshot", func(w http.ResponseWriter, r *http.Request) { 34 filter := r.URL.Query().Get("filter") 35 nodes := []ptSnapshotNode{ 36 {Ref: "e0", Role: "link", Name: "Home", Depth: 0}, 37 {Ref: "e1", Role: "button", Name: "Submit", Depth: 0}, 38 {Ref: "e2", Role: "textbox", Name: "Search", Depth: 0, Value: ""}, 39 } 40 if filter == "interactive" { 41 // same for this mock 42 } 43 json.NewEncoder(w).Encode(ptSnapshotResp{ 44 URL: "https://example.com", 45 Title: "Test Page", 46 Nodes: nodes, 47 Count: len(nodes), 48 }) 49 }) 50 51 mux.HandleFunc("/find", func(w http.ResponseWriter, r *http.Request) { 52 var req ptFindReq 53 json.NewDecoder(r.Body).Decode(&req) 54 json.NewEncoder(w).Encode(ptFindResp{ 55 BestRef: "e1", 56 Confidence: "high", 57 Score: 0.95, 58 Matches: []ptFindMatch{ 59 {Ref: "e1", Score: 0.95, Role: "button", Name: "Submit"}, 60 }, 61 }) 62 }) 63 64 mux.HandleFunc("/action", func(w http.ResponseWriter, r *http.Request) { 65 var req ptActionReq 66 json.NewDecoder(r.Body).Decode(&req) 67 json.NewEncoder(w).Encode(ptActionResp{ 68 Success: true, 69 Result: map[string]any{"clicked": true, "kind": req.Kind}, 70 }) 71 }) 72 73 mux.HandleFunc("/text", func(w http.ResponseWriter, r *http.Request) { 74 json.NewEncoder(w).Encode(ptTextResp{ 75 URL: "https://example.com", 76 Title: "Test Page", 77 Text: "Hello from the test page.", 78 }) 79 }) 80 81 mux.HandleFunc("/evaluate", func(w http.ResponseWriter, r *http.Request) { 82 json.NewEncoder(w).Encode(ptEvalResp{Result: "Test Page"}) 83 }) 84 85 mux.HandleFunc("/screenshot", func(w http.ResponseWriter, r *http.Request) { 86 // Return a minimal valid JPEG (SOI + EOI markers) 87 w.Header().Set("Content-Type", "image/jpeg") 88 w.Write([]byte{0xFF, 0xD8, 0xFF, 0xD9}) 89 }) 90 91 mux.HandleFunc("/shutdown", func(w http.ResponseWriter, r *http.Request) { 92 json.NewEncoder(w).Encode(map[string]string{"status": "shutting down"}) 93 }) 94 95 return httptest.NewServer(mux) 96 } 97 98 // newToolWithFakePinchtab creates a BrowserTool pre-wired to a fake pinchtab server. 99 func newToolWithFakePinchtab(t *testing.T, srv *httptest.Server) *BrowserTool { 100 t.Helper() 101 pt := &pinchtabClient{ 102 base: srv.URL, 103 http: srv.Client(), 104 } 105 return &BrowserTool{ 106 backend: backendPinchtab, 107 pt: pt, 108 } 109 } 110 111 // --- Test 1: snapshot/find on pinchtab path --- 112 113 func TestPinchtab_Snapshot(t *testing.T) { 114 srv := fakePinchtab(t) 115 defer srv.Close() 116 tool := newToolWithFakePinchtab(t, srv) 117 defer tool.Cleanup() 118 119 // Navigate first to get a tabID 120 result, err := tool.Run(context.Background(), `{"action":"navigate","url":"https://example.com"}`) 121 if err != nil { 122 t.Fatalf("navigate error: %v", err) 123 } 124 if result.IsError { 125 t.Fatalf("navigate failed: %s", result.Content) 126 } 127 if !contains(result.Content, "Test Page") { 128 t.Errorf("expected title in output, got: %s", result.Content) 129 } 130 131 // Snapshot 132 result, err = tool.Run(context.Background(), `{"action":"snapshot","filter":"interactive"}`) 133 if err != nil { 134 t.Fatalf("snapshot error: %v", err) 135 } 136 if result.IsError { 137 t.Fatalf("snapshot failed: %s", result.Content) 138 } 139 140 // Should contain element refs 141 if !contains(result.Content, "[e0]") { 142 t.Errorf("expected ref [e0] in snapshot, got: %s", result.Content) 143 } 144 if !contains(result.Content, "[e1]") { 145 t.Errorf("expected ref [e1] in snapshot, got: %s", result.Content) 146 } 147 if !contains(result.Content, "button") { 148 t.Errorf("expected role 'button' in snapshot, got: %s", result.Content) 149 } 150 if !contains(result.Content, "Elements: 3") { 151 t.Errorf("expected element count in snapshot, got: %s", result.Content) 152 } 153 } 154 155 func TestPinchtab_Find(t *testing.T) { 156 srv := fakePinchtab(t) 157 defer srv.Close() 158 tool := newToolWithFakePinchtab(t, srv) 159 defer tool.Cleanup() 160 161 result, err := tool.Run(context.Background(), `{"action":"find","query":"submit button"}`) 162 if err != nil { 163 t.Fatalf("find error: %v", err) 164 } 165 if result.IsError { 166 t.Fatalf("find failed: %s", result.Content) 167 } 168 if !contains(result.Content, "e1") { 169 t.Errorf("expected best ref e1, got: %s", result.Content) 170 } 171 if !contains(result.Content, "high") { 172 t.Errorf("expected confidence 'high', got: %s", result.Content) 173 } 174 } 175 176 func TestPinchtab_ClickByRef(t *testing.T) { 177 srv := fakePinchtab(t) 178 defer srv.Close() 179 tool := newToolWithFakePinchtab(t, srv) 180 defer tool.Cleanup() 181 182 result, err := tool.Run(context.Background(), `{"action":"click","ref":"e1"}`) 183 if err != nil { 184 t.Fatalf("click error: %v", err) 185 } 186 if result.IsError { 187 t.Fatalf("click failed: %s", result.Content) 188 } 189 if !contains(result.Content, "Clicked: e1") { 190 t.Errorf("expected 'Clicked: e1', got: %s", result.Content) 191 } 192 } 193 194 func TestPinchtab_ClickWithKey(t *testing.T) { 195 srv := fakePinchtab(t) 196 defer srv.Close() 197 tool := newToolWithFakePinchtab(t, srv) 198 defer tool.Cleanup() 199 200 // click with key should dispatch as "press" kind 201 result, err := tool.Run(context.Background(), `{"action":"click","ref":"e2","key":"Enter"}`) 202 if err != nil { 203 t.Fatalf("click+key error: %v", err) 204 } 205 if result.IsError { 206 t.Fatalf("click+key failed: %s", result.Content) 207 } 208 } 209 210 func TestPinchtab_ClickWithValue(t *testing.T) { 211 srv := fakePinchtab(t) 212 defer srv.Close() 213 tool := newToolWithFakePinchtab(t, srv) 214 defer tool.Cleanup() 215 216 // click with value should dispatch as "select" kind 217 result, err := tool.Run(context.Background(), `{"action":"click","ref":"e2","value":"option1"}`) 218 if err != nil { 219 t.Fatalf("click+value error: %v", err) 220 } 221 if result.IsError { 222 t.Fatalf("click+value failed: %s", result.Content) 223 } 224 } 225 226 func TestPinchtab_ScreenshotFeedsVision(t *testing.T) { 227 srv := fakePinchtab(t) 228 defer srv.Close() 229 tool := newToolWithFakePinchtab(t, srv) 230 defer tool.Cleanup() 231 232 result, err := tool.Run(context.Background(), `{"action":"screenshot"}`) 233 if err != nil { 234 t.Fatalf("screenshot error: %v", err) 235 } 236 if result.IsError { 237 t.Fatalf("screenshot failed: %s", result.Content) 238 } 239 if len(result.Images) == 0 { 240 t.Error("expected screenshot to populate Images for vision loop") 241 } 242 if result.Images[0].MediaType != "image/jpeg" { 243 t.Errorf("expected image/jpeg, got: %s", result.Images[0].MediaType) 244 } 245 } 246 247 // --- Test 2: fallback-to-chromedp transition --- 248 249 func TestPinchtab_SnapshotFallbackError(t *testing.T) { 250 // Call snapshotAction directly on a chromedp-backend tool to bypass ensureBackend 251 // (which would auto-start real pinchtab if installed). 252 tool := &BrowserTool{backend: backendChromedp} 253 254 result, err := tool.snapshotAction(context.Background(), browserArgs{Action: "snapshot"}) 255 if err != nil { 256 t.Fatalf("unexpected error: %v", err) 257 } 258 if !result.IsError { 259 t.Error("expected error for snapshot on chromedp fallback") 260 } 261 if !contains(result.Content, "pinchtab") { 262 t.Errorf("expected pinchtab message, got: %s", result.Content) 263 } 264 } 265 266 func TestPinchtab_FindFallbackError(t *testing.T) { 267 // Call findAction directly to bypass ensureBackend. 268 tool := &BrowserTool{backend: backendChromedp} 269 270 result, err := tool.findAction(context.Background(), browserArgs{Action: "find", Query: "search"}) 271 if err != nil { 272 t.Fatalf("unexpected error: %v", err) 273 } 274 if !result.IsError { 275 t.Error("expected error for find on chromedp fallback") 276 } 277 if !contains(result.Content, "pinchtab") { 278 t.Errorf("expected pinchtab message, got: %s", result.Content) 279 } 280 } 281 282 func TestPinchtab_FallbackTransition_ClearsTabID(t *testing.T) { 283 // Simulate: pinchtab was running with a tabID, then goes unhealthy. 284 // After detecting unhealthy, tabID should be cleared. 285 srv := fakePinchtab(t) 286 tool := newToolWithFakePinchtab(t, srv) 287 tool.tabID = "tab_old_stale" 288 289 // Kill the fake server to simulate pinchtab dying 290 srv.Close() 291 292 // Directly test the ensureBackend transition logic without triggering 293 // real pinchtab auto-start. Lock and simulate what ensureBackend does: 294 tool.mu.Lock() 295 ctx := context.Background() 296 if !tool.pt.available(ctx) { 297 tool.tabID = "" 298 tool.backend = backendNone 299 } 300 tool.mu.Unlock() 301 302 if tool.tabID != "" { 303 t.Errorf("expected tabID to be cleared after pinchtab failure, got: %q", tool.tabID) 304 } 305 if tool.backend != backendNone { 306 t.Errorf("expected backendNone, got: %d", tool.backend) 307 } 308 } 309 310 // --- Test 3: close after pinchtab terminates mid-run --- 311 312 func TestPinchtab_CloseAfterServerDies(t *testing.T) { 313 srv := fakePinchtab(t) 314 tool := newToolWithFakePinchtab(t, srv) 315 tool.tabID = "tab_test123" 316 317 // Simulate pinchtab dying mid-run 318 srv.Close() 319 320 // close should not panic, should report success 321 result, err := tool.Run(context.Background(), `{"action":"close"}`) 322 if err != nil { 323 t.Fatalf("close error: %v", err) 324 } 325 if result.IsError { 326 t.Errorf("expected clean close, got error: %s", result.Content) 327 } 328 329 // Backend should be reset 330 if tool.backend != backendNone { 331 t.Errorf("expected backendNone after close, got: %d", tool.backend) 332 } 333 if tool.tabID != "" { 334 t.Errorf("expected tabID cleared after close, got: %q", tool.tabID) 335 } 336 } 337 338 func TestPinchtab_CloseWhenPinchtabNeverStarted(t *testing.T) { 339 // BrowserTool with pinchtab client that was never connected 340 tool := &BrowserTool{ 341 pt: newPinchtabClient(), 342 } 343 344 result, err := tool.Run(context.Background(), `{"action":"close"}`) 345 if err != nil { 346 t.Fatalf("close error: %v", err) 347 } 348 if result.IsError { 349 t.Errorf("expected clean close, got error: %s", result.Content) 350 } 351 if !contains(result.Content, "not running") { 352 t.Errorf("expected 'not running', got: %s", result.Content) 353 } 354 } 355 356 // --- Test: new params in Info --- 357 358 func TestBrowser_InfoNewParams(t *testing.T) { 359 tool := &BrowserTool{} 360 info := tool.Info() 361 props := info.Parameters["properties"].(map[string]any) 362 363 // snapshot/find are pinchtab-only actions; pinchtab is legacy and the 364 // chromedp fallback can't honor them. They've been removed from the 365 // advertised schema so the model doesn't waste a call discovering the 366 // runtime failure. The corresponding `query` and `filter` params go with 367 // them. See fix C. 368 newParams := []string{"ref", "key", "value", "waitFor", "waitSelector", "blockImages", "blockAds", "textMode", "maxChars", "raw"} 369 for _, p := range newParams { 370 if _, exists := props[p]; !exists { 371 t.Errorf("expected parameter %q in properties", p) 372 } 373 } 374 for _, p := range []string{"query", "filter"} { 375 if _, exists := props[p]; exists { 376 t.Errorf("parameter %q should have been removed from schema", p) 377 } 378 } 379 // snapshot/find must not appear in the action list (the tool's primary 380 // instructions to the model). They may still appear in an explanatory 381 // note telling the model why those actions are unavailable — that note 382 // is a feature, not drift. 383 const actionListPrefix = "Actions: " 384 start := strings.Index(info.Description, actionListPrefix) 385 if start < 0 { 386 t.Fatalf("expected %q in description, got %q", actionListPrefix, info.Description) 387 } 388 tail := info.Description[start+len(actionListPrefix):] 389 end := strings.Index(tail, ". ") 390 if end < 0 { 391 end = len(tail) 392 } 393 actionList := tail[:end] 394 for _, kw := range []string{"snapshot", "find"} { 395 if strings.Contains(actionList, kw) { 396 t.Errorf("action list should no longer mention %q, got %q", kw, actionList) 397 } 398 } 399 } 400 401 func TestPinchtab_NavigateParamsForwarded(t *testing.T) { 402 mux := http.NewServeMux() 403 var mu sync.Mutex 404 var got ptNavigateReq 405 mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { 406 w.WriteHeader(http.StatusOK) 407 json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) 408 }) 409 mux.HandleFunc("/navigate", func(w http.ResponseWriter, r *http.Request) { 410 var req ptNavigateReq 411 if err := json.NewDecoder(r.Body).Decode(&req); err != nil { 412 t.Fatalf("decode /navigate request: %v", err) 413 } 414 mu.Lock() 415 got = req 416 mu.Unlock() 417 json.NewEncoder(w).Encode(ptNavigateResp{ 418 TabID: "tab_test123", 419 URL: req.URL, 420 Title: "Test Navigate", 421 }) 422 }) 423 mux.HandleFunc("/tabs", func(w http.ResponseWriter, r *http.Request) { 424 w.WriteHeader(http.StatusOK) 425 json.NewEncoder(w).Encode(ptTabsResp{Tabs: []struct { 426 ID string `json:"id"` 427 URL string `json:"url"` 428 Title string `json:"title"` 429 }{}}) 430 }) 431 srv := httptest.NewServer(mux) 432 defer srv.Close() 433 434 tool := newToolWithFakePinchtab(t, srv) 435 defer tool.Cleanup() 436 437 result, err := tool.Run(context.Background(), `{"action":"navigate","url":"https://example.com","blockImages":true,"blockAds":true,"waitFor":"networkidle","waitSelector":"#content"}`) 438 if err != nil { 439 t.Fatalf("navigate error: %v", err) 440 } 441 if result.IsError { 442 t.Fatalf("navigate failed: %s", result.Content) 443 } 444 445 mu.Lock() 446 gotReq := got 447 mu.Unlock() 448 if !gotReq.BlockImages { 449 t.Error("expected navigate request to include blockImages=true") 450 } 451 if !gotReq.BlockAds { 452 t.Error("expected navigate request to include blockAds=true") 453 } 454 if gotReq.WaitFor != "networkidle" { 455 t.Errorf("expected waitFor=networkidle, got=%q", gotReq.WaitFor) 456 } 457 if gotReq.WaitSelector != "#content" { 458 t.Errorf("expected waitSelector=#content, got=%q", gotReq.WaitSelector) 459 } 460 } 461 462 func TestPinchtab_ReadPageParamsForwarded(t *testing.T) { 463 mux := http.NewServeMux() 464 var mu sync.Mutex 465 var textQuery url.Values 466 mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { 467 w.WriteHeader(http.StatusOK) 468 json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) 469 }) 470 mux.HandleFunc("/navigate", func(w http.ResponseWriter, r *http.Request) { 471 json.NewEncoder(w).Encode(ptNavigateResp{ 472 TabID: "tab_test123", 473 URL: "https://example.com", 474 Title: "Test Navigate", 475 }) 476 }) 477 mux.HandleFunc("/text", func(w http.ResponseWriter, r *http.Request) { 478 mu.Lock() 479 textQuery = r.URL.Query() 480 mu.Unlock() 481 json.NewEncoder(w).Encode(ptTextResp{ 482 URL: "https://example.com", 483 Title: "Test Page", 484 Text: "some text", 485 }) 486 }) 487 srv := httptest.NewServer(mux) 488 defer srv.Close() 489 490 tool := newToolWithFakePinchtab(t, srv) 491 defer tool.Cleanup() 492 493 if _, err := tool.Run(context.Background(), `{"action":"navigate","url":"https://example.com"}`); err != nil { 494 t.Fatalf("navigate error: %v", err) 495 } 496 497 result, err := tool.Run(context.Background(), `{"action":"read_page","textMode":"raw","maxChars":123}`) 498 if err != nil { 499 t.Fatalf("read_page error: %v", err) 500 } 501 if result.IsError { 502 t.Fatalf("read_page failed: %s", result.Content) 503 } 504 505 mu.Lock() 506 got := textQuery 507 mu.Unlock() 508 if got.Get("mode") != "raw" { 509 t.Errorf("expected mode=raw, got=%q", got.Get("mode")) 510 } 511 if got.Get("maxChars") != "123" { 512 t.Errorf("expected maxChars=123, got=%q", got.Get("maxChars")) 513 } 514 if got.Get("tabId") != "tab_test123" { 515 t.Errorf("expected tabId=tab_test123, got=%q", got.Get("tabId")) 516 } 517 } 518 519 func TestResolvePinchtabBaseURL(t *testing.T) { 520 t.Run("pinchtab_url_preferred_over_bridge_port", func(t *testing.T) { 521 t.Setenv("PINCHTAB_URL", "127.0.0.1:9999") 522 t.Setenv("BRIDGE_PORT", "8888") 523 got := resolvePinchtabBaseURL() 524 want := "http://127.0.0.1:9999" 525 if got != want { 526 t.Fatalf("expected %q, got %q", want, got) 527 } 528 }) 529 530 t.Run("pinchtab_url_keeps_path_without_scheme", func(t *testing.T) { 531 t.Setenv("PINCHTAB_URL", "127.0.0.1:7777/api/") 532 t.Setenv("BRIDGE_PORT", "") 533 got := resolvePinchtabBaseURL() 534 want := "http://127.0.0.1:7777/api" 535 if got != want { 536 t.Fatalf("expected %q, got %q", want, got) 537 } 538 }) 539 540 t.Run("bridge_port_fallback", func(t *testing.T) { 541 t.Setenv("PINCHTAB_URL", "") 542 t.Setenv("BRIDGE_PORT", "7001") 543 got := resolvePinchtabBaseURL() 544 want := "http://127.0.0.1:7001" 545 if got != want { 546 t.Fatalf("expected %q, got %q", want, got) 547 } 548 }) 549 550 t.Run("default_url_when_unset", func(t *testing.T) { 551 t.Setenv("PINCHTAB_URL", "") 552 t.Setenv("BRIDGE_PORT", "") 553 got := resolvePinchtabBaseURL() 554 want := "http://127.0.0.1:9867" 555 if got != want { 556 t.Fatalf("expected %q, got %q", want, got) 557 } 558 }) 559 }