1  /**
  2   * CDP client — implements IPage by connecting directly to a Chrome/Electron CDP WebSocket.
  3   *
  4   * Fixes applied:
  5   * - send() now has a 30s timeout guard (P0 #4)
  6   * - goto() waits for Page.loadEventFired instead of hardcoded 1s sleep (P1 #3)
  7   * - Implemented scroll, autoScroll, screenshot, networkRequests (P1 #2)
  8   * - Shared DOM helper methods extracted to reduce duplication with Page (P1 #5)
  9   */
 10  
 11  import { WebSocket, type RawData } from 'ws';
 12  import { request as httpRequest } from 'node:http';
 13  import { request as httpsRequest } from 'node:https';
 14  import type { BrowserCookie, IPage, ScreenshotOptions } from '../types.js';
 15  import type { IBrowserFactory } from '../runtime.js';
 16  import { wrapForEval } from './utils.js';
 17  import { generateStealthJs } from './stealth.js';
 18  import { waitForDomStableJs } from './dom-helpers.js';
 19  import { isRecord, saveBase64ToFile } from '../utils.js';
 20  import { getAllElectronApps } from '../electron-apps.js';
 21  import { BasePage } from './base-page.js';
 22  
 23  export interface CDPTarget {
 24    type?: string;
 25    url?: string;
 26    title?: string;
 27    webSocketDebuggerUrl?: string;
 28  }
 29  
 30  interface RuntimeEvaluateResult {
 31    result?: {
 32      value?: unknown;
 33    };
 34    exceptionDetails?: {
 35      exception?: {
 36        description?: string;
 37      };
 38    };
 39  }
 40  
 41  const CDP_SEND_TIMEOUT = 30_000;
 42  
 43  // Memory guard for in-process capture. The 4k cap we used to apply everywhere
 44  // silently truncated JSON so `JSON.parse` failed or gave partial objects — the
 45  // primary agent-facing bug. Now we keep the full body up to a large cap and
 46  // surface `responseBodyFullSize` + `responseBodyTruncated` so downstream layers
 47  // can tell the agent what happened instead of lying about the payload.
 48  export const CDP_RESPONSE_BODY_CAPTURE_LIMIT = 8 * 1024 * 1024;
 49  
 50  export class CDPBridge implements IBrowserFactory {
 51    private _ws: WebSocket | null = null;
 52    private _idCounter = 0;
 53    private _pending = new Map<number, { resolve: (val: unknown) => void; reject: (err: Error) => void; timer: ReturnType<typeof setTimeout> }>();
 54    private _eventListeners = new Map<string, Set<(params: unknown) => void>>();
 55  
 56    async connect(opts?: { timeout?: number; workspace?: string; cdpEndpoint?: string }): Promise<IPage> {
 57      if (this._ws) throw new Error('CDPBridge is already connected. Call close() before reconnecting.');
 58  
 59      const endpoint = opts?.cdpEndpoint ?? process.env.OPENCLI_CDP_ENDPOINT;
 60      if (!endpoint) throw new Error('CDP endpoint not provided (pass cdpEndpoint or set OPENCLI_CDP_ENDPOINT)');
 61  
 62      let wsUrl = endpoint;
 63      if (endpoint.startsWith('http')) {
 64        const targets = await fetchJsonDirect(`${endpoint.replace(/\/$/, '')}/json`) as CDPTarget[];
 65        const target = selectCDPTarget(targets);
 66        if (!target || !target.webSocketDebuggerUrl) {
 67          throw new Error('No inspectable targets found at CDP endpoint');
 68        }
 69        wsUrl = target.webSocketDebuggerUrl;
 70      }
 71  
 72      return new Promise((resolve, reject) => {
 73        const ws = new WebSocket(wsUrl);
 74        const timeoutMs = (opts?.timeout ?? 10) * 1000;
 75        const timeout = setTimeout(() => {
 76          this._ws = null;
 77          ws.close();
 78          reject(new Error('CDP connect timeout'));
 79        }, timeoutMs);
 80  
 81        ws.on('open', async () => {
 82          clearTimeout(timeout);
 83          this._ws = ws;
 84          try {
 85            await this.send('Page.enable');
 86            await this.send('Page.addScriptToEvaluateOnNewDocument', { source: generateStealthJs() });
 87          } catch (err) {
 88            ws.close();
 89            reject(err instanceof Error ? err : new Error(String(err)));
 90            return;
 91          }
 92          resolve(new CDPPage(this));
 93        });
 94  
 95        ws.on('error', (err: Error) => {
 96          clearTimeout(timeout);
 97          reject(err);
 98        });
 99  
100        ws.on('message', (data: RawData) => {
101          try {
102            const msg = JSON.parse(data.toString());
103            if (msg.id && this._pending.has(msg.id)) {
104              const entry = this._pending.get(msg.id)!;
105              clearTimeout(entry.timer);
106              this._pending.delete(msg.id);
107              if (msg.error) {
108                entry.reject(new Error(msg.error.message));
109              } else {
110                entry.resolve(msg.result);
111              }
112            }
113            if (msg.method) {
114              const listeners = this._eventListeners.get(msg.method);
115              if (listeners) {
116                for (const fn of listeners) fn(msg.params);
117              }
118            }
119          } catch (err) {
120            if (process.env.OPENCLI_VERBOSE) {
121              // eslint-disable-next-line no-console
122              console.error('[cdp] Failed to parse WebSocket message:', err instanceof Error ? err.message : err);
123            }
124          }
125        });
126      });
127    }
128  
129    async close(): Promise<void> {
130      if (this._ws) {
131        this._ws.close();
132        this._ws = null;
133      }
134      for (const p of this._pending.values()) {
135        clearTimeout(p.timer);
136        p.reject(new Error('CDP connection closed'));
137      }
138      this._pending.clear();
139      this._eventListeners.clear();
140    }
141  
142    async send(method: string, params: Record<string, unknown> = {}, timeoutMs: number = CDP_SEND_TIMEOUT): Promise<unknown> {
143      if (!this._ws || this._ws.readyState !== WebSocket.OPEN) {
144        throw new Error('CDP connection is not open');
145      }
146      const id = ++this._idCounter;
147      return new Promise((resolve, reject) => {
148        const timer = setTimeout(() => {
149          this._pending.delete(id);
150          reject(new Error(`CDP command '${method}' timed out after ${timeoutMs / 1000}s`));
151        }, timeoutMs);
152        this._pending.set(id, { resolve, reject, timer });
153        this._ws!.send(JSON.stringify({ id, method, params }));
154      });
155    }
156  
157    on(event: string, handler: (params: unknown) => void): void {
158      let set = this._eventListeners.get(event);
159      if (!set) {
160        set = new Set();
161        this._eventListeners.set(event, set);
162      }
163      set.add(handler);
164    }
165  
166    off(event: string, handler: (params: unknown) => void): void {
167      this._eventListeners.get(event)?.delete(handler);
168    }
169  
170    waitForEvent(event: string, timeoutMs: number = 15_000): Promise<unknown> {
171      return new Promise((resolve, reject) => {
172        const timer = setTimeout(() => {
173          this.off(event, handler);
174          reject(new Error(`Timed out waiting for CDP event '${event}'`));
175        }, timeoutMs);
176        const handler = (params: unknown) => {
177          clearTimeout(timer);
178          this.off(event, handler);
179          resolve(params);
180        };
181        this.on(event, handler);
182      });
183    }
184  }
185  
186  class CDPPage extends BasePage {
187    private _pageEnabled = false;
188  
189    // Network capture state (mirrors extension/src/cdp.ts NetworkCaptureEntry shape)
190    private _networkCapturing = false;
191    private _networkCapturePattern = '';
192    private _networkEntries: Array<{
193      url: string; method: string; responseStatus?: number;
194      responseContentType?: string;
195      responsePreview?: string;
196      responseBodyFullSize?: number;
197      responseBodyTruncated?: boolean;
198      timestamp: number;
199    }> = [];
200    private _pendingRequests = new Map<string, number>(); // requestId → index in _networkEntries
201    private _pendingBodyFetches: Set<Promise<void>> = new Set(); // track in-flight getResponseBody calls
202    private _consoleMessages: Array<{ type: string; text: string; timestamp: number }> = [];
203    private _consoleCapturing = false;
204  
205    constructor(private bridge: CDPBridge) {
206      super();
207    }
208  
209    async goto(url: string, options?: { waitUntil?: 'load' | 'none'; settleMs?: number }): Promise<void> {
210      if (!this._pageEnabled) {
211        await this.bridge.send('Page.enable');
212        this._pageEnabled = true;
213      }
214      const loadPromise = this.bridge.waitForEvent('Page.loadEventFired', 30_000).catch(() => {});
215      await this.bridge.send('Page.navigate', { url });
216      await loadPromise;
217      this._lastUrl = url;
218      if (options?.waitUntil !== 'none') {
219        const maxMs = options?.settleMs ?? 1000;
220        await this.evaluate(waitForDomStableJs(maxMs, Math.min(500, maxMs)));
221      }
222    }
223  
224    async evaluate(js: string): Promise<unknown> {
225      const expression = wrapForEval(js);
226      const result = await this.bridge.send('Runtime.evaluate', {
227        expression,
228        returnByValue: true,
229        awaitPromise: true,
230      }) as RuntimeEvaluateResult;
231      if (result.exceptionDetails) {
232        throw new Error('Evaluate error: ' + (result.exceptionDetails.exception?.description || 'Unknown exception'));
233      }
234      return result.result?.value;
235    }
236  
237    async getCookies(opts: { domain?: string; url?: string } = {}): Promise<BrowserCookie[]> {
238      const result = await this.bridge.send('Network.getCookies', opts.url ? { urls: [opts.url] } : {});
239      const cookies = isRecord(result) && Array.isArray(result.cookies) ? result.cookies : [];
240      const domain = opts.domain;
241      return domain
242        ? cookies.filter((cookie): cookie is BrowserCookie => isCookie(cookie) && matchesCookieDomain(cookie.domain, domain))
243        : cookies;
244    }
245  
246    async screenshot(options: ScreenshotOptions = {}): Promise<string> {
247      const result = await this.bridge.send('Page.captureScreenshot', {
248        format: options.format ?? 'png',
249        quality: options.format === 'jpeg' ? (options.quality ?? 80) : undefined,
250        captureBeyondViewport: options.fullPage ?? false,
251      });
252      const base64 = isRecord(result) && typeof result.data === 'string' ? result.data : '';
253      if (options.path) {
254        await saveBase64ToFile(base64, options.path);
255      }
256      return base64;
257    }
258  
259    async startNetworkCapture(pattern: string = ''): Promise<boolean> {
260      // Always update the filter pattern
261      this._networkCapturePattern = pattern;
262  
263      // Reset state only on first start; avoid wiping entries if already capturing
264      if (!this._networkCapturing) {
265        this._networkEntries = [];
266        this._pendingRequests.clear();
267        this._pendingBodyFetches.clear();
268        await this.bridge.send('Network.enable');
269  
270        // Step 1: Record request method/url on requestWillBeSent
271        this.bridge.on('Network.requestWillBeSent', (params: unknown) => {
272          const p = params as { requestId: string; request: { method: string; url: string }; timestamp: number };
273          if (!this._networkCapturePattern || p.request.url.includes(this._networkCapturePattern)) {
274            const idx = this._networkEntries.push({
275              url: p.request.url,
276              method: p.request.method,
277              timestamp: p.timestamp,
278            }) - 1;
279            this._pendingRequests.set(p.requestId, idx);
280          }
281        });
282  
283        // Step 2: Fill in response metadata on responseReceived
284        this.bridge.on('Network.responseReceived', (params: unknown) => {
285          const p = params as { requestId: string; response: { status: number; mimeType?: string } };
286          const idx = this._pendingRequests.get(p.requestId);
287          if (idx !== undefined) {
288            this._networkEntries[idx].responseStatus = p.response.status;
289            this._networkEntries[idx].responseContentType = p.response.mimeType || '';
290          }
291        });
292  
293        // Step 3: Fetch body on loadingFinished (body is only reliably available after this)
294        this.bridge.on('Network.loadingFinished', (params: unknown) => {
295          const p = params as { requestId: string };
296          const idx = this._pendingRequests.get(p.requestId);
297          if (idx !== undefined) {
298            const bodyFetch = this.bridge.send('Network.getResponseBody', { requestId: p.requestId }).then((result: unknown) => {
299              const r = result as { body?: string; base64Encoded?: boolean } | undefined;
300              if (typeof r?.body === 'string') {
301                const fullSize = r.body.length;
302                const truncated = fullSize > CDP_RESPONSE_BODY_CAPTURE_LIMIT;
303                const body = truncated ? r.body.slice(0, CDP_RESPONSE_BODY_CAPTURE_LIMIT) : r.body;
304                this._networkEntries[idx].responsePreview = r.base64Encoded ? `base64:${body}` : body;
305                this._networkEntries[idx].responseBodyFullSize = fullSize;
306                this._networkEntries[idx].responseBodyTruncated = truncated;
307              }
308            }).catch((err) => {
309              // Body unavailable for some requests (e.g. uploads) — non-fatal
310              if (process.env.OPENCLI_VERBOSE) {
311                // eslint-disable-next-line no-console
312                console.error(`[cdp] getResponseBody failed for ${p.requestId}:`, err instanceof Error ? err.message : err);
313              }
314            }).finally(() => {
315              this._pendingBodyFetches.delete(bodyFetch);
316            });
317            this._pendingBodyFetches.add(bodyFetch);
318            this._pendingRequests.delete(p.requestId);
319          }
320        });
321  
322        this._networkCapturing = true;
323      }
324      return true;
325    }
326  
327    async readNetworkCapture(): Promise<unknown[]> {
328      // Await all in-flight body fetches so entries have responsePreview populated
329      if (this._pendingBodyFetches.size > 0) {
330        await Promise.all([...this._pendingBodyFetches]);
331      }
332      const entries = [...this._networkEntries];
333      this._networkEntries = [];
334      return entries;
335    }
336  
337    async consoleMessages(level: string = 'all'): Promise<Array<{ type: string; text: string; timestamp: number }>> {
338      if (!this._consoleCapturing) {
339        await this.bridge.send('Runtime.enable');
340        this.bridge.on('Runtime.consoleAPICalled', (params: unknown) => {
341          const p = params as { type: string; args: Array<{ value?: unknown; description?: string }>; timestamp: number };
342          const text = (p.args || []).map(a => a.value !== undefined ? String(a.value) : (a.description || '')).join(' ');
343          this._consoleMessages.push({ type: p.type, text, timestamp: p.timestamp });
344          if (this._consoleMessages.length > 500) this._consoleMessages.shift();
345        });
346        // Capture uncaught exceptions as error-level messages
347        this.bridge.on('Runtime.exceptionThrown', (params: unknown) => {
348          const p = params as { timestamp: number; exceptionDetails?: { exception?: { description?: string }; text?: string } };
349          const desc = p.exceptionDetails?.exception?.description || p.exceptionDetails?.text || 'Unknown exception';
350          this._consoleMessages.push({ type: 'error', text: desc, timestamp: p.timestamp });
351          if (this._consoleMessages.length > 500) this._consoleMessages.shift();
352        });
353        this._consoleCapturing = true;
354      }
355      if (level === 'all') return [...this._consoleMessages];
356      // 'error' level includes both console.error() and uncaught exceptions
357      if (level === 'error') return this._consoleMessages.filter(m => m.type === 'error' || m.type === 'warning');
358      return this._consoleMessages.filter(m => m.type === level);
359    }
360  
361    async tabs(): Promise<unknown[]> {
362      return [];
363    }
364  
365    async selectTab(_target: number | string): Promise<void> {
366      // Not supported in direct CDP mode
367    }
368  }
369  
370  function isCookie(value: unknown): value is BrowserCookie {
371    return isRecord(value)
372      && typeof value.name === 'string'
373      && typeof value.value === 'string'
374      && typeof value.domain === 'string';
375  }
376  
377  function matchesCookieDomain(cookieDomain: string, targetDomain: string): boolean {
378    const normalizedCookieDomain = cookieDomain.replace(/^\./, '').toLowerCase();
379    const normalizedTargetDomain = targetDomain.replace(/^\./, '').toLowerCase();
380    return normalizedTargetDomain === normalizedCookieDomain
381      || normalizedTargetDomain.endsWith(`.${normalizedCookieDomain}`);
382  }
383  
384  function selectCDPTarget(targets: CDPTarget[]): CDPTarget | undefined {
385    const preferredPattern = compilePreferredPattern(process.env.OPENCLI_CDP_TARGET);
386  
387    const ranked = targets
388      .map((target, index) => ({ target, index, score: scoreCDPTarget(target, preferredPattern) }))
389      .filter(({ score }) => Number.isFinite(score))
390      .sort((a, b) => {
391        if (b.score !== a.score) return b.score - a.score;
392        return a.index - b.index;
393      });
394  
395    return ranked[0]?.target;
396  }
397  
398  function scoreCDPTarget(target: CDPTarget, preferredPattern?: RegExp): number {
399    if (!target.webSocketDebuggerUrl) return Number.NEGATIVE_INFINITY;
400  
401    const type = (target.type ?? '').toLowerCase();
402    const url = (target.url ?? '').toLowerCase();
403    const title = (target.title ?? '').toLowerCase();
404    const haystack = `${title} ${url}`;
405  
406    if (!haystack.trim() && !type) return Number.NEGATIVE_INFINITY;
407    if (haystack.includes('devtools')) return Number.NEGATIVE_INFINITY;
408    if (type === 'background_page' || type === 'service_worker') return Number.NEGATIVE_INFINITY;
409  
410    let score = 0;
411  
412    if (preferredPattern && preferredPattern.test(haystack)) score += 1000;
413  
414    if (type === 'app') score += 120;
415    else if (type === 'webview') score += 100;
416    else if (type === 'page') score += 80;
417    else if (type === 'iframe') score += 20;
418  
419    if (url.startsWith('http://localhost') || url.startsWith('https://localhost')) score += 90;
420    if (url.startsWith('file://')) score += 60;
421    if (url.startsWith('http://127.0.0.1') || url.startsWith('https://127.0.0.1')) score += 50;
422    if (url.startsWith('about:blank')) score -= 120;
423    if (url === '' || url === 'about:blank') score -= 40;
424  
425    if (title && title !== 'devtools') score += 25;
426  
427    // Boost score for known Electron app names from the registry (builtin + user-defined)
428    const appNames = Object.values(getAllElectronApps()).map(a => (a.displayName ?? a.processName).toLowerCase());
429    for (const name of appNames) {
430      if (title.includes(name)) { score += 120; break; }
431    }
432    for (const name of appNames) {
433      if (url.includes(name)) { score += 100; break; }
434    }
435  
436    return score;
437  }
438  
439  function compilePreferredPattern(raw: string | undefined): RegExp | undefined {
440    const value = raw?.trim();
441    if (!value) return undefined;
442    return new RegExp(escapeRegExp(value.toLowerCase()));
443  }
444  
445  function escapeRegExp(value: string): string {
446    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
447  }
448  
449  export const __test__ = {
450    selectCDPTarget,
451    scoreCDPTarget,
452  };
453  
454  function fetchJsonDirect(url: string): Promise<unknown> {
455    return new Promise((resolve, reject) => {
456      const parsed = new URL(url);
457      const request = (parsed.protocol === 'https:' ? httpsRequest : httpRequest)(parsed, (res) => {
458        const statusCode = res.statusCode ?? 0;
459        if (statusCode < 200 || statusCode >= 300) {
460          res.resume();
461          reject(new Error(`Failed to fetch CDP targets: HTTP ${statusCode}`));
462          return;
463        }
464  
465        const chunks: Buffer[] = [];
466        res.on('data', (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
467        res.on('end', () => {
468          try {
469            resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')));
470          } catch (error) {
471            reject(error instanceof Error ? error : new Error(String(error)));
472          }
473        });
474      });
475  
476      request.on('error', reject);
477      request.setTimeout(10_000, () => request.destroy(new Error('Timed out fetching CDP targets')));
478      request.end();
479    });
480  }