/ src / lib / server / tool-retry.ts
tool-retry.ts
 1  /**
 2   * Structured retry with exponential backoff for transient tool failures.
 3   */
 4  
 5  import { sleep, jitteredBackoff } from '@/lib/shared-utils'
 6  import { log } from '@/lib/server/logger'
 7  
 8  const TAG = 'tool-retry'
 9  
10  export interface RetryOptions {
11    maxAttempts?: number
12    backoffMs?: number
13    retryable?: RegExp[]
14    onRetry?: (attempt: number, lastResult: string) => Promise<void> | void
15  }
16  
17  const DEFAULT_RETRYABLE: RegExp[] = [
18    /timeout/i,
19    /ECONNRESET/i,
20    /ENOTFOUND/i,
21    /429/,
22    /503/,
23    /rate.?limit/i,
24  ]
25  
26  const DEFAULT_MAX_ATTEMPTS = 3
27  const DEFAULT_BACKOFF_MS = 2000
28  
29  function isRetryableError(error: string, patterns: RegExp[]): boolean {
30    return patterns.some((p) => p.test(error))
31  }
32  
33  
34  /**
35   * Wraps a tool handler function with retry logic for transient failures.
36   * The wrapped function must return a string (tool output).
37   * Retries only when the returned string matches a retryable pattern
38   * (tool handlers typically return error strings rather than throwing).
39   */
40  export async function withRetry<TArgs>(
41    fn: (args: TArgs) => Promise<string>,
42    args: TArgs,
43    opts?: RetryOptions,
44  ): Promise<string> {
45    const maxAttempts = opts?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS
46    const backoffMs = opts?.backoffMs ?? DEFAULT_BACKOFF_MS
47    const retryable = opts?.retryable ?? DEFAULT_RETRYABLE
48  
49    let lastResult = ''
50    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
51      lastResult = await fn(args)
52  
53      // Only retry if the result looks like a retryable error
54      if (attempt < maxAttempts && isRetryableError(lastResult, retryable)) {
55        await opts?.onRetry?.(attempt, lastResult)
56        const delay = jitteredBackoff(backoffMs, attempt - 1, backoffMs * 16)
57        log.warn(TAG, `Attempt ${attempt}/${maxAttempts} matched retryable pattern, retrying in ${delay}ms`)
58        await sleep(delay)
59        continue
60      }
61      return lastResult
62    }
63    return lastResult
64  }