/ services / vcr.ts
vcr.ts
  1  import type { BetaContentBlock } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs'
  2  import { createHash, randomUUID, type UUID } from 'crypto'
  3  import { mkdir, readFile, writeFile } from 'fs/promises'
  4  import isPlainObject from 'lodash-es/isPlainObject.js'
  5  import mapValues from 'lodash-es/mapValues.js'
  6  import { dirname, join } from 'path'
  7  import { addToTotalSessionCost } from 'src/cost-tracker.js'
  8  import { calculateUSDCost } from 'src/utils/modelCost.js'
  9  import type {
 10    AssistantMessage,
 11    Message,
 12    StreamEvent,
 13    SystemAPIErrorMessage,
 14    UserMessage,
 15  } from '../types/message.js'
 16  import { getCwd } from '../utils/cwd.js'
 17  import { env } from '../utils/env.js'
 18  import { getClaudeConfigHomeDir, isEnvTruthy } from '../utils/envUtils.js'
 19  import { getErrnoCode } from '../utils/errors.js'
 20  import { normalizeMessagesForAPI } from '../utils/messages.js'
 21  import { jsonParse, jsonStringify } from '../utils/slowOperations.js'
 22  
 23  function shouldUseVCR(): boolean {
 24    if (process.env.NODE_ENV === 'test') {
 25      return true
 26    }
 27  
 28    if (process.env.USER_TYPE === 'ant' && isEnvTruthy(process.env.FORCE_VCR)) {
 29      return true
 30    }
 31  
 32    return false
 33  }
 34  
 35  /**
 36   * Generic fixture management helper
 37   * Handles caching, reading, writing fixtures for any data type
 38   */
 39  async function withFixture<T>(
 40    input: unknown,
 41    fixtureName: string,
 42    f: () => Promise<T>,
 43  ): Promise<T> {
 44    if (!shouldUseVCR()) {
 45      return await f()
 46    }
 47  
 48    // Create hash of input for fixture filename
 49    const hash = createHash('sha1')
 50      .update(jsonStringify(input))
 51      .digest('hex')
 52      .slice(0, 12)
 53    const filename = join(
 54      process.env.CLAUDE_CODE_TEST_FIXTURES_ROOT ?? getCwd(),
 55      `fixtures/${fixtureName}-${hash}.json`,
 56    )
 57  
 58    // Fetch cached fixture
 59    try {
 60      const cached = jsonParse(
 61        await readFile(filename, { encoding: 'utf8' }),
 62      ) as T
 63      return cached
 64    } catch (e: unknown) {
 65      const code = getErrnoCode(e)
 66      if (code !== 'ENOENT') {
 67        throw e
 68      }
 69    }
 70  
 71    if ((env.isCI || process.env.CI) && !isEnvTruthy(process.env.VCR_RECORD)) {
 72      throw new Error(
 73        `Fixture missing: ${filename}. Re-run tests with VCR_RECORD=1, then commit the result.`,
 74      )
 75    }
 76  
 77    // Create & write new fixture
 78    const result = await f()
 79  
 80    await mkdir(dirname(filename), { recursive: true })
 81    await writeFile(filename, jsonStringify(result, null, 2), {
 82      encoding: 'utf8',
 83    })
 84  
 85    return result
 86  }
 87  
 88  export async function withVCR(
 89    messages: Message[],
 90    f: () => Promise<(AssistantMessage | StreamEvent | SystemAPIErrorMessage)[]>,
 91  ): Promise<(AssistantMessage | StreamEvent | SystemAPIErrorMessage)[]> {
 92    if (!shouldUseVCR()) {
 93      return await f()
 94    }
 95  
 96    const messagesForAPI = normalizeMessagesForAPI(
 97      messages.filter(_ => {
 98        if (_.type !== 'user') {
 99          return true
100        }
101        if (_.isMeta) {
102          return false
103        }
104        return true
105      }),
106    )
107  
108    const dehydratedInput = mapMessages(
109      messagesForAPI.map(_ => _.message.content),
110      dehydrateValue,
111    )
112    const filename = join(
113      process.env.CLAUDE_CODE_TEST_FIXTURES_ROOT ?? getCwd(),
114      `fixtures/${dehydratedInput.map(_ => createHash('sha1').update(jsonStringify(_)).digest('hex').slice(0, 6)).join('-')}.json`,
115    )
116  
117    // Fetch cached fixture
118    try {
119      const cached = jsonParse(
120        await readFile(filename, { encoding: 'utf8' }),
121      ) as { output: (AssistantMessage | StreamEvent)[] }
122      cached.output.forEach(addCachedCostToTotalSessionCost)
123      return cached.output.map((message, index) =>
124        mapMessage(message, hydrateValue, index, randomUUID()),
125      )
126    } catch (e: unknown) {
127      const code = getErrnoCode(e)
128      if (code !== 'ENOENT') {
129        throw e
130      }
131    }
132  
133    if (env.isCI && !isEnvTruthy(process.env.VCR_RECORD)) {
134      throw new Error(
135        `Anthropic API fixture missing: ${filename}. Re-run tests with VCR_RECORD=1, then commit the result. Input messages:\n${jsonStringify(dehydratedInput, null, 2)}`,
136      )
137    }
138  
139    // Create & write new fixture
140    const results = await f()
141    if (env.isCI && !isEnvTruthy(process.env.VCR_RECORD)) {
142      return results
143    }
144  
145    await mkdir(dirname(filename), { recursive: true })
146    await writeFile(
147      filename,
148      jsonStringify(
149        {
150          input: dehydratedInput,
151          output: results.map((message, index) =>
152            mapMessage(message, dehydrateValue, index),
153          ),
154        },
155        null,
156        2,
157      ),
158      { encoding: 'utf8' },
159    )
160    return results
161  }
162  
163  function addCachedCostToTotalSessionCost(
164    message: AssistantMessage | StreamEvent,
165  ): void {
166    if (message.type === 'stream_event') {
167      return
168    }
169    const model = message.message.model
170    const usage = message.message.usage
171    const costUSD = calculateUSDCost(model, usage)
172    addToTotalSessionCost(costUSD, usage, model)
173  }
174  
175  function mapMessages(
176    messages: (UserMessage | AssistantMessage)['message']['content'][],
177    f: (s: unknown) => unknown,
178  ): (UserMessage | AssistantMessage)['message']['content'][] {
179    return messages.map(_ => {
180      if (typeof _ === 'string') {
181        return f(_)
182      }
183      return _.map(_ => {
184        switch (_.type) {
185          case 'tool_result':
186            if (typeof _.content === 'string') {
187              return { ..._, content: f(_.content) }
188            }
189            if (Array.isArray(_.content)) {
190              return {
191                ..._,
192                content: _.content.map(_ => {
193                  switch (_.type) {
194                    case 'text':
195                      return { ..._, text: f(_.text) }
196                    case 'image':
197                      return _
198                    default:
199                      return undefined
200                  }
201                }),
202              }
203            }
204            return _
205          case 'text':
206            return { ..._, text: f(_.text) }
207          case 'tool_use':
208            return {
209              ..._,
210              input: mapValuesDeep(_.input as Record<string, unknown>, f),
211            }
212          case 'image':
213            return _
214          default:
215            return undefined
216        }
217      })
218    }) as (UserMessage | AssistantMessage)['message']['content'][]
219  }
220  
221  function mapValuesDeep(
222    obj: {
223      [x: string]: unknown
224    },
225    f: (val: unknown, key: string, obj: Record<string, unknown>) => unknown,
226  ): Record<string, unknown> {
227    return mapValues(obj, (val, key) => {
228      if (Array.isArray(val)) {
229        return val.map(_ => mapValuesDeep(_, f))
230      }
231      if (isPlainObject(val)) {
232        return mapValuesDeep(val as Record<string, unknown>, f)
233      }
234      return f(val, key, obj)
235    })
236  }
237  
238  function mapAssistantMessage(
239    message: AssistantMessage,
240    f: (s: unknown) => unknown,
241    index: number,
242    uuid?: UUID,
243  ): AssistantMessage {
244    return {
245      // Use provided UUID if given (hydrate path uses randomUUID for globally unique IDs),
246      // otherwise fall back to deterministic index-based UUID (dehydrate/fixture path).
247      // sessionStorage.ts deduplicates messages by UUID, so without unique UUIDs across
248      // VCR calls, resumed sessions would treat different responses as duplicates.
249      uuid: uuid ?? (`UUID-${index}` as unknown as UUID),
250      requestId: 'REQUEST_ID',
251      timestamp: message.timestamp,
252      message: {
253        ...message.message,
254        content: message.message.content
255          .map(_ => {
256            switch (_.type) {
257              case 'text':
258                return {
259                  ..._,
260                  text: f(_.text) as string,
261                  citations: _.citations || [],
262                } // Ensure citations
263              case 'tool_use':
264                return {
265                  ..._,
266                  input: mapValuesDeep(_.input as Record<string, unknown>, f),
267                }
268              default:
269                return _ // Handle other block types unchanged
270            }
271          })
272          .filter(Boolean) as BetaContentBlock[],
273      },
274      type: 'assistant',
275    }
276  }
277  
278  function mapMessage(
279    message: AssistantMessage | SystemAPIErrorMessage | StreamEvent,
280    f: (s: unknown) => unknown,
281    index: number,
282    uuid?: UUID,
283  ): AssistantMessage | SystemAPIErrorMessage | StreamEvent {
284    if (message.type === 'assistant') {
285      return mapAssistantMessage(message, f, index, uuid)
286    } else {
287      return message
288    }
289  }
290  
291  function dehydrateValue(s: unknown): unknown {
292    if (typeof s !== 'string') {
293      return s
294    }
295    const cwd = getCwd()
296    const configHome = getClaudeConfigHomeDir()
297    let s1 = s
298      .replace(/num_files="\d+"/g, 'num_files="[NUM]"')
299      .replace(/duration_ms="\d+"/g, 'duration_ms="[DURATION]"')
300      .replace(/cost_usd="\d+"/g, 'cost_usd="[COST]"')
301      // Note: We intentionally don't replace all forward slashes with path.sep here.
302      // That would corrupt XML-like tags (e.g., </system-reminder> -> <\system-reminder>).
303      // The [CONFIG_HOME] and [CWD] replacements below handle path normalization.
304      .replaceAll(configHome, '[CONFIG_HOME]')
305      .replaceAll(cwd, '[CWD]')
306      .replace(/Available commands:.+/, 'Available commands: [COMMANDS]')
307    // On Windows, paths may appear in multiple forms:
308    // 1. Forward-slash variants (Git, some Node APIs)
309    // 2. JSON-escaped variants (backslashes doubled in serialized JSON within messages)
310    if (process.platform === 'win32') {
311      const cwdFwd = cwd.replaceAll('\\', '/')
312      const configHomeFwd = configHome.replaceAll('\\', '/')
313      // jsonStringify escapes \ to \\ - match paths embedded in JSON strings
314      const cwdJsonEscaped = jsonStringify(cwd).slice(1, -1)
315      const configHomeJsonEscaped = jsonStringify(configHome).slice(1, -1)
316      s1 = s1
317        .replaceAll(cwdJsonEscaped, '[CWD]')
318        .replaceAll(configHomeJsonEscaped, '[CONFIG_HOME]')
319        .replaceAll(cwdFwd, '[CWD]')
320        .replaceAll(configHomeFwd, '[CONFIG_HOME]')
321    }
322    // Normalize backslash path separators after placeholders so VCR fixture
323    // hashes match across platforms (e.g., [CWD]\foo\bar -> [CWD]/foo/bar)
324    // Handle both single backslashes and JSON-escaped double backslashes (\\)
325    s1 = s1
326      .replace(/\[CWD\][^\s"'<>]*/g, match =>
327        match.replaceAll('\\\\', '/').replaceAll('\\', '/'),
328      )
329      .replace(/\[CONFIG_HOME\][^\s"'<>]*/g, match =>
330        match.replaceAll('\\\\', '/').replaceAll('\\', '/'),
331      )
332    if (s1.includes('Files modified by user:')) {
333      return 'Files modified by user: [FILES]'
334    }
335    return s1
336  }
337  
338  function hydrateValue(s: unknown): unknown {
339    if (typeof s !== 'string') {
340      return s
341    }
342    return s
343      .replaceAll('[NUM]', '1')
344      .replaceAll('[DURATION]', '100')
345      .replaceAll('[CONFIG_HOME]', getClaudeConfigHomeDir())
346      .replaceAll('[CWD]', getCwd())
347  }
348  
349  export async function* withStreamingVCR(
350    messages: Message[],
351    f: () => AsyncGenerator<
352      StreamEvent | AssistantMessage | SystemAPIErrorMessage,
353      void
354    >,
355  ): AsyncGenerator<
356    StreamEvent | AssistantMessage | SystemAPIErrorMessage,
357    void
358  > {
359    if (!shouldUseVCR()) {
360      return yield* f()
361    }
362  
363    // Compute and yield messages
364    const buffer: (StreamEvent | AssistantMessage | SystemAPIErrorMessage)[] = []
365  
366    // Record messages (or fetch from cache)
367    const cachedBuffer = await withVCR(messages, async () => {
368      for await (const message of f()) {
369        buffer.push(message)
370      }
371      return buffer
372    })
373  
374    if (cachedBuffer.length > 0) {
375      yield* cachedBuffer
376      return
377    }
378  
379    yield* buffer
380  }
381  
382  export async function withTokenCountVCR(
383    messages: unknown[],
384    tools: unknown[],
385    f: () => Promise<number | null>,
386  ): Promise<number | null> {
387    // Dehydrate before hashing so fixture keys survive cwd/config-home/tempdir
388    // variation and message UUID/timestamp churn. System prompts embed the
389    // working directory (both raw and as a slash→dash project slug in the
390    // auto-memory path) and messages carry fresh UUIDs per run; without this,
391    // every test run produces a new hash and fixtures never hit in CI.
392    const cwdSlug = getCwd().replace(/[^a-zA-Z0-9]/g, '-')
393    const dehydrated = (
394      dehydrateValue(jsonStringify({ messages, tools })) as string
395    )
396      .replaceAll(cwdSlug, '[CWD_SLUG]')
397      .replace(
398        /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi,
399        '[UUID]',
400      )
401      .replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z?/g, '[TIMESTAMP]')
402    const result = await withFixture(dehydrated, 'token-count', async () => ({
403      tokenCount: await f(),
404    }))
405    return result.tokenCount
406  }