1  import { afterEach, describe, expect, it, vi } from 'vitest'
  2  
  3  import { testModelCapabilities } from '@/server/api/providers/test-model/run'
  4  
  5  describe('test model capability probes', () => {
  6    afterEach(() => {
  7      vi.unstubAllGlobals()
  8    })
  9  
 10    it('uses deterministic responses tool probe payloads', async () => {
 11      const fetchMock = vi
 12        .fn()
 13        .mockResolvedValueOnce(
 14          new Response('{}', {
 15            status: 200,
 16            headers: { 'content-type': 'application/json' },
 17          }),
 18        )
 19        .mockResolvedValueOnce(
 20          new Response('{}', {
 21            status: 200,
 22            headers: { 'content-type': 'application/json' },
 23          }),
 24        )
 25        .mockResolvedValueOnce(
 26          new Response(
 27            JSON.stringify({
 28              output: [{ type: 'function_call' }],
 29            }),
 30            {
 31              status: 200,
 32              headers: { 'content-type': 'application/json' },
 33            },
 34          ),
 35        )
 36  
 37      vi.stubGlobal('fetch', fetchMock)
 38  
 39      const result = await testModelCapabilities({
 40        baseUrl: 'https://api.openai.com/v1',
 41        apiKey: 'db-key',
 42        apiPath: '/responses',
 43        modelId: 'gpt-5-mini',
 44        timeoutMs: 1_000,
 45      })
 46  
 47      expect(result.basic.supported).toBe(true)
 48      expect(result.vision.supported).toBe(true)
 49      expect(result.toolUse.supported).toBe(true)
 50  
 51      const basicBody = JSON.parse(
 52        fetchMock.mock.calls[0]?.[1]?.body as string,
 53      ) as Record<string, unknown>
 54      const visionBody = JSON.parse(
 55        fetchMock.mock.calls[1]?.[1]?.body as string,
 56      ) as Record<string, unknown>
 57      const toolBody = JSON.parse(
 58        fetchMock.mock.calls[2]?.[1]?.body as string,
 59      ) as Record<string, unknown>
 60      expect((basicBody.max_output_tokens as number) >= 16).toBe(true)
 61      expect((visionBody.max_output_tokens as number) >= 16).toBe(true)
 62      expect((toolBody.max_output_tokens as number) >= 128).toBe(true)
 63      expect(toolBody.tool_choice).toBe('required')
 64    })
 65  
 66    it('retries responses tool probe without tool_choice when unsupported', async () => {
 67      const fetchMock = vi
 68        .fn()
 69        .mockResolvedValueOnce(
 70          new Response('{}', {
 71            status: 200,
 72            headers: { 'content-type': 'application/json' },
 73          }),
 74        )
 75        .mockResolvedValueOnce(
 76          new Response('{}', {
 77            status: 200,
 78            headers: { 'content-type': 'application/json' },
 79          }),
 80        )
 81        .mockResolvedValueOnce(
 82          new Response(
 83            JSON.stringify({
 84              error: { message: "Unsupported parameter: 'tool_choice'" },
 85            }),
 86            {
 87              status: 400,
 88              headers: { 'content-type': 'application/json' },
 89            },
 90          ),
 91        )
 92        .mockResolvedValueOnce(
 93          new Response(
 94            JSON.stringify({
 95              output: [{ type: 'function_call' }],
 96            }),
 97            {
 98              status: 200,
 99              headers: { 'content-type': 'application/json' },
100            },
101          ),
102        )
103  
104      vi.stubGlobal('fetch', fetchMock)
105  
106      const result = await testModelCapabilities({
107        baseUrl: 'https://api.openai.com/v1',
108        apiKey: 'db-key',
109        apiPath: '/responses',
110        modelId: 'gpt-5-mini',
111        timeoutMs: 1_000,
112      })
113  
114      expect(result.toolUse.supported).toBe(true)
115      expect(fetchMock).toHaveBeenCalledTimes(4)
116  
117      const firstToolAttempt = JSON.parse(
118        fetchMock.mock.calls[2]?.[1]?.body as string,
119      ) as Record<string, unknown>
120      const retryToolAttempt = JSON.parse(
121        fetchMock.mock.calls[3]?.[1]?.body as string,
122      ) as Record<string, unknown>
123      expect(firstToolAttempt.tool_choice).toBe('required')
124      expect('tool_choice' in retryToolAttempt).toBe(false)
125    })
126  
127    it('returns detailed HTTP body errors for failed basic probes', async () => {
128      vi.stubGlobal(
129        'fetch',
130        vi.fn().mockResolvedValue(
131          new Response(
132            JSON.stringify({
133              error: { message: 'Missing scopes: model.request' },
134            }),
135            {
136              status: 401,
137              statusText: 'Unauthorized',
138              headers: { 'content-type': 'application/json' },
139            },
140          ),
141        ),
142      )
143  
144      const result = await testModelCapabilities({
145        baseUrl: 'https://api.openai.com/v1',
146        apiKey: 'db-key',
147        apiPath: '/chat/completions',
148        modelId: 'gpt-5-mini',
149        timeoutMs: 1_000,
150      })
151  
152      expect(result.basic.supported).toBe(false)
153      expect(result.basic.error).toContain('HTTP 401')
154      expect(result.basic.error).toContain('model.request')
155      expect(result.vision.supported).toBe(false)
156      expect(result.toolUse.supported).toBe(false)
157    })
158  })