test_update_model_catalog.py
1 import json 2 import sys 3 from datetime import date 4 from pathlib import Path 5 6 import pytest 7 8 sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "dev")) 9 10 from update_model_catalog import ( 11 _extract_long_context_pricing, 12 _extract_modality_pricing, 13 _extract_service_tiers, 14 _extract_tool_pricing, 15 _is_deprecated, 16 _migrate_legacy_pricing, 17 _normalize_provider, 18 _transform_entry, 19 convert, 20 ) 21 22 23 @pytest.mark.parametrize( 24 ("provider", "expected"), 25 [ 26 ("openai", "openai"), 27 ("anthropic", "anthropic"), 28 ("vertex_ai", "vertex_ai"), 29 ("vertex_ai-anthropic", "vertex_ai"), 30 ("vertex_ai-llama_models", "vertex_ai"), 31 ("vertex_ai-chat-models", "vertex_ai"), 32 ("bedrock", "bedrock"), 33 ], 34 ) 35 def test_normalize_provider(provider, expected): 36 assert _normalize_provider(provider) == expected 37 38 39 def test_transform_entry_chat_model(): 40 info = { 41 "mode": "chat", 42 "input_cost_per_token": 3e-6, 43 "output_cost_per_token": 1.5e-5, 44 "cache_read_input_token_cost": 3e-7, 45 "cache_creation_input_token_cost": 3.75e-6, 46 "max_input_tokens": 200000, 47 "max_output_tokens": 64000, 48 "max_tokens": 64000, 49 "supports_function_calling": True, 50 "supports_vision": True, 51 "supports_reasoning": True, 52 "supports_prompt_caching": True, 53 "supports_response_schema": True, 54 } 55 result = _transform_entry(info) 56 assert result == { 57 "mode": "chat", 58 "context_window": {"max_input": 200000, "max_output": 64000, "max_tokens": 64000}, 59 "pricing": { 60 "input_per_million_tokens": 3.0, 61 "output_per_million_tokens": 15.0, 62 "cache_read_per_million_tokens": 0.3, 63 "cache_write_per_million_tokens": 3.75, 64 }, 65 "capabilities": { 66 "function_calling": True, 67 "vision": True, 68 "reasoning": True, 69 "prompt_caching": True, 70 "response_schema": True, 71 }, 72 } 73 74 75 def test_transform_entry_skips_image_generation(): 76 info = {"mode": "image_generation", "input_cost_per_token": 1e-6} 77 assert _transform_entry(info) is None 78 79 80 def test_transform_entry_includes_future_deprecation_date(): 81 info = { 82 "mode": "chat", 83 "deprecation_date": "2099-01-01", 84 } 85 result = _transform_entry(info) 86 assert result is not None 87 assert result["deprecation_date"] == "2099-01-01" 88 89 90 def test_transform_entry_skips_past_deprecation_date(): 91 info = { 92 "mode": "chat", 93 "deprecation_date": "2020-01-01", 94 } 95 assert _transform_entry(info) is None 96 97 98 def test_is_deprecated_past_date(): 99 assert _is_deprecated({"deprecation_date": "2020-01-01"}) is True 100 101 102 def test_is_deprecated_future_date(): 103 assert _is_deprecated({"deprecation_date": "2099-01-01"}) is False 104 105 106 def test_is_deprecated_no_date(): 107 assert _is_deprecated({}) is False 108 109 110 def test_is_deprecated_invalid_date(): 111 assert _is_deprecated({"deprecation_date": "not-a-date"}) is False 112 113 114 def test_transform_entry_with_service_tiers(): 115 info = { 116 "mode": "chat", 117 "input_cost_per_token": 2e-6, 118 "output_cost_per_token": 8e-6, 119 "cache_read_input_token_cost": 5e-7, 120 "input_cost_per_token_flex": 1e-6, 121 "output_cost_per_token_flex": 4e-6, 122 "cache_read_input_token_cost_flex": 2.5e-7, 123 "input_cost_per_token_priority": 3.5e-6, 124 "output_cost_per_token_priority": 1.4e-5, 125 "input_cost_per_token_batches": 1e-6, 126 "output_cost_per_token_batches": 4e-6, 127 } 128 result = _transform_entry(info) 129 tiers = result["pricing"]["service_tiers"] 130 assert tiers["flex"] == { 131 "input_per_million_tokens": 1.0, 132 "output_per_million_tokens": 4.0, 133 "cache_read_per_million_tokens": 0.25, 134 } 135 assert tiers["priority"] == { 136 "input_per_million_tokens": 3.5, 137 "output_per_million_tokens": 14.0, 138 } 139 assert tiers["batch"] == { 140 "input_per_million_tokens": 1.0, 141 "output_per_million_tokens": 4.0, 142 } 143 144 145 def test_transform_entry_with_long_context(): 146 info = { 147 "mode": "chat", 148 "input_cost_per_token": 1.25e-6, 149 "output_cost_per_token": 1e-5, 150 "input_cost_per_token_above_200k_tokens": 2.5e-6, 151 "output_cost_per_token_above_200k_tokens": 1.5e-5, 152 "cache_read_input_token_cost_above_200k_tokens": 2.5e-7, 153 } 154 result = _transform_entry(info) 155 long_ctx = result["pricing"]["long_context"] 156 assert len(long_ctx) == 1 157 assert long_ctx[0] == { 158 "threshold_tokens": 200000, 159 "input_per_million_tokens": 2.5, 160 "output_per_million_tokens": 15.0, 161 "cache_read_per_million_tokens": 0.25, 162 } 163 164 165 def test_extract_long_context_multiple_thresholds(): 166 info = { 167 "input_cost_per_token_above_128k_tokens": 1e-6, 168 "output_cost_per_token_above_128k_tokens": 2e-6, 169 "input_cost_per_token_above_256k_tokens": 2e-6, 170 "output_cost_per_token_above_256k_tokens": 4e-6, 171 } 172 result = _extract_long_context_pricing(info) 173 assert len(result) == 2 174 assert result[0]["threshold_tokens"] == 128000 175 assert result[1]["threshold_tokens"] == 256000 176 177 178 def test_extract_service_tiers_empty_when_no_tiers(): 179 info = {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} 180 assert _extract_service_tiers(info) == {} 181 182 183 def test_extract_modality_pricing(): 184 info = { 185 "input_cost_per_audio_token": 7e-7, 186 "output_cost_per_audio_token": 1.1e-6, 187 "cache_read_input_audio_token_cost": 2e-7, 188 "cache_creation_input_audio_token_cost": 4e-7, 189 } 190 assert _extract_modality_pricing(info) == { 191 "audio": { 192 "input_per_million_tokens": 0.7, 193 "output_per_million_tokens": 1.1, 194 "cache_read_per_million_tokens": 0.2, 195 "cache_write_per_million_tokens": 0.4, 196 } 197 } 198 199 200 def test_extract_modality_pricing_skips_reasoning(): 201 info = { 202 "input_cost_per_audio_token": 7e-7, 203 "output_cost_per_reasoning_token": 4e-7, 204 } 205 assert _extract_modality_pricing(info) == {"audio": {"input_per_million_tokens": 0.7}} 206 207 208 def test_extract_tool_pricing(): 209 info = { 210 "computer_use_input_cost_per_1k_tokens": 0.00225, 211 "computer_use_output_cost_per_1k_tokens": 0.009, 212 "search_context_cost_per_query": { 213 "search_context_size_low": 0.01, 214 "search_context_size_medium": 0.01, 215 "search_context_size_high": 0.01, 216 }, 217 "tool_use_system_prompt_tokens": 159, 218 } 219 assert _extract_tool_pricing(info) == { 220 "computer_use": { 221 "input_per_million_tokens": 2.25, 222 "output_per_million_tokens": 9.0, 223 }, 224 "search_context_per_query": { 225 "search_context_size_low": 0.01, 226 "search_context_size_medium": 0.01, 227 "search_context_size_high": 0.01, 228 }, 229 "tool_use_system_prompt_tokens": 159, 230 } 231 232 233 def test_transform_entry_with_modality_and_tool_pricing(): 234 info = { 235 "mode": "chat", 236 "input_cost_per_token": 1e-7, 237 "output_cost_per_token": 4e-7, 238 "input_cost_per_audio_token": 7e-7, 239 "computer_use_input_cost_per_1k_tokens": 0.00225, 240 "tool_use_system_prompt_tokens": 159, 241 } 242 result = _transform_entry(info) 243 assert result["pricing"]["modality"] == {"audio": {"input_per_million_tokens": 0.7}} 244 assert result["pricing"]["tooling"] == { 245 "computer_use": {"input_per_million_tokens": 2.25}, 246 "tool_use_system_prompt_tokens": 159, 247 } 248 249 250 def test_convert_end_to_end(tmp_path): 251 input_data = { 252 "sample_spec": {"mode": "chat"}, 253 "gpt-4o": { 254 "litellm_provider": "openai", 255 "mode": "chat", 256 "input_cost_per_token": 2.5e-6, 257 "output_cost_per_token": 1e-5, 258 "max_input_tokens": 128000, 259 "max_output_tokens": 16384, 260 "supports_function_calling": True, 261 "supports_vision": True, 262 }, 263 "openai/gpt-4o-mini": { 264 "litellm_provider": "openai", 265 "mode": "chat", 266 "input_cost_per_token": 1.5e-7, 267 "output_cost_per_token": 6e-7, 268 "max_input_tokens": 128000, 269 "max_output_tokens": 16384, 270 }, 271 "claude-3-5-sonnet": { 272 "litellm_provider": "anthropic", 273 "mode": "chat", 274 "input_cost_per_token": 3e-6, 275 "output_cost_per_token": 1.5e-5, 276 }, 277 "dall-e-3": { 278 "litellm_provider": "openai", 279 "mode": "image_generation", 280 }, 281 "ft:gpt-4o:org::id": { 282 "litellm_provider": "openai", 283 "mode": "chat", 284 }, 285 "bedrock_converse/model": { 286 "litellm_provider": "bedrock_converse", 287 "mode": "chat", 288 }, 289 } 290 291 output_dir = tmp_path / "output" 292 293 stats = convert(input_data, output_dir) 294 295 assert stats == {"anthropic": 1, "bedrock": 1, "openai": 2} 296 assert (output_dir / "openai.json").exists() 297 assert (output_dir / "anthropic.json").exists() 298 assert (output_dir / "bedrock.json").exists() 299 assert not (output_dir / "bedrock_converse.json").exists() 300 301 openai_catalog = json.loads((output_dir / "openai.json").read_text()) 302 assert openai_catalog["schema_version"] == "1.0" 303 assert "gpt-4o" in openai_catalog["models"] 304 assert "gpt-4o-mini" in openai_catalog["models"] 305 # Fine-tuned and image_generation should be excluded 306 assert "ft:gpt-4o:org::id" not in openai_catalog["models"] 307 assert "dall-e-3" not in openai_catalog["models"] 308 309 310 def test_convert_preserves_existing_models(tmp_path): 311 312 input_data = { 313 "gpt-4o": { 314 "litellm_provider": "openai", 315 "mode": "chat", 316 "input_cost_per_token": 2.5e-6, 317 }, 318 } 319 320 output_dir = tmp_path / "output" 321 output_dir.mkdir() 322 323 # Pre-populate with a manually-added model 324 existing_catalog = { 325 "schema_version": "1.0", 326 "models": { 327 "custom-model": { 328 "mode": "chat", 329 "pricing": {"input_per_million_tokens": 1.0}, 330 "capabilities": { 331 "function_calling": False, 332 "vision": False, 333 "reasoning": False, 334 "prompt_caching": False, 335 "response_schema": False, 336 }, 337 } 338 }, 339 } 340 (output_dir / "openai.json").write_text(json.dumps(existing_catalog)) 341 342 stats = convert(input_data, output_dir) 343 344 catalog = json.loads((output_dir / "openai.json").read_text()) 345 # Both upstream and manually-added models should be present 346 assert "gpt-4o" in catalog["models"] 347 assert "custom-model" in catalog["models"] 348 assert stats["openai"] == 2 349 350 351 def test_convert_preserves_community_provider(tmp_path): 352 353 input_data = { 354 "gpt-4o": { 355 "litellm_provider": "openai", 356 "mode": "chat", 357 }, 358 } 359 360 output_dir = tmp_path / "output" 361 output_dir.mkdir() 362 363 # Pre-populate with a community-maintained provider 364 community_catalog = { 365 "schema_version": "1.0", 366 "models": { 367 "my-model": { 368 "mode": "chat", 369 "capabilities": { 370 "function_calling": False, 371 "vision": False, 372 "reasoning": False, 373 "prompt_caching": False, 374 "response_schema": False, 375 }, 376 } 377 }, 378 } 379 (output_dir / "custom_provider.json").write_text(json.dumps(community_catalog)) 380 381 stats = convert(input_data, output_dir) 382 383 # Community provider should be preserved 384 assert (output_dir / "custom_provider.json").exists() 385 assert "custom_provider" in stats 386 assert stats["custom_provider"] == 1 387 388 389 def test_convert_skips_deprecated_models(tmp_path): 390 input_data = { 391 "old-model": { 392 "litellm_provider": "openai", 393 "mode": "chat", 394 "deprecation_date": "2020-01-01", 395 }, 396 "new-model": { 397 "litellm_provider": "openai", 398 "mode": "chat", 399 "deprecation_date": "2099-12-31", 400 }, 401 } 402 403 output_dir = tmp_path / "output" 404 405 stats = convert(input_data, output_dir) 406 407 catalog = json.loads((output_dir / "openai.json").read_text()) 408 assert "old-model" not in catalog["models"] 409 assert "new-model" in catalog["models"] 410 assert stats["openai"] == 1 411 412 413 def test_convert_upstream_overrides_existing_model(tmp_path): 414 415 input_data = { 416 "gpt-4o": { 417 "litellm_provider": "openai", 418 "mode": "chat", 419 "input_cost_per_token": 9.99e-6, 420 }, 421 } 422 423 output_dir = tmp_path / "output" 424 output_dir.mkdir() 425 426 # Pre-populate with old pricing 427 existing_catalog = { 428 "schema_version": "1.0", 429 "models": { 430 "gpt-4o": { 431 "mode": "chat", 432 "pricing": {"input_per_million_tokens": 1.0}, 433 "capabilities": { 434 "function_calling": False, 435 "vision": False, 436 "reasoning": False, 437 "prompt_caching": False, 438 "response_schema": False, 439 }, 440 } 441 }, 442 } 443 (output_dir / "openai.json").write_text(json.dumps(existing_catalog)) 444 445 convert(input_data, output_dir) 446 447 catalog = json.loads((output_dir / "openai.json").read_text()) 448 # Upstream price should win 449 assert catalog["models"]["gpt-4o"]["pricing"]["input_per_million_tokens"] == pytest.approx(9.99) 450 451 452 def test_convert_sets_last_updated_at_for_new_models(tmp_path): 453 input_data = { 454 "gpt-4o": { 455 "litellm_provider": "openai", 456 "mode": "chat", 457 "input_cost_per_token": 2.5e-6, 458 }, 459 } 460 461 output_dir = tmp_path / "output" 462 convert(input_data, output_dir) 463 464 catalog = json.loads((output_dir / "openai.json").read_text()) 465 assert catalog["models"]["gpt-4o"]["last_updated_at"] == date.today().isoformat() 466 467 468 def test_convert_preserves_last_updated_at_when_entry_unchanged(tmp_path): 469 input_data = { 470 "gpt-4o": { 471 "litellm_provider": "openai", 472 "mode": "chat", 473 "input_cost_per_token": 2.5e-6, 474 "max_input_tokens": 128000, 475 "max_output_tokens": 16384, 476 "supports_function_calling": True, 477 "supports_vision": True, 478 "supports_reasoning": False, 479 "supports_prompt_caching": False, 480 "supports_response_schema": False, 481 }, 482 } 483 484 output_dir = tmp_path / "output" 485 output_dir.mkdir() 486 487 # Pre-populate with the same data and an existing last_updated_at 488 existing_catalog = { 489 "schema_version": "1.0", 490 "models": { 491 "gpt-4o": { 492 "mode": "chat", 493 "context_window": {"max_input": 128000, "max_output": 16384}, 494 "pricing": {"input_per_million_tokens": 2.5}, 495 "capabilities": { 496 "function_calling": True, 497 "vision": True, 498 "reasoning": False, 499 "prompt_caching": False, 500 "response_schema": False, 501 }, 502 "last_updated_at": "2025-01-01", 503 } 504 }, 505 } 506 (output_dir / "openai.json").write_text(json.dumps(existing_catalog)) 507 508 convert(input_data, output_dir) 509 510 catalog = json.loads((output_dir / "openai.json").read_text()) 511 assert catalog["models"]["gpt-4o"]["last_updated_at"] == "2025-01-01" 512 513 514 def test_convert_updates_last_updated_at_when_entry_changes(tmp_path): 515 input_data = { 516 "gpt-4o": { 517 "litellm_provider": "openai", 518 "mode": "chat", 519 "input_cost_per_token": 9.99e-6, 520 }, 521 } 522 523 output_dir = tmp_path / "output" 524 output_dir.mkdir() 525 526 # Pre-populate with different pricing and an old last_updated_at 527 existing_catalog = { 528 "schema_version": "1.0", 529 "models": { 530 "gpt-4o": { 531 "mode": "chat", 532 "pricing": {"input_per_million_tokens": 1.0}, 533 "capabilities": { 534 "function_calling": False, 535 "vision": False, 536 "reasoning": False, 537 "prompt_caching": False, 538 "response_schema": False, 539 }, 540 "last_updated_at": "2025-01-01", 541 } 542 }, 543 } 544 (output_dir / "openai.json").write_text(json.dumps(existing_catalog)) 545 546 convert(input_data, output_dir) 547 548 catalog = json.loads((output_dir / "openai.json").read_text()) 549 assert catalog["models"]["gpt-4o"]["last_updated_at"] == date.today().isoformat() 550 551 552 def test_convert_sets_last_updated_at_for_unchanged_entry_without_existing_date(tmp_path): 553 input_data = { 554 "gpt-4o": { 555 "litellm_provider": "openai", 556 "mode": "chat", 557 "input_cost_per_token": 2.5e-6, 558 "max_input_tokens": 128000, 559 "max_output_tokens": 16384, 560 "supports_function_calling": True, 561 "supports_vision": True, 562 "supports_reasoning": False, 563 "supports_prompt_caching": False, 564 "supports_response_schema": False, 565 }, 566 } 567 568 output_dir = tmp_path / "output" 569 output_dir.mkdir() 570 571 # Pre-populate with the same data but NO last_updated_at (simulates pre-feature catalog) 572 existing_catalog = { 573 "schema_version": "1.0", 574 "models": { 575 "gpt-4o": { 576 "mode": "chat", 577 "context_window": {"max_input": 128000, "max_output": 16384}, 578 "pricing": {"input_per_million_tokens": 2.5}, 579 "capabilities": { 580 "function_calling": True, 581 "vision": True, 582 "reasoning": False, 583 "prompt_caching": False, 584 "response_schema": False, 585 }, 586 } 587 }, 588 } 589 (output_dir / "openai.json").write_text(json.dumps(existing_catalog)) 590 591 convert(input_data, output_dir) 592 593 catalog = json.loads((output_dir / "openai.json").read_text()) 594 assert catalog["models"]["gpt-4o"]["last_updated_at"] == date.today().isoformat() 595 596 entry = { 597 "mode": "chat", 598 "pricing": { 599 "input_per_token": 3e-6, 600 "output_per_token": 1.5e-5, 601 "cache_read_per_token": 3e-7, 602 "cache_write_per_token": 3.75e-6, 603 }, 604 } 605 result = _migrate_legacy_pricing(entry) 606 assert result["pricing"] == { 607 "input_per_million_tokens": 3.0, 608 "output_per_million_tokens": 15.0, 609 "cache_read_per_million_tokens": 0.3, 610 "cache_write_per_million_tokens": 3.75, 611 } 612 613 614 def test_migrate_legacy_pricing_service_tiers(): 615 entry = { 616 "mode": "chat", 617 "pricing": { 618 "input_per_token": 2e-6, 619 "output_per_token": 8e-6, 620 "service_tiers": { 621 "batch": { 622 "input_per_token": 1e-6, 623 "output_per_token": 4e-6, 624 }, 625 "priority": { 626 "input_per_token": 3e-6, 627 "output_per_token": 1.2e-5, 628 "cache_read_per_token": 3e-7, 629 }, 630 }, 631 }, 632 } 633 result = _migrate_legacy_pricing(entry) 634 assert result["pricing"]["input_per_million_tokens"] == pytest.approx(2.0) 635 tiers = result["pricing"]["service_tiers"] 636 assert tiers["batch"] == { 637 "input_per_million_tokens": 1.0, 638 "output_per_million_tokens": 4.0, 639 } 640 assert tiers["priority"] == { 641 "input_per_million_tokens": 3.0, 642 "output_per_million_tokens": 12.0, 643 "cache_read_per_million_tokens": 0.3, 644 } 645 646 647 def test_migrate_legacy_pricing_long_context(): 648 entry = { 649 "mode": "chat", 650 "pricing": { 651 "input_per_token": 1e-6, 652 "output_per_token": 4e-6, 653 "long_context": [ 654 { 655 "threshold_tokens": 200000, 656 "input_per_token": 2e-6, 657 "output_per_token": 8e-6, 658 "cache_read_per_token": 2e-7, 659 } 660 ], 661 }, 662 } 663 result = _migrate_legacy_pricing(entry) 664 ctx = result["pricing"]["long_context"] 665 assert len(ctx) == 1 666 assert ctx[0] == { 667 "threshold_tokens": 200000, 668 "input_per_million_tokens": 2.0, 669 "output_per_million_tokens": 8.0, 670 "cache_read_per_million_tokens": 0.2, 671 } 672 673 674 def test_migrate_legacy_pricing_modality(): 675 entry = { 676 "mode": "chat", 677 "pricing": { 678 "input_per_token": 1e-7, 679 "output_per_token": 4e-7, 680 "modality": { 681 "audio": { 682 "input_per_token": 7e-7, 683 "output_per_token": 1.1e-6, 684 } 685 }, 686 }, 687 } 688 result = _migrate_legacy_pricing(entry) 689 assert result["pricing"]["modality"] == { 690 "audio": { 691 "input_per_million_tokens": 0.7, 692 "output_per_million_tokens": 1.1, 693 } 694 } 695 696 697 def test_migrate_legacy_pricing_noop_when_already_normalized(): 698 entry = { 699 "mode": "chat", 700 "pricing": { 701 "input_per_million_tokens": 2.5, 702 "output_per_million_tokens": 10.0, 703 }, 704 } 705 result = _migrate_legacy_pricing(entry) 706 assert result["pricing"] == { 707 "input_per_million_tokens": 2.5, 708 "output_per_million_tokens": 10.0, 709 } 710 711 712 def test_migrate_legacy_pricing_noop_when_no_pricing(): 713 entry = {"mode": "chat", "capabilities": {}} 714 assert _migrate_legacy_pricing(entry) is entry 715 716 717 def test_convert_migrates_legacy_pricing_in_preserved_models(tmp_path): 718 input_data = { 719 "gpt-4o": { 720 "litellm_provider": "openai", 721 "mode": "chat", 722 "input_cost_per_token": 2.5e-6, 723 }, 724 } 725 726 output_dir = tmp_path / "output" 727 output_dir.mkdir() 728 729 # Pre-populate with a community model that uses the legacy per-token format 730 existing_catalog = { 731 "schema_version": "1.0", 732 "models": { 733 "legacy-model": { 734 "mode": "chat", 735 "pricing": { 736 "input_per_token": 3e-6, 737 "output_per_token": 1.5e-5, 738 "cache_read_per_token": 3e-7, 739 "service_tiers": { 740 "batch": { 741 "input_per_token": 1.5e-6, 742 "output_per_token": 7.5e-6, 743 } 744 }, 745 }, 746 "capabilities": { 747 "function_calling": False, 748 "vision": False, 749 "reasoning": False, 750 "prompt_caching": True, 751 "response_schema": False, 752 }, 753 } 754 }, 755 } 756 (output_dir / "openai.json").write_text(json.dumps(existing_catalog)) 757 758 convert(input_data, output_dir) 759 760 catalog = json.loads((output_dir / "openai.json").read_text()) 761 legacy_pricing = catalog["models"]["legacy-model"]["pricing"] 762 assert "input_per_token" not in legacy_pricing 763 assert legacy_pricing["input_per_million_tokens"] == pytest.approx(3.0) 764 assert legacy_pricing["output_per_million_tokens"] == pytest.approx(15.0) 765 assert legacy_pricing["cache_read_per_million_tokens"] == pytest.approx(0.3) 766 batch = legacy_pricing["service_tiers"]["batch"] 767 assert "input_per_token" not in batch 768 assert batch["input_per_million_tokens"] == pytest.approx(1.5) 769 assert batch["output_per_million_tokens"] == pytest.approx(7.5)