test_sdk_workspace.py
1 """Tests for Workspace SDK: prompts, artifacts, and configs.""" 2 3 import pytest 4 5 from evidently.llm.models import LLMMessage 6 from evidently.llm.prompts.content import MessagesPromptContent 7 from evidently.llm.prompts.content import PromptContent 8 from evidently.sdk.models import ProjectModel 9 from evidently.sdk.prompts import Prompt 10 from evidently.sdk.prompts import PromptMetadata 11 from evidently.ui.workspace import Workspace 12 13 14 @pytest.fixture 15 def workspace(tmp_path): 16 """Create a Workspace instance with a temporary directory.""" 17 return Workspace(str(tmp_path)) 18 19 20 @pytest.fixture 21 def test_project_id(workspace: Workspace): 22 """Create a test project and return its ID.""" 23 project_data = ProjectModel(name="test project", description="test") 24 project = workspace.add_project(project_data) 25 return project.id 26 27 28 def test_workspace_prompts_list(workspace: Workspace, test_project_id): 29 """Test Workspace.prompts.list_prompts().""" 30 # Initially empty 31 prompts = workspace.prompts.list_prompts(test_project_id) 32 assert len(prompts) == 0 33 34 # Create a prompt 35 prompt = workspace.prompts.create_prompt(test_project_id, "test prompt") 36 assert prompt.name == "test prompt" 37 38 # List prompts 39 prompts = workspace.prompts.list_prompts(test_project_id) 40 assert len(prompts) == 1 41 assert prompts[0].id == prompt.id 42 assert prompts[0].name == "test prompt" 43 44 45 def test_workspace_prompts_get(workspace: Workspace, test_project_id): 46 """Test Workspace.prompts.get_prompt().""" 47 # Create a prompt 48 created = workspace.prompts.create_prompt(test_project_id, "my prompt") 49 50 # Get by ID 51 retrieved = workspace.prompts.get_prompt_by_id(test_project_id, created.id) 52 assert retrieved.id == created.id 53 assert retrieved.name == "my prompt" 54 55 # Get by name 56 retrieved_by_name = workspace.prompts.get_prompt(test_project_id, "my prompt") 57 assert retrieved_by_name.id == created.id 58 assert retrieved_by_name.name == "my prompt" 59 60 61 def test_workspace_prompts_get_or_create(workspace: Workspace, test_project_id): 62 """Test Workspace.prompts.get_or_create_prompt().""" 63 # Get or create (should create) 64 prompt1 = workspace.prompts.get_or_create_prompt(test_project_id, "new prompt") 65 assert prompt1.name == "new prompt" 66 67 # Get or create (should get existing) 68 prompt2 = workspace.prompts.get_or_create_prompt(test_project_id, "new prompt") 69 assert prompt2.id == prompt1.id 70 assert prompt2.name == "new prompt" 71 72 73 def test_workspace_prompts_versions(workspace: Workspace, test_project_id): 74 """Test Workspace.prompts version operations.""" 75 # Create a prompt 76 prompt = workspace.prompts.create_prompt(test_project_id, "versioned prompt") 77 78 # Initially no versions 79 versions = workspace.prompts.list_versions(prompt.id) 80 assert len(versions) == 0 81 82 # Create a version with PromptContent (using MessagesPromptContent) 83 prompt_content = MessagesPromptContent( 84 messages=[ 85 LLMMessage(role="system", content="You are a helpful assistant."), 86 LLMMessage(role="user", content="Hello!"), 87 ] 88 ) 89 90 version = workspace.prompts.create_version(prompt.id, 1, prompt_content) 91 assert version.version == 1 92 assert version.prompt_id == prompt.id 93 94 # List versions 95 versions = workspace.prompts.list_versions(prompt.id) 96 assert len(versions) == 1 97 98 # Get version 99 retrieved = workspace.prompts.get_version(prompt.id, 1) 100 assert retrieved.version == 1 101 assert isinstance(retrieved.content, PromptContent) 102 103 # Get latest 104 latest = workspace.prompts.get_version(prompt.id, "latest") 105 assert latest.version == 1 106 107 108 def test_workspace_prompts_bump_version(workspace: Workspace, test_project_id): 109 """Test Workspace.prompts.bump_prompt_version().""" 110 # Create a prompt 111 prompt = workspace.prompts.create_prompt(test_project_id, "bump prompt") 112 113 # Bump version (creates version 1) 114 prompt_content1 = MessagesPromptContent( 115 messages=[ 116 LLMMessage(role="system", content="Version 1"), 117 ] 118 ) 119 version1 = workspace.prompts.bump_prompt_version(prompt.id, prompt_content1) 120 assert version1.version == 1 121 122 # Bump again (creates version 2) 123 prompt_content2 = MessagesPromptContent( 124 messages=[ 125 LLMMessage(role="system", content="Version 2"), 126 ] 127 ) 128 version2 = workspace.prompts.bump_prompt_version(prompt.id, prompt_content2) 129 assert version2.version == 2 130 131 # Get latest should be version 2 132 latest = workspace.prompts.get_version(prompt.id, "latest") 133 assert latest.version == 2 134 135 136 def test_workspace_prompts_update(workspace: Workspace, test_project_id): 137 """Test Workspace.prompts.update_prompt().""" 138 # Create a prompt 139 prompt = workspace.prompts.create_prompt(test_project_id, "original name") 140 141 # Update it 142 updated = Prompt( 143 id=prompt.id, 144 project_id=prompt.project_id, 145 name="updated name", 146 metadata=PromptMetadata(), 147 ) 148 workspace.prompts.update_prompt(updated) 149 150 # Verify update 151 retrieved = workspace.prompts.get_prompt_by_id(test_project_id, prompt.id) 152 assert retrieved.name == "updated name" 153 154 155 def test_workspace_prompts_delete(workspace: Workspace, test_project_id): 156 """Test Workspace.prompts.delete_prompt().""" 157 # Create a prompt 158 prompt = workspace.prompts.create_prompt(test_project_id, "to delete") 159 160 # Delete it 161 workspace.prompts.delete_prompt(prompt.id) 162 163 # Should not exist 164 prompts = workspace.prompts.list_prompts(test_project_id) 165 assert len(prompts) == 0 166 167 168 def test_workspace_prompts_delete_version(workspace: Workspace, test_project_id): 169 """Test Workspace.prompts.delete_version().""" 170 # Create a prompt with a version 171 prompt = workspace.prompts.create_prompt(test_project_id, "versioned prompt") 172 173 prompt_content = MessagesPromptContent( 174 messages=[ 175 LLMMessage(role="system", content="Test"), 176 ] 177 ) 178 version = workspace.prompts.create_version(prompt.id, 1, prompt_content) 179 180 # Delete the version 181 workspace.prompts.delete_version(version.id) 182 183 # Version should not exist 184 versions = workspace.prompts.list_versions(prompt.id) 185 assert len(versions) == 0 186 187 188 def test_workspace_prompts_get_version_by_id(workspace: Workspace, test_project_id): 189 """Test Workspace.prompts.get_version_by_id().""" 190 # Create a prompt with a version 191 prompt = workspace.prompts.create_prompt(test_project_id, "versioned prompt") 192 193 prompt_content = MessagesPromptContent( 194 messages=[ 195 LLMMessage(role="system", content="Test"), 196 ] 197 ) 198 version = workspace.prompts.create_version(prompt.id, 1, prompt_content) 199 200 # Get version by ID 201 retrieved = workspace.prompts.get_version_by_id(version.id) 202 assert retrieved.id == version.id 203 assert retrieved.version == 1 204 205 206 def test_workspace_prompts_multiple_prompts(workspace: Workspace, test_project_id): 207 """Test Workspace.prompts with multiple prompts.""" 208 # Create multiple prompts 209 workspace.prompts.create_prompt(test_project_id, "prompt 1") 210 workspace.prompts.create_prompt(test_project_id, "prompt 2") 211 workspace.prompts.create_prompt(test_project_id, "prompt 3") 212 213 # List all prompts 214 prompts = workspace.prompts.list_prompts(test_project_id) 215 assert len(prompts) == 3 216 217 # Verify all prompts are present 218 prompt_names = {p.name for p in prompts} 219 assert prompt_names == {"prompt 1", "prompt 2", "prompt 3"} 220 221 222 def test_workspace_prompts_multiple_versions(workspace: Workspace, test_project_id): 223 """Test Workspace.prompts with multiple versions.""" 224 # Create a prompt 225 prompt = workspace.prompts.create_prompt(test_project_id, "multi-version prompt") 226 227 # Create multiple versions 228 for i in range(1, 4): 229 prompt_content = MessagesPromptContent( 230 messages=[ 231 LLMMessage(role="system", content=f"Version {i}"), 232 ] 233 ) 234 workspace.prompts.create_version(prompt.id, i, prompt_content) 235 236 # List all versions 237 versions = workspace.prompts.list_versions(prompt.id) 238 assert len(versions) == 3 239 240 # Verify versions are in order 241 version_numbers = [v.version for v in versions] 242 assert version_numbers == [1, 2, 3] 243 244 # Get specific versions 245 v1 = workspace.prompts.get_version(prompt.id, 1) 246 v2 = workspace.prompts.get_version(prompt.id, 2) 247 v3 = workspace.prompts.get_version(prompt.id, 3) 248 249 assert v1.version == 1 250 assert v2.version == 2 251 assert v3.version == 3 252 253 # Latest should be version 3 254 latest = workspace.prompts.get_version(prompt.id, "latest") 255 assert latest.version == 3