agentic_systems_tracing.ipynb
1 { 2 "cells": [ 3 { 4 "cell_type": "markdown", 5 "id": "767f3682-1a00-4ca2-92db-8235879315ee", 6 "metadata": {}, 7 "source": [ 8 "# Agentic Systems: Tracing and Evaluation" 9 ] 10 }, 11 { 12 "cell_type": "code", 13 "execution_count": null, 14 "id": "1fbcabcc-b6ec-4ed4-9ae5-6c7c6a692084", 15 "metadata": {}, 16 "outputs": [], 17 "source": [ 18 "import openai\n", 19 "import tracely\n", 20 "import evidently.ui.runner as service_runner" 21 ] 22 }, 23 { 24 "cell_type": "markdown", 25 "id": "a23f3141-1636-4c66-baa8-c85044610740", 26 "metadata": {}, 27 "source": [ 28 "### Let's run evidently ui service in background" 29 ] 30 }, 31 { 32 "cell_type": "code", 33 "execution_count": null, 34 "id": "440d97b8-eff8-40e2-87bc-e15b30647375", 35 "metadata": {}, 36 "outputs": [], 37 "source": [ 38 "runner = service_runner.EvidentlyUIRunner()" 39 ] 40 }, 41 { 42 "cell_type": "code", 43 "execution_count": null, 44 "id": "d95faf4e-1334-4af2-b352-2d52b1ae8a86", 45 "metadata": {}, 46 "outputs": [], 47 "source": [ 48 "runner.run()" 49 ] 50 }, 51 { 52 "cell_type": "markdown", 53 "id": "54a954b7-05e4-4c83-b74a-1013735cf968", 54 "metadata": {}, 55 "source": [ 56 "### You can explore service directly in this notebook, or you can open it in the new tab using the button above" 57 ] 58 }, 59 { 60 "cell_type": "code", 61 "execution_count": null, 62 "id": "be900229-ffcb-4692-ab17-61c7d7306fa3", 63 "metadata": {}, 64 "outputs": [], 65 "source": [ 66 "runner.show_service()" 67 ] 68 }, 69 { 70 "cell_type": "markdown", 71 "id": "e757b64a-26ce-42e2-a33e-0e982d833e0a", 72 "metadata": {}, 73 "source": [ 74 "### Now you can get workspace" 75 ] 76 }, 77 { 78 "cell_type": "code", 79 "execution_count": null, 80 "id": "fec2ac97-3f70-4e2d-8c38-bbadc44d3aff", 81 "metadata": {}, 82 "outputs": [], 83 "source": [ 84 "workspace = runner.get_workspace()" 85 ] 86 }, 87 { 88 "cell_type": "markdown", 89 "id": "8e28fdcd-c4e1-41d1-9486-39d9f46a4dd6", 90 "metadata": {}, 91 "source": [ 92 "### Let's create a new project and show traces page" 93 ] 94 }, 95 { 96 "cell_type": "code", 97 "execution_count": null, 98 "id": "24a85ba6-0de8-45bb-855c-eb3a09485445", 99 "metadata": {}, 100 "outputs": [], 101 "source": [ 102 "project = workspace.create_project(name=\"Example project\")\n", 103 "project_id = project.id\n", 104 "\n", 105 "project_id" 106 ] 107 }, 108 { 109 "cell_type": "code", 110 "execution_count": null, 111 "id": "f74c8320-9013-4fab-9161-7715cb7c3c2f", 112 "metadata": {}, 113 "outputs": [], 114 "source": [ 115 "runner.show_service(service_runner.ServicePage(project_id=project_id, project_list='traces'))" 116 ] 117 }, 118 { 119 "cell_type": "markdown", 120 "id": "d22d82f9-2e5e-42f6-a543-dbbec27c509c", 121 "metadata": {}, 122 "source": [ 123 "### So, you don't have any traces yet.\n", 124 "### Let's setup tracing using `tracely`" 125 ] 126 }, 127 { 128 "cell_type": "code", 129 "execution_count": null, 130 "id": "686da536-3904-4427-8f34-c2a170bb6cbd", 131 "metadata": {}, 132 "outputs": [], 133 "source": [ 134 "provider = tracely.init_tracing(\n", 135 " address=runner.get_service_url(),\n", 136 " api_key=\"\",\n", 137 " project_id=project_id,\n", 138 " export_name=\"Tweets\",\n", 139 ")" 140 ] 141 }, 142 { 143 "cell_type": "markdown", 144 "id": "450fa3ff949f806b", 145 "metadata": {}, 146 "source": [ 147 "### (Optional) Instrument calls openinfrence" 148 ] 149 }, 150 { 151 "cell_type": "code", 152 "execution_count": null, 153 "id": "be6e66e7d084e7f9", 154 "metadata": {}, 155 "outputs": [], 156 "source": [ 157 "!pip install openinference-instrumentation-openai" 158 ] 159 }, 160 { 161 "cell_type": "code", 162 "execution_count": null, 163 "id": "ec38a8ac95e632f", 164 "metadata": {}, 165 "outputs": [], 166 "source": [ 167 "from openinference.instrumentation.openai import OpenAIInstrumentor\n", 168 "\n", 169 "OpenAIInstrumentor().instrument(tracer_provider=provider)" 170 ] 171 }, 172 { 173 "cell_type": "markdown", 174 "id": "2652a8b6-94a2-412b-9821-0bbe22b51b77", 175 "metadata": {}, 176 "source": [ 177 "### Let's generate some tweets using OpenAI LLM" 178 ] 179 }, 180 { 181 "cell_type": "code", 182 "execution_count": null, 183 "id": "e3809a57-13bf-4804-b087-e53c47086a91", 184 "metadata": {}, 185 "outputs": [], 186 "source": [ 187 "@tracely.trace_event()\n", 188 "def basic_tweet_generation(topic, model=\"gpt-3.5-turbo\", instructions=\"\"):\n", 189 " response = openai.chat.completions.create(\n", 190 " model=model,\n", 191 " messages=[\n", 192 " {\"role\": \"system\", \"content\": instructions},\n", 193 " {\"role\": \"user\", \"content\": f\"Write a short paragraph about {topic}\"}\n", 194 " ]\n", 195 " )\n", 196 " return response.choices[0].message.content\n", 197 "\n", 198 "@tracely.trace_event()\n", 199 "def generate_tweet(topic):\n", 200 " return basic_tweet_generation(topic)" 201 ] 202 }, 203 { 204 "cell_type": "code", 205 "execution_count": null, 206 "id": "750cc20f-ba75-4b62-9934-56a875743189", 207 "metadata": {}, 208 "outputs": [], 209 "source": [ 210 "interesting_topics = [\n", 211 " \"Artificial Intelligence and Consciousness\",\n", 212 " \"Quantum Computing Applications\",\n", 213 " \"Space Exploration and Exoplanets\",\n", 214 " \"Climate Change Solutions\",\n", 215 " \"Neuroscience and Brain Mapping\",\n", 216 " \"Ancient Civilizations and Lost Cities\",\n", 217 " \"Renewable Energy Revolution\",\n", 218 " \"Gene Editing and CRISPR\",\n", 219 " \"Virtual Reality and the Metaverse\",\n", 220 " \"Ocean Exploration and Deep Sea Life\",\n", 221 " \"Cryptocurrency and Blockchain\",\n", 222 " \"Meditation and Mindfulness Science\",\n", 223 " \"Robotics and Human-Robot Interaction\",\n", 224 " \"Food Science and Future Nutrition\",\n", 225 " \"Urban Planning and Smart Cities\",\n", 226 "]" 227 ] 228 }, 229 { 230 "cell_type": "code", 231 "execution_count": null, 232 "id": "d72ee312-6527-410f-910a-8ed530fd0f3e", 233 "metadata": {}, 234 "outputs": [], 235 "source": [ 236 "for topic in interesting_topics:\n", 237 " generate_tweet(topic)" 238 ] 239 }, 240 { 241 "cell_type": "markdown", 242 "id": "43d9cf99-bd14-4bda-8552-3e261b88d869", 243 "metadata": {}, 244 "source": [ 245 "### Now we can explore traces in the UI" 246 ] 247 }, 248 { 249 "cell_type": "code", 250 "execution_count": null, 251 "id": "515bd729-90e0-440e-94a5-e40b562388c4", 252 "metadata": {}, 253 "outputs": [], 254 "source": [ 255 "export_id = tracely.get_info()['export_id']" 256 ] 257 }, 258 { 259 "cell_type": "code", 260 "execution_count": null, 261 "id": "034c6f9c-7f52-405e-815c-0d4b2fa14c9f", 262 "metadata": {}, 263 "outputs": [], 264 "source": [ 265 "runner.show_service(page=service_runner.ServicePage(project_id=project_id, export_id=export_id))" 266 ] 267 }, 268 { 269 "cell_type": "markdown", 270 "id": "4c3c14f4-430d-4502-8ebf-880df64747eb", 271 "metadata": {}, 272 "source": [ 273 "### And load this tracing data as dataset to this notebook" 274 ] 275 }, 276 { 277 "cell_type": "code", 278 "execution_count": null, 279 "id": "2dd0d564-12dc-4fb7-9a73-ac0f3b527606", 280 "metadata": {}, 281 "outputs": [], 282 "source": [ 283 "dataset = workspace.load_dataset(dataset_id=export_id)" 284 ] 285 }, 286 { 287 "cell_type": "code", 288 "execution_count": null, 289 "id": "85fed394-f1e4-4a59-9f46-2e16d018716e", 290 "metadata": {}, 291 "outputs": [], 292 "source": [ 293 "dataset.as_dataframe()" 294 ] 295 }, 296 { 297 "cell_type": "markdown", 298 "id": "97ec638b-3041-46a2-a206-cb87142cd9a2", 299 "metadata": {}, 300 "source": [ 301 "### Terminate ui service " 302 ] 303 }, 304 { 305 "cell_type": "code", 306 "execution_count": null, 307 "id": "9aa30107-ea95-46b0-a0e9-fec3613a4bb9", 308 "metadata": {}, 309 "outputs": [], 310 "source": [ 311 "runner.terminate()" 312 ] 313 } 314 ], 315 "metadata": { 316 "kernelspec": { 317 "display_name": "Python 3 (ipykernel)", 318 "language": "python", 319 "name": "python3" 320 }, 321 "language_info": { 322 "codemirror_mode": { 323 "name": "ipython", 324 "version": 3 325 }, 326 "file_extension": ".py", 327 "mimetype": "text/x-python", 328 "name": "python", 329 "nbconvert_exporter": "python", 330 "pygments_lexer": "ipython3", 331 "version": "3.13.11" 332 } 333 }, 334 "nbformat": 4, 335 "nbformat_minor": 5 336 }