diff --git a/packages/traceloop-sdk/pyproject.toml b/packages/traceloop-sdk/pyproject.toml index d7e3680b98..4028f68cae 100644 --- a/packages/traceloop-sdk/pyproject.toml +++ b/packages/traceloop-sdk/pyproject.toml @@ -62,7 +62,7 @@ dependencies = [ "aiohttp>=3.11.11,<4", "requests>=2.31.0,<3", "httpx>=0.23.0,<1", - "cuid>=0.4,<0.5", + "cuid2>=2.0.1,<3", ] [project.urls] @@ -136,7 +136,7 @@ exclude = [ [[tool.mypy.overrides]] module = [ - "cuid.*", + "cuid2.*", "posthog.*", "opentelemetry.semconv_ai.*", ] diff --git a/packages/traceloop-sdk/tests/experiment/test_experiment.py b/packages/traceloop-sdk/tests/experiment/test_experiment.py index 1e85323301..b8a3d87aa9 100644 --- a/packages/traceloop-sdk/tests/experiment/test_experiment.py +++ b/packages/traceloop-sdk/tests/experiment/test_experiment.py @@ -1,6 +1,11 @@ +import re + import pytest from unittest.mock import Mock, AsyncMock -from traceloop.sdk.experiment.experiment import Experiment +from traceloop.sdk.experiment.experiment import ( + Experiment, + _EXPERIMENT_SLUG_CUID, +) from traceloop.sdk.client.http import HTTPClient from traceloop.sdk.evaluator.config import EvaluatorDetails @@ -369,3 +374,54 @@ async def task_any_output(row): # Should succeed because string evaluators don't trigger validation assert len(errors) == 0 assert len(results) == 1 + + +class TestGeneratedExperimentSlug: + """The slug the SDK generates when the caller does not pass one.""" + + def test_shape_is_exp_plus_eleven_lowercase_alphanumerics(self): + """The old code hand-truncated a cuid to 11 characters. + + The backend keys experiment runs by this slug, so the generated part + has to keep that width and alphabet whichever generator produces it. + """ + slugs = {_EXPERIMENT_SLUG_CUID.generate() for _ in range(500)} + + assert len(slugs) == 500 + for slug in slugs: + assert re.fullmatch(r"[a-z0-9]{11}", slug), slug + + @pytest.mark.anyio(backends=["asyncio"]) + async def test_generated_slug_reaches_the_experiment_and_the_tasks(self): + mock_http_client = Mock(spec=HTTPClient) + mock_http_client.base_url = "https://api.example.com" + mock_async_http_client = Mock() + + mock_experiment_response = Mock() + mock_experiment_response.run.id = "run-123" + mock_experiment_response.experiment.id = "exp-456" + + experiment = Experiment(mock_http_client, mock_async_http_client, None) + experiment._init_experiment = Mock(return_value=mock_experiment_response) + experiment._create_task = Mock(return_value=Mock(id="task-1")) + experiment._datasets.get_version_jsonl = Mock( + return_value='{"columns":{}}\n{"input": "test"}' + ) + + async def task(row): + return {"output": "ok"} + + results, errors = await experiment._run_locally( + task=task, + evaluators=[], + dataset_slug="test-dataset", + dataset_version="v1", + ) + + assert errors == [] + assert len(results) == 1 + + slug = experiment._last_experiment_slug + assert re.fullmatch(r"exp-[a-z0-9]{11}", slug), slug + assert experiment._init_experiment.call_args.args[0] == slug + assert experiment._create_task.call_args.kwargs["experiment_slug"] == slug diff --git a/packages/traceloop-sdk/traceloop/sdk/experiment/experiment.py b/packages/traceloop-sdk/traceloop/sdk/experiment/experiment.py index 08f4976ff0..ec6e0b0411 100644 --- a/packages/traceloop-sdk/traceloop/sdk/experiment/experiment.py +++ b/packages/traceloop-sdk/traceloop/sdk/experiment/experiment.py @@ -1,4 +1,3 @@ -import cuid import asyncio import json import os @@ -21,6 +20,12 @@ from traceloop.sdk.evaluator.config import EvaluatorDetails import httpx +from cuid2 import Cuid + +# The backend keys experiment runs by this slug. The old call was +# str(cuid.cuid())[:11], so the generated part stays 11 characters wide. +_EXPERIMENT_SLUG_CUID = Cuid(length=11) + class Experiment: """Main Experiment class for creating experiment contexts""" @@ -122,7 +127,7 @@ async def _run_locally( """ if not experiment_slug: - experiment_slug = self._experiment_slug or "exp-" + str(cuid.cuid())[:11] + experiment_slug = self._experiment_slug or "exp-" + _EXPERIMENT_SLUG_CUID.generate() self._last_experiment_slug = experiment_slug experiment_run_metadata = { @@ -307,7 +312,7 @@ async def _run_in_github( ) if not experiment_slug: - experiment_slug = self._experiment_slug or "exp-" + str(cuid.cuid())[:11] + experiment_slug = self._experiment_slug or "exp-" + _EXPERIMENT_SLUG_CUID.generate() self._last_experiment_slug = experiment_slug # Fetch dataset rows diff --git a/packages/traceloop-sdk/uv.lock b/packages/traceloop-sdk/uv.lock index c262ac749b..0afdbf2ad7 100644 --- a/packages/traceloop-sdk/uv.lock +++ b/packages/traceloop-sdk/uv.lock @@ -419,10 +419,13 @@ wheels = [ ] [[package]] -name = "cuid" -version = "0.4" +name = "cuid2" +version = "2.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/ca/d323556e2bf9bfb63219fbb849ce61bb830cc42d1b25b91cde3815451b91/cuid-0.4.tar.gz", hash = "sha256:74eaba154916a2240405c3631acee708c263ef8fa05a86820b87d0f59f84e978", size = 4986, upload-time = "2023-03-06T00:41:12.708Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/63/97ffc74f33e5a5f913bf073e8250a5b5a64d52d411b09a9c36c902db2cc4/cuid2-2.0.1.tar.gz", hash = "sha256:8d262eb467c16b81419361e18e47f41da77c4446dd2cf0640eac2616680bc924", size = 7033, upload-time = "2024-04-16T23:51:52.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/d2/90fce0050c5a9196d259ac8f0c4720c69ec6b5a322612edf3892f0036c5d/cuid2-2.0.1-py3-none-any.whl", hash = "sha256:943bdf86dc3ed07f32253e1be6e3c34dda8c7bda1c453f851f4ebaaa5a2dcfbf", size = 8154, upload-time = "2024-04-16T23:51:50.953Z" }, +] [[package]] name = "datamodel-code-generator" @@ -3909,7 +3912,7 @@ source = { editable = "." } dependencies = [ { name = "aiohttp" }, { name = "colorama" }, - { name = "cuid" }, + { name = "cuid2" }, { name = "deprecated" }, { name = "httpx" }, { name = "jinja2" }, @@ -3994,7 +3997,7 @@ test = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.11.11,<4" }, { name = "colorama", specifier = ">=0.4.6,<0.5.0" }, - { name = "cuid", specifier = ">=0.4,<0.5" }, + { name = "cuid2", specifier = ">=2.0.1,<3" }, { name = "deprecated", specifier = ">=1.2.14,<2" }, { name = "httpx", specifier = ">=0.23.0,<1" }, { name = "jinja2", specifier = ">=3.1.5,<4" },