Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/traceloop-sdk/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ dependencies = [
"aiohttp>=3.11.11,<4",
"requests>=2.31.0,<3",
"httpx>=0.23.0,<1",
"cuid>=0.4,<0.5",
"cuid2>=2.0.1,<3",
]

[project.urls]
Expand Down Expand Up @@ -136,7 +136,7 @@ exclude = [

[[tool.mypy.overrides]]
module = [
"cuid.*",
"cuid2.*",
"posthog.*",
"opentelemetry.semconv_ai.*",
]
Expand Down
58 changes: 57 additions & 1 deletion packages/traceloop-sdk/tests/experiment/test_experiment.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import re

import pytest
from unittest.mock import Mock, AsyncMock
from traceloop.sdk.experiment.experiment import Experiment
from traceloop.sdk.experiment.experiment import (
Experiment,
_EXPERIMENT_SLUG_CUID,
)
from traceloop.sdk.client.http import HTTPClient
from traceloop.sdk.evaluator.config import EvaluatorDetails

Expand Down Expand Up @@ -369,3 +374,54 @@ async def task_any_output(row):
# Should succeed because string evaluators don't trigger validation
assert len(errors) == 0
assert len(results) == 1


class TestGeneratedExperimentSlug:
"""The slug the SDK generates when the caller does not pass one."""

def test_shape_is_exp_plus_eleven_lowercase_alphanumerics(self):
"""The old code hand-truncated a cuid to 11 characters.

The backend keys experiment runs by this slug, so the generated part
has to keep that width and alphabet whichever generator produces it.
"""
slugs = {_EXPERIMENT_SLUG_CUID.generate() for _ in range(500)}

assert len(slugs) == 500
for slug in slugs:
assert re.fullmatch(r"[a-z0-9]{11}", slug), slug

@pytest.mark.anyio(backends=["asyncio"])
async def test_generated_slug_reaches_the_experiment_and_the_tasks(self):
mock_http_client = Mock(spec=HTTPClient)
mock_http_client.base_url = "https://api.example.com"
mock_async_http_client = Mock()

mock_experiment_response = Mock()
mock_experiment_response.run.id = "run-123"
mock_experiment_response.experiment.id = "exp-456"

experiment = Experiment(mock_http_client, mock_async_http_client, None)
experiment._init_experiment = Mock(return_value=mock_experiment_response)
experiment._create_task = Mock(return_value=Mock(id="task-1"))
experiment._datasets.get_version_jsonl = Mock(
return_value='{"columns":{}}\n{"input": "test"}'
)

async def task(row):
return {"output": "ok"}

results, errors = await experiment._run_locally(
task=task,
evaluators=[],
dataset_slug="test-dataset",
dataset_version="v1",
)

assert errors == []
assert len(results) == 1

slug = experiment._last_experiment_slug
assert re.fullmatch(r"exp-[a-z0-9]{11}", slug), slug
assert experiment._init_experiment.call_args.args[0] == slug
assert experiment._create_task.call_args.kwargs["experiment_slug"] == slug
11 changes: 8 additions & 3 deletions packages/traceloop-sdk/traceloop/sdk/experiment/experiment.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import cuid
import asyncio
import json
import os
Expand All @@ -21,6 +20,12 @@
from traceloop.sdk.evaluator.config import EvaluatorDetails
import httpx

from cuid2 import Cuid

# The backend keys experiment runs by this slug. The old call was
# str(cuid.cuid())[:11], so the generated part stays 11 characters wide.
_EXPERIMENT_SLUG_CUID = Cuid(length=11)


class Experiment:
"""Main Experiment class for creating experiment contexts"""
Expand Down Expand Up @@ -122,7 +127,7 @@ async def _run_locally(
"""

if not experiment_slug:
experiment_slug = self._experiment_slug or "exp-" + str(cuid.cuid())[:11]
experiment_slug = self._experiment_slug or "exp-" + _EXPERIMENT_SLUG_CUID.generate()
self._last_experiment_slug = experiment_slug

experiment_run_metadata = {
Expand Down Expand Up @@ -307,7 +312,7 @@ async def _run_in_github(
)

if not experiment_slug:
experiment_slug = self._experiment_slug or "exp-" + str(cuid.cuid())[:11]
experiment_slug = self._experiment_slug or "exp-" + _EXPERIMENT_SLUG_CUID.generate()
self._last_experiment_slug = experiment_slug

# Fetch dataset rows
Expand Down
13 changes: 8 additions & 5 deletions packages/traceloop-sdk/uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading