fix(studio): fall back to TOS intranet endpoint

frontend/README.md

@@ -636,10 +636,15 @@ export VEADK_STUDIO_TOS_BUCKET=teststudio
 ```
 
 The server derives the provider-specific endpoint, such as
-`tos-cn-beijing.volces.com`, and never sends TOS credentials to the
-browser. Local Studio uses the configured Volcengine or BytePlus AK/SK; VeFaaS
-uses its IAM role credentials. Studio objects use the versioned, user-first
-layout
+`tos-cn-beijing.volces.com`, and never sends TOS credentials to the browser.
+For Volcengine, Studio probes the public endpoint once and automatically uses
+the matching `tos-<region>.ivolces.com` intranet endpoint when the public
+endpoint has a transport-level connection failure. Authentication, permission,
+and other TOS service errors do not trigger fallback. Browser-facing signed URLs
+continue to use the public endpoint. BytePlus and custom endpoints are left
+unchanged. Local Studio uses the configured Volcengine or BytePlus AK/SK;
+VeFaaS uses its IAM role credentials. Studio objects use the versioned,
+user-first layout
 `veadk-studio/v1/users/<encoded-user-id>/<namespace>/<scope>/<resource-id>/`.
 Video reference assets currently use the `video/<asset-role>/<asset-id>/`
 namespace and store `content` plus `metadata.json` below it.

frontend/server/evaluation_automation/__init__.py

@@ -22,6 +22,7 @@
 import httpx
 
 from frontend.server.storage import StudioProvider, StudioStorageConfig
+from frontend.server.storage.tos import create_tos_client_factory
 from veadk.utils.logger import get_logger
 
 from .datasets import ensure_feedback_sets
@@ -51,22 +52,9 @@ def create_service(
     models = StructuredEvaluationModels()
     storage = StudioStorageConfig.from_env(provider)
     if storage.configured and resolve_credentials is not None:
-
-        def tos_client() -> Any:
-            import tos
-
-            access_key, secret_key, session_token = resolve_credentials()
-            return tos.TosClientV2(
-                ak=access_key,
-                sk=secret_key,
-                security_token=session_token,
-                endpoint=storage.endpoint,
-                region=storage.region,
-            )
-
         optimizations = TosOptimizationRepository(
             bucket=storage.bucket,
-            client_factory=tos_client,
+            client_factory=create_tos_client_factory(storage, resolve_credentials),
         )
     else:
         logger.warning(

frontend/server/knowledge/uploads.py

@@ -92,7 +92,7 @@ def _delete_objects(client: Any, *, bucket: str, keys: tuple[str, ...]) -> None:
     for key in keys:
         try:
             client.delete_object(bucket=bucket, key=key)
-        except Exception as error:
+        except Exception as error:  # noqa: BLE001 - collect every failed delete
             errors.append(error)
     if not errors:
         return
@@ -297,6 +297,7 @@ def __init__(
         if self.max_file_bytes <= 0:
             raise ValueError("VEADK_KNOWLEDGE_MAX_FILE_BYTES must be positive")
         self._prepared_targets: set[tuple[str, str]] = set()
+        self._client_factories: dict[tuple[str, str, str], Callable[[], Any]] = {}
         self._lock = RLock()
         configured_account_id = str(
             environment.get("VEADK_STUDIO_ACCOUNT_ID") or ""
@@ -680,13 +681,22 @@ def _resolve_account_id(self, normalized_region: str) -> str:
             return account_id
 
     def _client(self, target: _UploadTarget) -> Any:
-        config = StudioStorageConfig(
-            provider=self._provider,
-            bucket=target.bucket,
-            region=target.region,
-            endpoint=target.endpoint,
-        )
-        return create_tos_client_factory(config, self._resolve_credentials)()
+        cache_key = (target.bucket, target.region, target.endpoint)
+        with self._lock:
+            factory = self._client_factories.get(cache_key)
+            if factory is None:
+                config = StudioStorageConfig(
+                    provider=self._provider,
+                    bucket=target.bucket,
+                    region=target.region,
+                    endpoint=target.endpoint,
+                )
+                factory = create_tos_client_factory(
+                    config,
+                    self._resolve_credentials,
+                )
+                self._client_factories[cache_key] = factory
+        return factory()
 
     def _prepare_target(self, client: Any, target: _UploadTarget) -> None:
         cache_key = (target.bucket, target.region)

frontend/server/skills/storage.py

@@ -25,6 +25,7 @@
 from typing import Any, Literal
 
 from frontend.server.storage import StudioProvider, StudioStorageConfig
+from frontend.server.storage.tos import create_tos_client_factory
 from veadk.utils.cloud_provider import cloud_provider_from_env
 
 _IAM_CREDENTIAL_PATH = Path("/var/run/secrets/iam/credential")
@@ -206,15 +207,20 @@ def _create_tos_client(
     storage: SkillPublishStorage,
     credentials: SkillPublishCredentials,
 ) -> Any:
-    import tos
-
-    return tos.TosClientV2(
-        ak=credentials.access_key,
-        sk=credentials.secret_key,
-        security_token=credentials.session_token,
-        endpoint=storage.endpoint,
+    config = StudioStorageConfig(
+        provider=storage.provider,
+        bucket=storage.bucket,
         region=storage.region,
+        endpoint=storage.endpoint,
     )
+    return create_tos_client_factory(
+        config,
+        lambda: (
+            credentials.access_key,
+            credentials.secret_key,
+            credentials.session_token,
+        ),
+    )()
 
 
 def _listed_buckets(client: Any) -> dict[str, str]:

frontend/server/storage/tos.py

@@ -17,32 +17,144 @@
 from __future__ import annotations
 
 from collections.abc import Callable
+from threading import Lock
 from typing import Any
 
+from veadk.utils.logger import get_logger
+
 from . import StudioStorageConfig
 
 CredentialResolver = Callable[[], tuple[str, str, str | None]]
 TosClientFactory = Callable[[], Any]
 
+_PUBLIC_ENDPOINT_PROBE_TIMEOUT_SECONDS = 3
+logger = get_logger(__name__)
+
+
+def _endpoint_candidates(config: StudioStorageConfig) -> tuple[str, ...]:
+    """Return safe endpoint candidates without overriding custom endpoints."""
+    public_endpoint = f"tos-{config.region}.volces.com"
+    if config.provider == "volcengine" and config.endpoint == public_endpoint:
+        return public_endpoint, f"tos-{config.region}.ivolces.com"
+    return (config.endpoint,)
+
+
+def _new_client(
+    tos_module: Any,
+    *,
+    endpoint: str,
+    region: str,
+    access_key: str,
+    secret_key: str,
+    session_token: str | None,
+    probe: bool = False,
+) -> Any:
+    options: dict[str, Any] = {
+        "ak": access_key,
+        "sk": secret_key,
+        "security_token": session_token,
+        "endpoint": endpoint,
+        "region": region,
+    }
+    client = tos_module.TosClientV2(**options)
+    if probe:
+        # Endpoint selection should not inherit the SDK's three retries and turn
+        # a predictable private-network fallback into a long cold-start delay.
+        client.max_retry_count = 0
+        client.connection_time = _PUBLIC_ENDPOINT_PROBE_TIMEOUT_SECONDS
+    return client
+
+
+def _is_network_error(error: Exception, tos_module: Any) -> bool:
+    """Return whether the SDK error represents a transport failure."""
+    client_error = getattr(
+        getattr(tos_module, "exceptions", None),
+        "TosClientError",
+        None,
+    )
+    if client_error is None or not isinstance(error, client_error):
+        return False
+    try:
+        import requests
+    except ImportError:
+        return False
+    return isinstance(
+        getattr(error, "cause", None),
+        requests.exceptions.RequestException,
+    )
+
 
 def create_tos_client_factory(
     config: StudioStorageConfig,
     resolve_credentials: CredentialResolver,
 ) -> TosClientFactory:
-    """Create clients lazily so refreshed temporary credentials are respected."""
+    """Create clients lazily and select a reachable Volcengine endpoint once."""
     if not config.configured:
         raise ValueError(config.unavailable_reason)
 
+    candidates = _endpoint_candidates(config)
+    selected_endpoint = candidates[0] if len(candidates) == 1 else ""
+    selection_lock = Lock()
+
     def factory() -> Any:
+        nonlocal selected_endpoint
         import tos
 
         access_key, secret_key, session_token = resolve_credentials()
-        return tos.TosClientV2(
-            ak=access_key,
-            sk=secret_key,
-            security_token=session_token,
-            endpoint=config.endpoint,
+        if not selected_endpoint:
+            with selection_lock:
+                if not selected_endpoint:
+                    public_endpoint = candidates[0]
+                    intranet_endpoint = f"tos-{config.region}.ivolces.com"
+                    probe = _new_client(
+                        tos,
+                        endpoint=public_endpoint,
+                        region=config.region,
+                        access_key=access_key,
+                        secret_key=secret_key,
+                        session_token=session_token,
+                        probe=True,
+                    )
+                    head_bucket = getattr(probe, "head_bucket", None)
+                    if not callable(head_bucket):
+                        selected_endpoint = public_endpoint
+                    else:
+                        try:
+                            head_bucket(bucket=config.bucket)
+                        except Exception as error:
+                            tos_exceptions = getattr(tos, "exceptions", None)
+                            expected_errors = tuple(
+                                error_type
+                                for error_type in (
+                                    getattr(tos_exceptions, "TosClientError", None),
+                                    getattr(tos_exceptions, "TosServerError", None),
+                                )
+                                if isinstance(error_type, type)
+                            )
+                            if not expected_errors or not isinstance(
+                                error, expected_errors
+                            ):
+                                raise
+                            if not _is_network_error(error, tos):
+                                selected_endpoint = public_endpoint
+                            else:
+                                selected_endpoint = intranet_endpoint
+                                logger.warning(
+                                    "Studio TOS public endpoint %s is unreachable; "
+                                    "using intranet endpoint %s.",
+                                    public_endpoint,
+                                    intranet_endpoint,
+                                )
+                        else:
+                            selected_endpoint = public_endpoint
+
+        return _new_client(
+            tos,
+            endpoint=selected_endpoint,
             region=config.region,
+            access_key=access_key,
+            secret_key=secret_key,
+            session_token=session_token,
         )
 
     return factory

frontend/server/video/storage.py

@@ -31,6 +31,7 @@
     StudioStorageConfig,
     StudioTosMediaStorage,
 )
+from frontend.server.storage.tos import create_tos_client_factory
 from veadk.multimodal.models import MediaRecord, MediaRef
 from veadk.multimodal.service import MediaService
 
@@ -246,16 +247,18 @@ def video_asset_repository_factory(
     if not config.configured:
         return None, max_bytes
 
+    client_factory = create_tos_client_factory(config, resolve_credentials)
+
     def factory() -> VideoAssetRepository:
-        access_key, secret_key, session_token = resolve_credentials()
         storage = StudioTosMediaStorage(
             bucket=config.bucket,
             region=config.region,
             endpoint=config.endpoint,
-            access_key=access_key,
-            secret_key=secret_key,
-            session_token=session_token or "",
+            access_key="",
+            secret_key="",
             key_prefix=STUDIO_STORAGE_ROOT_PREFIX,
+            client=client_factory(),
+            signed_url_endpoint=config.endpoint,
         )
         return VideoAssetRepository(MediaService(storage, max_file_bytes=max_bytes))
 

frontend/service/studio_scheduler/app.py

@@ -21,6 +21,9 @@
 from dataclasses import dataclass
 from typing import Any, cast
 
+from frontend.server.storage import StudioStorageConfig
+from frontend.server.storage.tos import create_tos_client_factory
+
 from .dispatcher import Dispatcher
 from .entrypoint import make_handler
 from .executor import ProviderRuntimeExecutor
@@ -130,19 +133,22 @@ def create_dispatcher(
 
 
 def _tos_client_factory(settings: SchedulerSettings) -> Callable[[], Any]:
-    def create() -> Any:
-        import tos
+    config = StudioStorageConfig(
+        provider=settings.provider,
+        bucket=settings.bucket,
+        region=settings.storage_region,
+        endpoint=settings.storage_endpoint,
+    )
 
+    def credentials() -> tuple[str, str, str | None]:
         credentials = resolve_service_credentials(settings.provider)
-        return tos.TosClientV2(
-            ak=credentials.access_key,
-            sk=credentials.secret_key,
-            security_token=credentials.session_token or None,
-            endpoint=settings.storage_endpoint,
-            region=settings.storage_region,
+        return (
+            credentials.access_key,
+            credentials.secret_key,
+            credentials.session_token or None,
         )
 
-    return create
+    return create_tos_client_factory(config, credentials)
 
 
 def handler(event: Any, context: Any) -> dict[str, int]: