From 2630753fe47554f142cad35f4923de7588b52337 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:15:03 +0100 Subject: [PATCH 1/2] fix(operator): report LLMModel gateway apply failures instead of claiming Ready LLMModel logged and skipped a failed InferencePool or AIGatewayRoute apply, then reported Ready. A missing AI Gateway or Inference Extension CRD left a model that looked healthy and returned 404 or 500 at request time, with nothing in its status to explain it. Report each apply as a condition, InferencePoolReady, ExternalEndpointReady and InternalEndpointReady, using the reasons the PassthroughModel reconciler already uses: GatewayCRDUnavailable for a missing CRD, ApplyFailed otherwise, EndpointDisabled for an endpoint the spec turned off. A failure turns Ready into Degraded and requeues the reconcile, after 15 seconds for a missing CRD or a minute otherwise. On an LLMModel the endpoint conditions cover the route only; the policies keep their own SecurityPoliciesReady condition, applied earlier in the reconcile so key revocation isn't held up while a model starts. updateStatus now sets conditions individually instead of replacing the list, so SecurityPoliciesReady survives the final status write. The shared condition constants and helpers move from the PassthroughModel controller to conditions.go. Closes #109. --- docs/src/content/docs/architecture.mdx | 13 +- docs/src/content/docs/configuration.mdx | 2 +- operator/internal/controller/conditions.go | 137 +++++++++++++ .../internal/controller/conditions_test.go | 183 ++++++++++++++++++ .../controller/gateway_conditions_test.go | 97 ++++++++++ .../controller/llmmodel_controller.go | 100 +++++++--- .../controller/llmmodel_controller_test.go | 63 ++++++ .../controller/passthroughmodel_controller.go | 40 +--- 8 files changed, 564 insertions(+), 71 deletions(-) create mode 100644 operator/internal/controller/conditions.go create mode 100644 operator/internal/controller/conditions_test.go create mode 100644 operator/internal/controller/gateway_conditions_test.go diff --git a/docs/src/content/docs/architecture.mdx b/docs/src/content/docs/architecture.mdx index e3e05242..45a4f38a 100644 --- a/docs/src/content/docs/architecture.mdx +++ b/docs/src/content/docs/architecture.mdx @@ -231,6 +231,17 @@ policy writes set `ApplyFailed` and trigger controller retries. A missing while policy creation is retried every 15 seconds. `Applied` means the policies were written, not that Envoy has finished applying them to traffic. +Once the model is serving, the reconciler applies its `InferencePool` and +`AIGatewayRoute`s and reports each outcome: `InferencePoolReady`, +`ExternalEndpointReady` and `InternalEndpointReady`. On an `LLMModel` the +endpoint conditions cover the route only; the policies are reported by +`SecurityPoliciesReady`. A disabled endpoint reports `False` with +`EndpointDisabled`, which is not a failure. A failed apply does not stop the +reconcile: the condition reads `GatewayCRDUnavailable` (retried every 15 +seconds) or `ApplyFailed` (retried every minute), and a model that would +otherwise be `Ready` reports `Degraded`, because its pods serve but nothing +can route to them. + ### Internal endpoint For JupyterLab, in-cluster chat UIs, notebooks. @@ -307,7 +318,7 @@ Semantics worth knowing: - **`modelsOwnedBy` is set on both routes.** Each endpoint's `/v1/models` aggregates only the routes declaring its own hostname, so each endpoint carries its own catalog without depending on the other endpoint being enabled. - **Catch-all conflicts are not validated.** Two catch-all PassthroughModels on the same gateway race for unclaimed model ids the same way two identical HTTPRoutes would; the webhook validates per-CR only, consistent with the LLMModel webhook dropping cross-CR collision checks. Document one catch-all per cluster as the supported shape. - **Names are unique across both kinds.** Both an LLMModel and a PassthroughModel derive their api-keys Secret name as `-api-keys`, so sharing a name would make the two controllers fight over one Secret. Both webhooks reject a CR whose name is already taken by the other kind in the same namespace. The key-manager cache also keeps kind-prefixed entries as defense in depth. -- **Status.** Phase is `Ready`/`Error` with conditions `BackendConfigured`, `ExternalEndpointReady`, and `InternalEndpointReady`. Secret-backed providers also report the informational `UpstreamCredentialResolved` condition without blocking `Ready`: `False` with `SecretNotFound` or `APIKeyMissing` means the Secret or its non-empty `apiKey` entry is missing; `Unknown` with `LookupFailed` means the read failed and is retried after one minute. Workload identity has no upstream Secret, so this condition is omitted. Missing AI Gateway CRDs surface as `GatewayCRDUnavailable` with a 15-second requeue; real apply failures report `ApplyFailed` with a one-minute requeue. Both controllers share the reason string and the interval, but the LLMModel reconciler applies this convention to its `SecurityPolicy` only: a failed `AIGatewayRoute` apply is logged and skipped with no condition and no requeue, so a served model can report `Ready` while unroutable. Converging the remaining LLMModel applies is tracked in [#109](https://github.com/nebari-dev/llm-serving-pack/issues/109). +- **Status.** Phase is `Ready`/`Error` with conditions `BackendConfigured`, `ExternalEndpointReady`, and `InternalEndpointReady`. Secret-backed providers also report the informational `UpstreamCredentialResolved` condition without blocking `Ready`: `False` with `SecretNotFound` or `APIKeyMissing` means the Secret or its non-empty `apiKey` entry is missing; `Unknown` with `LookupFailed` means the read failed and is retried after one minute. Workload identity has no upstream Secret, so this condition is omitted. Missing AI Gateway CRDs surface as `GatewayCRDUnavailable` with a 15-second requeue; real apply failures report `ApplyFailed` with a one-minute requeue. Both controllers share the reasons and the intervals. On a PassthroughModel each endpoint condition covers its route and its policy together. - **Generated gateway resources are reconciled on-change-only.** Because the AI Gateway kinds (`Backend`, `BackendTLSPolicy`, `AIServiceBackend`, `BackendSecurityPolicy`, `AIGatewayRoute`, `SecurityPolicy`) are applied as unstructured objects, `SetupWithManager` registers `For(&PassthroughModel{})`, a watch on operator-managed api-keys Secrets, and a targeted watch that maps provider credential Secret events to the PassthroughModels that reference them. It does not `Owns(...)` the generated gateway resources. If a generated resource is deleted out-of-band, it is not recreated until the next PassthroughModel edit, api-keys Secret change in the namespace, or referenced provider credential Secret change. --- diff --git a/docs/src/content/docs/configuration.mdx b/docs/src/content/docs/configuration.mdx index 089b856e..1987f32d 100644 --- a/docs/src/content/docs/configuration.mdx +++ b/docs/src/content/docs/configuration.mdx @@ -363,7 +363,7 @@ The operator writes these fields to `status`; they are read-only. | `status.endpoints.external` | string | Actual external endpoint URL once provisioned. | | `status.endpoints.internal` | string | Actual internal endpoint URL once provisioned. | | `status.modelSize` | string | Actual model size after download. | -| `status.conditions` | []Condition | Standard Kubernetes conditions (type, status, reason, message). | +| `status.conditions` | []Condition | `SecurityPoliciesReady`, `InferencePoolReady`, `ExternalEndpointReady` and `InternalEndpointReady` (type, status, reason, message). A failed gateway apply turns `Ready` into `Degraded`; see [Architecture](/architecture/#dual-endpoint-auth). | --- diff --git a/operator/internal/controller/conditions.go b/operator/internal/controller/conditions.go new file mode 100644 index 00000000..fbd01cfe --- /dev/null +++ b/operator/internal/controller/conditions.go @@ -0,0 +1,137 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + llmv1alpha1 "github.com/nebari-dev/nebari-llm-serving-pack/operator/api/v1alpha1" +) + +// Status condition types shared by the LLMModel and PassthroughModel +// reconcilers. +// +// Both controllers apply gateway resources whose CRDs are installed +// separately from this pack (section 4 of the install runbook). A missing CRD +// does not fail a reconcile: everything the pack owns still converges, and the +// applies succeed on a later pass once the CRDs exist. It must be visible, +// though, because a route that was never created shows up only as an HTTP 404 +// or 500 at request time. +const ( + // CondBackendConfigured covers a PassthroughModel's provider plumbing: + // Backend, BackendTLSPolicy, AIServiceBackend, BackendSecurityPolicy. + CondBackendConfigured = "BackendConfigured" + // CondExternalEndpointReady covers the external endpoint. On a + // PassthroughModel that is the route and its apiKeyAuth policy; on an + // LLMModel it is the route only, because the policy is applied earlier and + // reported by SecurityPoliciesReady. + CondExternalEndpointReady = "ExternalEndpointReady" + // CondInternalEndpointReady covers the internal endpoint: the route and its + // JWT policy on a PassthroughModel, the route only on an LLMModel. + CondInternalEndpointReady = "InternalEndpointReady" + // CondInferencePoolReady covers the InferencePool an LLMModel needs. It is + // separate from the endpoint conditions because it comes from a different + // CRD bundle: a cluster can have the AI Gateway CRDs and not the Inference + // Extension's, and the llm-d End-Point Picker crashloops without + // InferencePool while routing looks fine. + CondInferencePoolReady = "InferencePoolReady" +) + +const ( + reasonApplied = "Applied" + reasonDisabled = "EndpointDisabled" + // applyFailedRequeue is how long to wait before retrying a gateway apply + // that failed for a reason other than a missing CRD. + applyFailedRequeue = time.Minute +) + +// conditionFor turns the outcome of an apply into a status condition. A nil +// err reports Applied with okMessage. A missing CRD reports +// GatewayCRDUnavailable; any other error reports ApplyFailed. Both carry the +// error text, which is where a reader learns which resource failed. +func conditionFor(condType string, err error, okMessage string) metav1.Condition { + if err != nil { + reason := reasonApplyFailed + if meta.IsNoMatchError(err) { + reason = reasonGatewayCRDUnavailable + } + return metav1.Condition{ + Type: condType, + Status: metav1.ConditionFalse, + Reason: reason, + Message: err.Error(), + } + } + return metav1.Condition{ + Type: condType, + Status: metav1.ConditionTrue, + Reason: reasonApplied, + Message: okMessage, + } +} + +// disabledCondition reports an endpoint the spec switched off. It is False +// like a failure, but for a deliberate reason, and hasGatewayFailure does not +// count it. +func disabledCondition(condType string) metav1.Condition { + return metav1.Condition{ + Type: condType, + Status: metav1.ConditionFalse, + Reason: reasonDisabled, + Message: "endpoint disabled in spec", + } +} + +// hasGatewayFailure reports whether any condition records a gateway apply +// that failed or is waiting on a missing CRD. +func hasGatewayFailure(conds []metav1.Condition) bool { + return gatewayRequeue(conds) > 0 +} + +// gatewayRequeue returns how soon to retry after the gateway applies recorded +// in conds, or 0 when none failed. A missing CRD retries on the shorter CRD +// interval, since installing the CRD is enough for the next pass to succeed. +func gatewayRequeue(conds []metav1.Condition) time.Duration { + var requeue time.Duration + for _, c := range conds { + if c.Status != metav1.ConditionFalse { + continue + } + switch c.Reason { + case reasonGatewayCRDUnavailable: + return gatewayCRDRequeue + case reasonApplyFailed: + requeue = applyFailedRequeue + } + } + return requeue +} + +// phaseWithGatewayFailure downgrades a Ready model to Degraded when one of +// its gateway resources could not be applied. The workload is serving but +// nothing can route to it, which is what Degraded describes; Error would +// overstate it. Other phases already say something more specific and are +// left alone. +func phaseWithGatewayFailure(phase llmv1alpha1.LLMModelPhase, conds []metav1.Condition) llmv1alpha1.LLMModelPhase { + if phase == llmv1alpha1.PhaseReady && hasGatewayFailure(conds) { + return llmv1alpha1.PhaseDegraded + } + return phase +} diff --git a/operator/internal/controller/conditions_test.go b/operator/internal/controller/conditions_test.go new file mode 100644 index 00000000..f9f57238 --- /dev/null +++ b/operator/internal/controller/conditions_test.go @@ -0,0 +1,183 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "errors" + "testing" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + llmv1alpha1 "github.com/nebari-dev/nebari-llm-serving-pack/operator/api/v1alpha1" +) + +func TestConditionFor(t *testing.T) { + noMatch := &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "aigateway.envoyproxy.io", Kind: "AIGatewayRoute"}} + + tests := []struct { + name string + err error + wantStatus metav1.ConditionStatus + wantReason string + wantMessage string + }{ + { + name: "no error reports Applied with the ok message", + err: nil, + wantStatus: metav1.ConditionTrue, + wantReason: reasonApplied, + wantMessage: "route applied", + }, + { + name: "missing CRD reports GatewayCRDUnavailable", + err: noMatch, + wantStatus: metav1.ConditionFalse, + wantReason: reasonGatewayCRDUnavailable, + wantMessage: noMatch.Error(), + }, + { + name: "other errors report ApplyFailed and carry the message", + err: errors.New("admission webhook denied the request"), + wantStatus: metav1.ConditionFalse, + wantReason: reasonApplyFailed, + wantMessage: "admission webhook denied the request", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := conditionFor(CondExternalEndpointReady, tt.err, "route applied") + if got.Type != CondExternalEndpointReady { + t.Errorf("Type = %q, want %q", got.Type, CondExternalEndpointReady) + } + if got.Status != tt.wantStatus { + t.Errorf("Status = %q, want %q", got.Status, tt.wantStatus) + } + if got.Reason != tt.wantReason { + t.Errorf("Reason = %q, want %q", got.Reason, tt.wantReason) + } + if got.Message != tt.wantMessage { + t.Errorf("Message = %q, want %q", got.Message, tt.wantMessage) + } + }) + } +} + +func TestGatewayRequeue(t *testing.T) { + applied := metav1.Condition{Type: CondExternalEndpointReady, Status: metav1.ConditionTrue, Reason: reasonApplied} + failed := metav1.Condition{Type: CondInternalEndpointReady, Status: metav1.ConditionFalse, Reason: reasonApplyFailed} + crdMissing := metav1.Condition{Type: CondInferencePoolReady, Status: metav1.ConditionFalse, Reason: reasonGatewayCRDUnavailable} + disabled := disabledCondition(CondInternalEndpointReady) + + tests := []struct { + name string + conds []metav1.Condition + wantFailed bool + wantRequeue time.Duration + }{ + {name: "no conditions", conds: nil, wantFailed: false, wantRequeue: 0}, + {name: "all applied", conds: []metav1.Condition{applied}, wantFailed: false, wantRequeue: 0}, + { + // A disabled endpoint is False, but deliberately so. Treating it as a + // failure would degrade every model that turns an endpoint off. + name: "disabled endpoint is not a failure", + conds: []metav1.Condition{applied, disabled}, + wantFailed: false, + wantRequeue: 0, + }, + {name: "apply failure retries after a minute", conds: []metav1.Condition{applied, failed}, wantFailed: true, wantRequeue: time.Minute}, + {name: "missing CRD retries on the CRD interval", conds: []metav1.Condition{crdMissing}, wantFailed: true, wantRequeue: gatewayCRDRequeue}, + { + // A missing CRD is the faster case to recover from (install it and + // the next pass succeeds), so it sets the interval when both occur. + name: "missing CRD wins over an apply failure", + conds: []metav1.Condition{failed, crdMissing}, + wantFailed: true, + wantRequeue: gatewayCRDRequeue, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hasGatewayFailure(tt.conds); got != tt.wantFailed { + t.Errorf("hasGatewayFailure() = %v, want %v", got, tt.wantFailed) + } + if got := gatewayRequeue(tt.conds); got != tt.wantRequeue { + t.Errorf("gatewayRequeue() = %v, want %v", got, tt.wantRequeue) + } + }) + } +} + +func TestPhaseWithGatewayFailure(t *testing.T) { + failed := []metav1.Condition{{Type: CondExternalEndpointReady, Status: metav1.ConditionFalse, Reason: reasonApplyFailed}} + crdMissing := []metav1.Condition{{Type: CondInferencePoolReady, Status: metav1.ConditionFalse, Reason: reasonGatewayCRDUnavailable}} + ok := []metav1.Condition{{Type: CondExternalEndpointReady, Status: metav1.ConditionTrue, Reason: reasonApplied}} + + tests := []struct { + name string + phase llmv1alpha1.LLMModelPhase + conds []metav1.Condition + want llmv1alpha1.LLMModelPhase + }{ + { + // The pod is up but nothing can route to it. That is what Degraded + // means; Error would overstate it. + name: "Ready degrades when an apply failed", + phase: llmv1alpha1.PhaseReady, + conds: failed, + want: llmv1alpha1.PhaseDegraded, + }, + { + name: "Ready degrades when a gateway CRD is missing", + phase: llmv1alpha1.PhaseReady, + conds: crdMissing, + want: llmv1alpha1.PhaseDegraded, + }, + { + name: "Ready stays Ready when every apply succeeded", + phase: llmv1alpha1.PhaseReady, + conds: ok, + want: llmv1alpha1.PhaseReady, + }, + { + // Degraded from determinePhase (some replicas down) already says + // the model is not fully healthy. + name: "Degraded stays Degraded", + phase: llmv1alpha1.PhaseDegraded, + conds: failed, + want: llmv1alpha1.PhaseDegraded, + }, + { + name: "Error is not softened to Degraded", + phase: llmv1alpha1.PhaseError, + conds: failed, + want: llmv1alpha1.PhaseError, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := phaseWithGatewayFailure(tt.phase, tt.conds); got != tt.want { + t.Errorf("phaseWithGatewayFailure(%q) = %q, want %q", tt.phase, got, tt.want) + } + }) + } +} diff --git a/operator/internal/controller/gateway_conditions_test.go b/operator/internal/controller/gateway_conditions_test.go new file mode 100644 index 00000000..980336f2 --- /dev/null +++ b/operator/internal/controller/gateway_conditions_test.go @@ -0,0 +1,97 @@ +package controller + +import ( + "context" + "errors" + "testing" + + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + llmv1alpha1 "github.com/nebari-dev/nebari-llm-serving-pack/operator/api/v1alpha1" +) + +// routeFailureClient rejects AIGatewayRoute reads, so every route apply fails +// with an error that is not a missing CRD. SecurityPolicies still apply, which +// keeps the CRD-pending requeue out of the way. +type routeFailureClient struct { + client.Client + fail bool +} + +func (c *routeFailureClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if c.fail && obj.GetObjectKind().GroupVersionKind().Kind == "AIGatewayRoute" { + gr := schema.GroupResource{Group: "aigateway.envoyproxy.io", Resource: "aigatewayroutes"} + return apierrors.NewForbidden(gr, key.Name, errors.New("denied")) + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func TestRouteApplyFailureDegradesAndRecovers(t *testing.T) { + ctx := context.Background() + r, policyClient, model := newPolicyTestReconciler(t) + c := &routeFailureClient{Client: policyClient, fail: true} + r.Client = c + req := reconcileRequest(model.Name) + + if _, err := r.Reconcile(ctx, req); err != nil { + t.Fatal(err) + } + dep := &appsv1.Deployment{} + if err := c.Get(ctx, client.ObjectKeyFromObject(model), dep); err != nil { + t.Fatal(err) + } + dep.Status.Replicas = 1 + dep.Status.ReadyReplicas = 1 + if err := c.Status().Update(ctx, dep); err != nil { + t.Fatal(err) + } + + result, err := r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("a failed route apply must not fail the reconcile: %v", err) + } + if result.RequeueAfter != applyFailedRequeue { + t.Errorf("RequeueAfter = %v, want %v", result.RequeueAfter, applyFailedRequeue) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(model), model); err != nil { + t.Fatal(err) + } + if model.Status.Phase != llmv1alpha1.PhaseDegraded { + t.Errorf("Phase = %q, want %q", model.Status.Phase, llmv1alpha1.PhaseDegraded) + } + for _, condType := range []string{CondExternalEndpointReady, CondInternalEndpointReady} { + cond := meta.FindStatusCondition(model.Status.Conditions, condType) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != reasonApplyFailed { + t.Errorf("%s = %+v, want False/%s", condType, cond, reasonApplyFailed) + } + } + if cond := meta.FindStatusCondition(model.Status.Conditions, condSecurityPoliciesReady); cond == nil || cond.Status != metav1.ConditionTrue { + t.Errorf("SecurityPoliciesReady = %+v, want True", cond) + } + + c.fail = false + result, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter != 0 { + t.Errorf("RequeueAfter after recovery = %v, want 0", result.RequeueAfter) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(model), model); err != nil { + t.Fatal(err) + } + if model.Status.Phase != llmv1alpha1.PhaseReady { + t.Errorf("Phase after recovery = %q, want %q", model.Status.Phase, llmv1alpha1.PhaseReady) + } + for _, condType := range []string{CondExternalEndpointReady, CondInternalEndpointReady, CondInferencePoolReady} { + cond := meta.FindStatusCondition(model.Status.Conditions, condType) + if cond == nil || cond.Status != metav1.ConditionTrue || cond.Reason != reasonApplied { + t.Errorf("%s after recovery = %+v, want True/%s", condType, cond, reasonApplied) + } + } +} diff --git a/operator/internal/controller/llmmodel_controller.go b/operator/internal/controller/llmmodel_controller.go index 66c40d3a..fc1b2055 100644 --- a/operator/internal/controller/llmmodel_controller.go +++ b/operator/internal/controller/llmmodel_controller.go @@ -27,6 +27,8 @@ import ( networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -173,42 +175,67 @@ func (r *LLMModelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // 10. Check deployment readiness to determine phase phase := r.determinePhase(ctx, model) if phase == llmv1alpha1.PhaseDownloading || phase == llmv1alpha1.PhaseStarting { - // Update status with phase and replica counts even during intermediate phases - if err := r.updateStatus(ctx, log, model, phase); err != nil { + // Update status with phase and replica counts even during intermediate + // phases. Step 11 has not run, so there are no gateway conditions to + // report yet. + if err := r.updateStatus(ctx, log, model, phase, nil); err != nil { log.Error(err, "failed to update status during intermediate phase", "phase", phase) } return ctrl.Result{RequeueAfter: 15 * time.Second}, nil } - // 11. Once model is (or will be) serving, create downstream resources + // 11. Once model is (or will be) serving, create downstream resources. + // + // The InferencePool and AIGatewayRoutes come from CRDs this pack does not + // install. A failed apply does not fail the reconcile, because everything + // the pack owns still converges. Each outcome becomes a status condition, + // and a failure downgrades Ready to Degraded, so a model never claims Ready + // while nothing can route to it. + var conditions []metav1.Condition if r.Config != nil { // InferencePool + EPP poolResources, err := reconcilers.BuildInferencePoolResources(model, r.Config) if err != nil { return ctrl.Result{}, fmt.Errorf("building inference pool resources: %w", err) } - if err := r.reconcileInferencePoolResources(ctx, log, model, poolResources); err != nil { + poolErr, err := r.reconcileInferencePoolResources(ctx, log, model, poolResources) + if err != nil { return ctrl.Result{}, err } + conditions = append(conditions, conditionFor(CondInferencePoolReady, poolErr, "InferencePool applied")) // Routing (AIGatewayRoutes) routingResources, err := reconcilers.BuildRoutingResources(model, r.Config) if err != nil { return ctrl.Result{}, fmt.Errorf("building routing resources: %w", err) } - if err := r.reconcileRoutingResources(ctx, log, model, routingResources); err != nil { - return ctrl.Result{}, err + externalErr, internalErr := r.reconcileRoutingResources(ctx, log, model, routingResources) + if routingResources.ExternalRoute != nil { + conditions = append(conditions, conditionFor(CondExternalEndpointReady, externalErr, "external AIGatewayRoute applied")) + } else { + conditions = append(conditions, disabledCondition(CondExternalEndpointReady)) + } + if routingResources.InternalRoute != nil { + conditions = append(conditions, conditionFor(CondInternalEndpointReady, internalErr, "internal AIGatewayRoute applied")) + } else { + conditions = append(conditions, disabledCondition(CondInternalEndpointReady)) } - } // 12. Update status - if err := r.updateStatus(ctx, log, model, phase); err != nil { + phase = phaseWithGatewayFailure(phase, conditions) + if err := r.updateStatus(ctx, log, model, phase, conditions); err != nil { return ctrl.Result{}, err } if authPending { return ctrl.Result{RequeueAfter: gatewayCRDRequeue}, nil } + if requeue := gatewayRequeue(conditions); requeue > 0 { + // The usual cause is a gateway CRD that is not installed yet, which a + // later pass fixes with no change to the LLMModel. Without the retry + // the model would stay Degraded until something else reconciled it. + return ctrl.Result{RequeueAfter: requeue}, nil + } return ctrl.Result{}, nil } @@ -384,68 +411,72 @@ func (r *LLMModelReconciler) reconcileInferencePoolResources( log controllerLogger, model *llmv1alpha1.LLMModel, pool *reconcilers.InferencePoolResources, -) error { - // InferencePool (unstructured CRD - non-fatal if missing) +) (poolErr error, err error) { + // InferencePool (unstructured CRD). Its error goes back as poolErr, for a + // status condition, instead of failing the reconcile: the EPP Deployment + // and RBAC below still converge, and the pool applies on a later pass + // once the Inference Extension CRDs exist. pool.InferencePool.SetNamespace(model.Namespace) - if err := applyModelResourcePreservingFinalizers(ctx, r.Client, pool.InferencePool); err != nil { - log.Error(err, "failed to reconcile InferencePool - CRD may not be installed, skipping") + poolErr = applyModelResourcePreservingFinalizers(ctx, r.Client, pool.InferencePool) + if poolErr != nil { + log.Error(poolErr, "failed to reconcile InferencePool - CRD may not be installed") } // EPP ServiceAccount pool.EPPServiceAccount.Namespace = model.Namespace if err := reconcilers.SetOwnerReference(model, pool.EPPServiceAccount, r.Scheme); err != nil { - return fmt.Errorf("setting owner on EPP ServiceAccount: %w", err) + return poolErr, fmt.Errorf("setting owner on EPP ServiceAccount: %w", err) } if err := r.createOrUpdateServiceAccount(ctx, pool.EPPServiceAccount); err != nil { - return fmt.Errorf("reconciling EPP ServiceAccount: %w", err) + return poolErr, fmt.Errorf("reconciling EPP ServiceAccount: %w", err) } // EPP Role pool.EPPRole.Namespace = model.Namespace if err := reconcilers.SetOwnerReference(model, pool.EPPRole, r.Scheme); err != nil { - return fmt.Errorf("setting owner on EPP Role: %w", err) + return poolErr, fmt.Errorf("setting owner on EPP Role: %w", err) } if err := r.createOrUpdateRole(ctx, pool.EPPRole); err != nil { - return fmt.Errorf("reconciling EPP Role: %w", err) + return poolErr, fmt.Errorf("reconciling EPP Role: %w", err) } // EPP RoleBinding pool.EPPRoleBinding.Namespace = model.Namespace if err := reconcilers.SetOwnerReference(model, pool.EPPRoleBinding, r.Scheme); err != nil { - return fmt.Errorf("setting owner on EPP RoleBinding: %w", err) + return poolErr, fmt.Errorf("setting owner on EPP RoleBinding: %w", err) } if err := r.createOrUpdateRoleBinding(ctx, pool.EPPRoleBinding); err != nil { - return fmt.Errorf("reconciling EPP RoleBinding: %w", err) + return poolErr, fmt.Errorf("reconciling EPP RoleBinding: %w", err) } // EPP Service pool.EPPService.Namespace = model.Namespace if err := reconcilers.SetOwnerReference(model, pool.EPPService, r.Scheme); err != nil { - return fmt.Errorf("setting owner on EPP Service: %w", err) + return poolErr, fmt.Errorf("setting owner on EPP Service: %w", err) } if err := r.createOrUpdateService(ctx, pool.EPPService); err != nil { - return fmt.Errorf("reconciling EPP Service: %w", err) + return poolErr, fmt.Errorf("reconciling EPP Service: %w", err) } // EPP ConfigMap (must exist before Deployment so the volume mount resolves) pool.EPPConfigMap.Namespace = model.Namespace if err := reconcilers.SetOwnerReference(model, pool.EPPConfigMap, r.Scheme); err != nil { - return fmt.Errorf("setting owner on EPP ConfigMap: %w", err) + return poolErr, fmt.Errorf("setting owner on EPP ConfigMap: %w", err) } if err := r.createOrUpdateConfigMap(ctx, pool.EPPConfigMap); err != nil { - return fmt.Errorf("reconciling EPP ConfigMap: %w", err) + return poolErr, fmt.Errorf("reconciling EPP ConfigMap: %w", err) } // EPP Deployment pool.EPPDeployment.Namespace = model.Namespace if err := reconcilers.SetOwnerReference(model, pool.EPPDeployment, r.Scheme); err != nil { - return fmt.Errorf("setting owner on EPP Deployment: %w", err) + return poolErr, fmt.Errorf("setting owner on EPP Deployment: %w", err) } if err := r.createOrUpdateDeployment(ctx, pool.EPPDeployment); err != nil { - return fmt.Errorf("reconciling EPP Deployment: %w", err) + return poolErr, fmt.Errorf("reconciling EPP Deployment: %w", err) } - return nil + return poolErr, nil } // reconcileRoutingResources creates or updates AIGatewayRoute resources. @@ -454,20 +485,20 @@ func (r *LLMModelReconciler) reconcileRoutingResources( log controllerLogger, model *llmv1alpha1.LLMModel, routing *reconcilers.RoutingResources, -) error { //nolint:unparam // error return kept for future extensibility +) (externalErr, internalErr error) { if routing.ExternalRoute != nil { routing.ExternalRoute.SetNamespace(model.Namespace) - if err := applyModelResourcePreservingFinalizers(ctx, r.Client, routing.ExternalRoute); err != nil { - log.Error(err, "failed to reconcile external AIGatewayRoute - CRD may not be installed, skipping") + if externalErr = applyModelResourcePreservingFinalizers(ctx, r.Client, routing.ExternalRoute); externalErr != nil { + log.Error(externalErr, "failed to reconcile external AIGatewayRoute - CRD may not be installed") } } if routing.InternalRoute != nil { routing.InternalRoute.SetNamespace(model.Namespace) - if err := applyModelResourcePreservingFinalizers(ctx, r.Client, routing.InternalRoute); err != nil { - log.Error(err, "failed to reconcile internal AIGatewayRoute - CRD may not be installed, skipping") + if internalErr = applyModelResourcePreservingFinalizers(ctx, r.Client, routing.InternalRoute); internalErr != nil { + log.Error(internalErr, "failed to reconcile internal AIGatewayRoute - CRD may not be installed") } } - return nil + return externalErr, internalErr } // determinePhase inspects the model Deployment and its pods to determine the current lifecycle phase. @@ -524,6 +555,7 @@ func (r *LLMModelReconciler) updateStatus( log controllerLogger, model *llmv1alpha1.LLMModel, phase llmv1alpha1.LLMModelPhase, + conditions []metav1.Condition, ) error { // Fetch fresh copy to avoid update conflicts fresh := &llmv1alpha1.LLMModel{} @@ -533,6 +565,12 @@ func (r *LLMModelReconciler) updateStatus( fresh.Status.Phase = phase fresh.Status.ObservedGeneration = fresh.Generation + // Set conditions one at a time rather than replacing the list, so + // SecurityPoliciesReady (patched earlier in the reconcile) survives. + for _, c := range conditions { + c.ObservedGeneration = fresh.Generation + meta.SetStatusCondition(&fresh.Status.Conditions, c) + } // Update replica counts dep := &appsv1.Deployment{} diff --git a/operator/internal/controller/llmmodel_controller_test.go b/operator/internal/controller/llmmodel_controller_test.go index 255e1df9..aac9c888 100644 --- a/operator/internal/controller/llmmodel_controller_test.go +++ b/operator/internal/controller/llmmodel_controller_test.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -465,6 +466,68 @@ var _ = Describe("LLMModel Controller", func() { }) }) + // The envtest API server has this operator's CRDs but not the AI Gateway's, + // Envoy Gateway's or the Gateway API Inference Extension's, so applying an + // AIGatewayRoute or InferencePool fails with no-kind-match: what a real + // cluster looks like when section 4 of the install runbook was skipped. The + // reconciler used to log those failures and report the model Ready anyway. + Describe("Missing gateway CRDs", func() { + const modelName = "test-missing-crds" + + AfterEach(func() { + model := &llmv1alpha1.LLMModel{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: modelName, Namespace: testNamespace}, model); err == nil { + model.Finalizers = nil + _ = k8sClient.Update(ctx, model) + _ = k8sClient.Delete(ctx, model) + } + }) + + // reconcileServing creates the model, reconciles until the Deployment + // exists, marks it ready by hand, and reconciles once more. Nothing runs + // the Deployment controller in envtest, so without the manual status + // update determinePhase returns Starting and the reconcile stops before + // any gateway resource is applied. + reconcileServing := func() { + Expect(k8sClient.Create(ctx, newMinimalModel(modelName))).To(Succeed()) + r := newReconciler() + Expect(reconcileUntilStable(r, ctx, reconcileRequest(modelName))).To(Succeed()) + + dep := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: modelName, Namespace: testNamespace}, dep)).To(Succeed()) + dep.Status.Replicas = 1 + dep.Status.ReadyReplicas = 1 + Expect(k8sClient.Status().Update(ctx, dep)).To(Succeed()) + + _, err := r.Reconcile(ctx, reconcileRequest(modelName)) + Expect(err).NotTo(HaveOccurred(), "a missing CRD must not fail the whole reconcile") + } + + It("reports Degraded with failing gateway conditions instead of Ready", func() { + reconcileServing() + + fresh := &llmv1alpha1.LLMModel{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: modelName, Namespace: testNamespace}, fresh)).To(Succeed()) + Expect(fresh.Status.Phase).To(Equal(llmv1alpha1.PhaseDegraded), + "a model whose routes could not be applied is not Ready") + + for _, condType := range []string{CondExternalEndpointReady, CondInternalEndpointReady, CondInferencePoolReady} { + cond := meta.FindStatusCondition(fresh.Status.Conditions, condType) + Expect(cond).NotTo(BeNil(), "%s must be reported", condType) + Expect(cond.Status).To(Equal(metav1.ConditionFalse), condType) + Expect(cond.Reason).To(Equal(reasonGatewayCRDUnavailable), condType) + Expect(cond.Message).NotTo(BeEmpty(), "%s must say what failed", condType) + Expect(cond.ObservedGeneration).To(Equal(fresh.Generation), condType) + } + + // SecurityPoliciesReady is written earlier in the reconcile, by its + // own status patch. Writing the gateway conditions must not drop it. + policyCond := meta.FindStatusCondition(fresh.Status.Conditions, condSecurityPoliciesReady) + Expect(policyCond).NotTo(BeNil(), "the gateway conditions must not overwrite SecurityPoliciesReady") + Expect(policyCond.Reason).To(Equal(reasonGatewayCRDUnavailable)) + }) + }) + Describe("Idempotency", func() { const modelName = "test-idempotent" diff --git a/operator/internal/controller/passthroughmodel_controller.go b/operator/internal/controller/passthroughmodel_controller.go index 054bf948..945d755c 100644 --- a/operator/internal/controller/passthroughmodel_controller.go +++ b/operator/internal/controller/passthroughmodel_controller.go @@ -42,18 +42,12 @@ import ( "github.com/nebari-dev/nebari-llm-serving-pack/operator/internal/provider" ) -// Condition types reported on PassthroughModel status. +// Condition types reported only on PassthroughModel status. The gateway +// conditions it shares with LLMModel are in conditions.go. const ( // CondUpstreamCredentialResolved reports whether the provider Secret has a non-empty API key. // "Upstream" distinguishes it from inbound client API-key Secrets. CondUpstreamCredentialResolved = "UpstreamCredentialResolved" - // CondBackendConfigured covers the provider plumbing: Backend, - // BackendTLSPolicy, AIServiceBackend, BackendSecurityPolicy. - CondBackendConfigured = "BackendConfigured" - // CondExternalEndpointReady covers the external route + apiKeyAuth policy. - CondExternalEndpointReady = "ExternalEndpointReady" - // CondInternalEndpointReady covers the internal route + JWT policy. - CondInternalEndpointReady = "InternalEndpointReady" ) // PassthroughModelReconciler reconciles a PassthroughModel object. @@ -303,36 +297,6 @@ func (r *PassthroughModelReconciler) createOrUpdateConfigMap(ctx context.Context return r.Update(ctx, existing) } -func conditionFor(condType string, err error, okMessage string) metav1.Condition { - if err != nil { - reason := reasonApplyFailed - if meta.IsNoMatchError(err) { - reason = reasonGatewayCRDUnavailable - } - return metav1.Condition{ - Type: condType, - Status: metav1.ConditionFalse, - Reason: reason, - Message: err.Error(), - } - } - return metav1.Condition{ - Type: condType, - Status: metav1.ConditionTrue, - Reason: "Applied", - Message: okMessage, - } -} - -func disabledCondition(condType string) metav1.Condition { - return metav1.Condition{ - Type: condType, - Status: metav1.ConditionFalse, - Reason: "EndpointDisabled", - Message: "endpoint disabled in spec", - } -} - func (r *PassthroughModelReconciler) updateStatus( ctx context.Context, log controllerLogger, From c3f5f3fbf29b1ede4152e70b112b9f5856399f99 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:57:14 +0100 Subject: [PATCH 2/2] fix(operator): degrade Ready while the SecurityPolicy CRD is missing SecurityPoliciesReady is patched earlier in the reconcile and is not in the conditions phaseWithGatewayFailure checks, so a model whose routes applied but whose SecurityPolicies could not (Envoy Gateway's CRD not installed) still reported Ready. Downgrade Ready to Degraded when the policies are pending; the existing 15s requeue retries them. The missing-CRD test now checks the phase is Degraded while the CRD is missing and Ready again once it is installed. --- operator/internal/controller/llmmodel_controller.go | 6 ++++++ .../internal/controller/securitypolicy_status_test.go | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/operator/internal/controller/llmmodel_controller.go b/operator/internal/controller/llmmodel_controller.go index fc1b2055..d3e99f63 100644 --- a/operator/internal/controller/llmmodel_controller.go +++ b/operator/internal/controller/llmmodel_controller.go @@ -224,6 +224,12 @@ func (r *LLMModelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // 12. Update status phase = phaseWithGatewayFailure(phase, conditions) + // A missing SecurityPolicy CRD leaves the endpoints without authentication. + // SecurityPoliciesReady is patched in step 6, not carried in conditions, so + // downgrade for it here. + if authPending && phase == llmv1alpha1.PhaseReady { + phase = llmv1alpha1.PhaseDegraded + } if err := r.updateStatus(ctx, log, model, phase, conditions); err != nil { return ctrl.Result{}, err } diff --git a/operator/internal/controller/securitypolicy_status_test.go b/operator/internal/controller/securitypolicy_status_test.go index d467d0d8..58e5cc5b 100644 --- a/operator/internal/controller/securitypolicy_status_test.go +++ b/operator/internal/controller/securitypolicy_status_test.go @@ -172,6 +172,11 @@ func TestMissingSecurityPolicyCRDRetriesWithoutBlockingProvisioning(t *testing.T if cond == nil || cond.Reason != "GatewayCRDUnavailable" || cond.Status != metav1.ConditionFalse { t.Fatalf("missing CRD not reported: %+v", cond) } + // The workload is serving but has no endpoint authentication, so it must + // not claim Ready. + if model.Status.Phase != llmv1alpha1.PhaseDegraded { + t.Fatalf("phase with missing SecurityPolicy CRD = %q, want %q", model.Status.Phase, llmv1alpha1.PhaseDegraded) + } c.missing = false if _, err := r.Reconcile(ctx, req); err != nil { t.Fatal(err) @@ -182,4 +187,7 @@ func TestMissingSecurityPolicyCRDRetriesWithoutBlockingProvisioning(t *testing.T if !meta.IsStatusConditionTrue(model.Status.Conditions, condSecurityPoliciesReady) { t.Fatal("condition did not recover after CRD installation") } + if model.Status.Phase != llmv1alpha1.PhaseReady { + t.Fatalf("phase after CRD installation = %q, want %q", model.Status.Phase, llmv1alpha1.PhaseReady) + } }