fix: bootstrap trusted kubelet serving certificates

api/v1alpha1/nodepool_types.go

@@ -266,6 +266,7 @@ type NodePoolStatus struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:scope=Cluster,shortName=np
 // +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
 // +kubebuilder:printcolumn:name="Strategy",type=string,JSONPath=`.spec.strategy`
 // +kubebuilder:printcolumn:name="Providers",type=string,JSONPath=`.status.providers`
 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`

cmd/main.go

@@ -84,6 +84,7 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var kubeletAddr, kubeletClientCA string
+	var kubeletServingTLSBootstrap bool
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -110,6 +111,10 @@ func main() {
 			"serves TLS without client verification, because which CA signs the API server's kubelet "+
 			"client certificate is not portable across distributions; restrict the port with a "+
 			"NetworkPolicy, or set this to your API server's kubelet client CA.")
+	flag.BoolVar(&kubeletServingTLSBootstrap, "kubelet-serving-tls-bootstrap", true,
+		"Request a serving certificate for the manager Pod IP through the "+
+			"kubernetes.io/kubelet-serving CSR signer. The self-signed certificate remains active "+
+			"until an external approver approves the CSR.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -270,7 +275,7 @@ func main() {
 	// The kubelet endpoint for `kubectl logs` — one listener shared by every provider's
 	// node, hence built here rather than in setupVirtualNodes. Nil is supported: the
 	// nodes then advertise no address, and logs report NotFound.
-	kubeletSrv := setupKubeletServer(mgr, kubeletAddr, kubeletClientCA)
+	kubeletSrv := setupKubeletServer(mgr, kubeletAddr, kubeletClientCA, kubeletServingTLSBootstrap)
 
 	// Controller and webhook registration is deferred until the cert exists, so it
 	// runs in a goroutine: the cert cannot be minted until the manager is STARTED
@@ -417,7 +422,9 @@ func setupControllers(mgr ctrl.Manager, blocklist *failover.Blocklist, kubeletSr
 // what the API server dials and nothing substitutes for it: a Service would balance to
 // a non-leader replica, which holds no tracked Pods. Either way only logs degrade, so
 // it is logged loudly and the manager carries on.
-func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string) *vnode.KubeletServer {
+// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=create;delete;get
+
+func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string, servingTLSBootstrap bool) *vnode.KubeletServer {
 	if addr == "" {
 		setupLog.Info("kubelet API disabled by configuration; `kubectl logs` will not work for Nebula pods")
 		return nil
@@ -439,7 +446,31 @@ func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string) *vnode.KubeletS
 		setupLog.Error(err, "unable to add the kubelet API to the manager")
 		return nil
 	}
-	setupLog.Info("kubelet API enabled", "addr", addr, "advertisedIP", podIP, "clientCertRequired", clientCA != "")
+	if servingTLSBootstrap {
+		clientset, err := kubernetes.NewForConfig(mgr.GetConfig())
+		if err != nil {
+			setupLog.Error(err, "failed to create Kubernetes client for kubelet serving certificate bootstrap")
+		} else {
+			bootstrapper, err := vnode.NewKubeletServingCertificateBootstrapper(
+				clientset,
+				srv,
+				podIP,
+				managerNamespace(),
+				os.Getenv("POD_NAME"),
+				os.Getenv("POD_UID"),
+			)
+			if err != nil {
+				setupLog.Error(err, "failed to configure kubelet serving certificate bootstrap")
+			} else if err := mgr.Add(bootstrapper); err != nil {
+				setupLog.Error(err, "failed to add kubelet serving certificate bootstrap to the manager")
+			}
+		}
+	}
+	setupLog.Info("kubelet API enabled",
+		"addr", addr,
+		"advertisedIP", podIP,
+		"clientCertRequired", clientCA != "",
+		"servingTLSBootstrap", servingTLSBootstrap)
 	return srv
 }
 

config/crd/bases/nebula.inftyai.com_nodepools.yaml

@@ -17,10 +17,13 @@ spec:
   scope: Cluster
   versions:
   - additionalPrinterColumns:
+    - jsonPath: .status.conditions[?(@.type=="Ready")].status
+      name: Status
+      type: string
     - jsonPath: .spec.strategy
       name: Strategy
       type: string
-    - jsonPath: .spec.providers[*].name
+    - jsonPath: .status.providers
       name: Providers
       type: string
     - jsonPath: .metadata.creationTimestamp
@@ -296,6 +299,12 @@ spec:
                   Placed counts existing instances per provider (booting included), for
                   at-a-glance balance.
                 type: object
+              providers:
+                description: |-
+                  Providers is a comma-separated list of provider names from the pool
+                  spec. kubectl printcolumns cannot join array fields via JSONPath, so
+                  the controller materializes this summary for `kubectl get nodepool`.
+                type: string
             type: object
         type: object
     served: true

config/manager/manager.yaml

@@ -93,6 +93,17 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
+          # Identity used to give the kubelet-serving CSR a stable name for this
+          # exact Pod. The private key remains in memory; a recreated Pod gets a
+          # new UID and a separate request.
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_UID
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.uid
         envFrom:
           # Provider credentials live in a per-provider Secret, one secretRef per
           # provider — NOT a single shared secret. This matches the "creds-absent →
@@ -123,11 +134,11 @@ spec:
         # kubelet). Declaring it is documentation and NetworkPolicy surface; the
         # listener binds either way.
         #
-        # It serves TLS with a self-signed cert but does NOT verify client certs by
-        # default, because which CA signs the API server's kubelet client cert is not
-        # portable — requiring it would break logs on managed control planes. So
-        # anything able to reach this port can read any Nebula pod's logs: restrict it
-        # with a NetworkPolicy, or set --kubelet-client-ca to require mTLS.
+        # It starts with a self-signed cert, then requests a kubelet-serving cert for
+        # POD_IP. Managed control planes that verify kubelet certificates use the
+        # signed cert after an external approver approves its CSR. Client certs are
+        # still not verified by default: restrict this port with a NetworkPolicy, or
+        # set --kubelet-client-ca to require mTLS.
         - name: kubelet-api
           containerPort: 10250
           protocol: TCP

config/rbac/role.yaml

@@ -61,6 +61,14 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - create
+  - delete
+  - get
 - apiGroups:
   - coordination.k8s.io
   resources:

docs/architecture.md

@@ -406,6 +406,12 @@ Responsibilities:
 - compute `status.placed` from Bound NodeClaims per provider;
 - watch NodeClaims so placement counts update as instances come and go.
 
+The default `kubectl get nodepools` table exposes the `Ready` condition's value
+as `STATUS`, followed by strategy, providers, and age. The CRD printer column
+reads the condition directly, so `status.conditions` remains the source of truth.
+See the [printer-column design](design/nodepool-status-column.md) for the empty
+condition and compatibility behavior.
+
 Static spec rules are admission-time CEL validations. Examples: `Weighted`
 requires a weight on every provider entry, and AWS provider entries require at
 least one region.
@@ -502,6 +508,7 @@ spec:
   failover:
     blocklistTTL: 30s
 status:
+  providers: modal,aws
   placed:
     modal: 2
     aws: 1

docs/deploy.md

@@ -128,13 +128,37 @@ Manager flags worth knowing (edit `config/manager/manager.yaml` `args`):
 | Flag | Default | Meaning |
 |---|---|---|
 | `--kubelet-bind-address` | `:10250` | Where the kubelet log endpoint listens — the address the API server proxies `kubectl logs` to. Set it empty to disable the endpoint, which disables logs and nothing else. |
+| `--kubelet-serving-tls-bootstrap` | `true` | Request a certificate for the advertised Pod IP from the `kubernetes.io/kubelet-serving` signer. Until it is approved and issued, the endpoint retains its self-signed fallback. Disable this only when the API server does not verify kubelet serving certificates. |
 | `--kubelet-client-ca` | *(empty)* | PEM bundle of CAs whose client certificates are accepted on that port. **Empty means client certificates are not verified**, so anything able to reach port 10250 can read the logs of any Pod on Nebula's virtual nodes. Set it to your API server's kubelet client CA to require mTLS, or keep the port closed with a NetworkPolicy. The default is open because which CA signs that client cert is not portable — kubeadm uses the cluster CA, EKS/GKE their own — so requiring it by default would break logs on managed control planes. |
 
 The endpoint needs `POD_IP` (projected via `fieldRef` in `config/manager/manager.yaml`)
 because virtual nodes advertise the leader's Pod IP, not a Service. Running the manager
 off-cluster leaves it unset, and logs degrade to unsupported. See
 [kubelet-api.md](kubelet-api.md).
 
+The Kubernetes signer does not approve kubelet-serving requests itself. On a cluster
+without a dedicated approver, inspect and approve Nebula's request after each manager
+Pod recreation and certificate renewal:
+
+```bash
+CSR=$(kubectl get csr \
+  -l app.kubernetes.io/name=nebula,app.kubernetes.io/component=kubelet-serving-certificate \
+  --sort-by=.metadata.creationTimestamp -o name | tail -n1)
+
+# Confirm the requested IP SAN matches the manager Pod IP before approving it.
+kubectl get csr "$CSR" -o jsonpath='{.spec.request}' \
+  | openssl base64 -d -A | openssl req -text -noout
+kubectl -n nebula-system get pod -l control-plane=controller-manager -o wide
+
+kubectl certificate approve "$CSR"
+kubectl -n nebula-system logs deploy/nebula-controller-manager \
+  | grep 'installed trusted kubelet serving certificate'
+```
+
+An installation with an external CSR approver should restrict it to requests that
+match Nebula's ServiceAccount, `system:nodes` organization, manager Pod identity, and
+current Pod IP. Nebula intentionally receives no permission to approve certificates.
+
 ---
 
 ## Manual deployment
@@ -187,6 +211,10 @@ kubectl -n nebula-system logs deploy/nebula-controller-manager | grep -i provide
 # Virtual nodes exist, one per registered provider.
 kubectl get nodes -l nebula.inftyai.com/provider
 
+# Kubelet serving CSR is signed (required by control planes that verify kubelet TLS).
+kubectl get csr \
+  -l app.kubernetes.io/name=nebula,app.kubernetes.io/component=kubelet-serving-certificate
+
 # Webhook TLS is wired: the caBundle matches the serving cert Secret.
 diff <(kubectl get secret nebula-webhook-server-cert -n nebula-system -o jsonpath='{.data.tls\.crt}') \
      <(kubectl get mutatingwebhookconfiguration nebula-mutating-webhook-configuration \
@@ -197,7 +225,12 @@ diff <(kubectl get secret nebula-webhook-server-cert -n nebula-system -o jsonpat
 A pool referencing an unregistered provider shows it plainly:
 
 ```bash
-kubectl get nodepool <name> -o jsonpath='{.status.conditions}'
+kubectl get nodepools
+# NAME       STATUS   STRATEGY   PROVIDERS   AGE
+# gpu-pool   False    Ordered    modal,aws   2m
+
+# Inspect the condition reason and message when STATUS is False.
+kubectl get nodepool <name> -o jsonpath='{.status.conditions[?(@.type=="Ready")]}'
 # Ready=False / UnknownProvider means that provider's creds are missing or wrong.
 ```
 

docs/design/nodepool-status-column.md

@@ -0,0 +1,46 @@
+# NodePool status printer column
+
+## Context
+
+`NodePool.status.conditions` already reports whether a pool can be used. The
+controller owns a standard `Ready` condition and sets it to `True` for a valid
+pool or `False` when an environment-dependent validation, such as provider
+registration, fails. However, the default `kubectl get nodepools` table does not
+show that signal, so operators must request the full object or write a JSONPath.
+
+## Decision
+
+Add a `Status` CRD printer column whose JSONPath selects the status of the
+`Ready` condition:
+
+```text
+.status.conditions[?(@.type=="Ready")].status
+```
+
+The column is derived directly by the Kubernetes API server when it renders the
+table. No duplicate status field or controller change is introduced. This keeps
+the condition as the single source of truth and uses the standard condition
+values `True`, `False`, and `Unknown`.
+
+The column appears before policy details so pool health is visible immediately:
+
+```text
+NAME       STATUS   STRATEGY   PROVIDERS       AGE
+gpu-pool   True     Ordered    modal,runpod    2m
+```
+
+Before the controller has written the `Ready` condition, the table cell has no
+value. This is preferable to manufacturing a fourth status value because absence
+already means the controller has not observed the object.
+
+## Compatibility and rollout
+
+This is an additive change to `additionalPrinterColumns`; the stored and served
+resource schema is unchanged. Existing clients that read `NodePool` objects are
+unaffected. Installing the regenerated CRD is sufficient to enable the column
+for existing pools, and the next `kubectl get` uses their existing conditions.
+
+## Verification
+
+Generation is checked into `config/crd/bases`. Regenerating the manifests keeps
+the CRD printer column aligned with the marker in `nodepool_types.go`.

docs/kubelet-api.md

@@ -23,13 +23,18 @@ Pod IP and that port. Consequences worth knowing:
 - The endpoint is **leader-scoped and dialed by Pod IP**, not through a Service. The
   tracked Pods live in one process's memory, so a Service balancing across replicas
   would send requests to a replica that answers `NotFound`.
-- It serves TLS with a self-signed, in-memory certificate — what the API server
-  expects of a kubelet, which does not verify it unless
-  `--kubelet-certificate-authority` is set. Client certificates are **not** verified
-  by default, because which CA signs the API server's kubelet client cert is not
-  portable across distributions. Anything that can reach the port can therefore read the
-  logs of, and **run commands in**, any Pod on these virtual nodes, with no RBAC check:
-  keep it closed with a NetworkPolicy, or pass `--kubelet-client-ca` to require mTLS.
+- It starts with a self-signed, in-memory certificate and, by default, creates a
+  `kubernetes.io/kubelet-serving` CSR whose IP SAN is the advertised Pod IP. This is
+  required by control planes such as EKS that verify kubelet serving certificates.
+  The built-in signer requires an external approval decision; once the certificate is
+  issued, new TLS handshakes use it immediately without restarting the manager. See
+  [deploy.md](deploy.md#configuration) for approval and inspection commands.
+- Client certificates are **not** verified by default, because which CA signs the API
+  server's kubelet client cert is not portable across distributions. Serving-certificate
+  bootstrap secures the opposite direction and does not change that. Anything that can
+  reach the port can therefore read logs and **run commands in** any Pod on these virtual
+  nodes with no RBAC check: keep it closed with a NetworkPolicy, or pass
+  `--kubelet-client-ca` to require mTLS.
 - No POD_IP (running the manager off-cluster) means no endpoint. Logs and exec degrade
   to unsupported; nothing else is affected.
 

pkg/vnode/kubelet.go

@@ -33,6 +33,7 @@ import (
 	"os"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
@@ -81,10 +82,9 @@ const (
 // is resolved by asking each registered Handler whether it tracks that Pod — at most one
 // can. Cheaper than a port per provider, and than reading the Pod to learn its node.
 //
-// TLS uses a self-signed in-memory cert, which is what the API server expects: it does
-// not verify a kubelet's serving cert unless --kubelet-certificate-authority is set. The
-// webhook cert rotator cannot help, since it mints for a Service DNS name and this
-// endpoint is dialed by Pod IP.
+// TLS starts with a self-signed in-memory cert. Clusters that verify kubelet serving
+// certificates can replace it at runtime with a certificate issued through the
+// kubernetes.io/kubelet-serving signer; see KubeletServingCertificateBootstrapper.
 //
 // Client certs are verified only when ClientCAPath is set. Off by default because which CA
 // signs the API server's kubelet client cert is not portable (kubeadm uses the cluster CA,
@@ -105,6 +105,10 @@ type KubeletServer struct {
 	// others are refused at the TLS layer. Empty disables verification — see above.
 	clientCAPath string
 
+	// servingCert is read on every TLS handshake, so an approved kubelet-serving
+	// certificate takes effect without restarting this listener or dropping streams.
+	servingCert atomic.Pointer[tls.Certificate]
+
 	mu       sync.RWMutex
 	handlers map[string]*Handler
 }
@@ -150,6 +154,12 @@ func (s *KubeletServer) Register(nodeName string, h *Handler) {
 	s.handlers[nodeName] = h
 }
 
+// SetServingCertificate atomically replaces the certificate used for new TLS
+// handshakes. Existing log and exec streams keep their current connections.
+func (s *KubeletServer) SetServingCertificate(cert tls.Certificate) {
+	s.servingCert.Store(&cert)
+}
+
 // nodeAddress is what a node advertises so the API server can find this endpoint.
 // InternalIP ONLY, which is load-bearing: --kubelet-preferred-address-types tries
 // Hostname first, so also advertising one would have the API server try to resolve
@@ -260,15 +270,26 @@ func (s *KubeletServer) runInContainer(
 	return h.RunInContainer(ctx, namespace, podName, containerName, cmd, attach)
 }
 
-// tlsConfig: a fresh self-signed keypair, plus client verification if a CA is set.
+// tlsConfig installs a self-signed fallback and reads servingCert on every handshake,
+// allowing TLS bootstrap to replace it without restarting the server. Client
+// verification is added independently when a CA is configured.
 func (s *KubeletServer) tlsConfig() (*tls.Config, error) {
 	cert, err := selfSignedCert(s.nodeIP)
 	if err != nil {
 		return nil, err
 	}
+	if s.servingCert.Load() == nil {
+		s.SetServingCertificate(cert)
+	}
 	cfg := &tls.Config{
-		Certificates: []tls.Certificate{cert},
-		MinVersion:   tls.VersionTLS12,
+		GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			cert := s.servingCert.Load()
+			if cert == nil {
+				return nil, errors.New("kubelet api: no serving certificate")
+			}
+			return cert, nil
+		},
+		MinVersion: tls.VersionTLS12,
 		// http/1.1 only, like a real kubelet: logs need nothing HTTP/2 offers, and this is
 		// the streaming path every kubelet client already exercises.
 		NextProtos: []string{"http/1.1"},