From ca17d7225f0694995b85300169fc7638df5f068a Mon Sep 17 00:00:00 2001 From: Greg Maher Date: Wed, 5 Aug 2026 14:40:13 -0700 Subject: [PATCH] Filter kube informers server-side to operator-generated objects The kube informer factory watches Pods, Services, EndpointSlices, ConfigMaps and StatefulSets across the whole cluster whenever watchNamespaces is anything other than a single literal namespace name, because GetInformerNamespace() falls back to meta.NamespaceAll in every other case, including regexp patterns. Filtering then happens client-side in isTrackedObject(), i.e. after every object in the cluster has already been transferred, decoded and queued in the DeltaFIFO. Informer memory therefore scales with total cluster size rather than with the number of managed installations, which OOM-kills the operator on large clusters with many namespaces. Push the same filter server-side with WithTweakListOptions. The selector is built through the same labeler that IsCHOPGeneratedObject() resolves through, rather than from the raw LabelAppName constant, so the server-side selector and the client-side predicate cannot drift apart. This is intended to be behavior-preserving: every handler registered on the kube informer factory already gates on isTrackedObject(), and nothing reads these caches expecting to find objects the operator did not generate - there are no Lister() or GetIndexer() reads against them anywhere in the tree. CHOp custom resource informers are deliberately left unfiltered. Co-Authored-By: Claude Opus 5 --- cmd/operator/app/thread_chi.go | 24 ++++++++ cmd/operator/app/thread_chi_test.go | 92 +++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 cmd/operator/app/thread_chi_test.go diff --git a/cmd/operator/app/thread_chi.go b/cmd/operator/app/thread_chi.go index c2ac3e326..e1d746eaf 100644 --- a/cmd/operator/app/thread_chi.go +++ b/cmd/operator/app/thread_chi.go @@ -18,12 +18,16 @@ import ( "context" "time" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sLabels "k8s.io/apimachinery/pkg/labels" kubeinformers "k8s.io/client-go/informers" log "github.com/altinity/clickhouse-operator/pkg/announcer" "github.com/altinity/clickhouse-operator/pkg/chop" chopinformers "github.com/altinity/clickhouse-operator/pkg/client/informers/externalversions" "github.com/altinity/clickhouse-operator/pkg/controller/chi" + chiLabeler "github.com/altinity/clickhouse-operator/pkg/model/chi/tags/labeler" + commonLabeler "github.com/altinity/clickhouse-operator/pkg/model/common/tags/labeler" ) // Prometheus exporter defaults @@ -43,6 +47,16 @@ var ( func init() { } +// chopGeneratedObjectsLabelSelector builds a label selector that matches objects generated by the operator. +// This is the server-side counterpart of the client-side Controller.isTrackedObject() check, which requires +// Labeler.IsCHOPGeneratedObject(). Both are built out of the same labeler, so that the two can not diverge. +func chopGeneratedObjectsLabelSelector() string { + labeler := chiLabeler.New(nil) + return k8sLabels.SelectorFromSet(k8sLabels.Set{ + labeler.Get(commonLabeler.LabelAppName): labeler.Get(commonLabeler.LabelAppValue), + }).String() +} + var chiController *chi.Controller // initClickHouse is an entry point of the application @@ -90,10 +104,20 @@ func initClickHouse(ctx context.Context) { } // Create Informers + // Kube objects are filtered server-side down to operator-generated objects only. GetInformerNamespace() + // falls back to all namespaces as soon as watched namespaces are specified as anything other than a single + // literal namespace name, so without this selector the informer caches - and the DeltaFIFO feeding them - + // scale with the size of the whole cluster instead of with the number of managed installations. + // CHOp custom resource informers below are intentionally left unfiltered. + kubeObjectsLabelSelector := chopGeneratedObjectsLabelSelector() + log.V(1).F().Info("Kube informers label selector: '%s'", kubeObjectsLabelSelector) kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions( kubeClient, kubeInformerFactoryResyncPeriod, kubeinformers.WithNamespace(chop.Config().GetInformerNamespace()), + kubeinformers.WithTweakListOptions(func(options *meta.ListOptions) { + options.LabelSelector = kubeObjectsLabelSelector + }), ) chopInformerFactory := chopinformers.NewSharedInformerFactoryWithOptions( chopClient, diff --git a/cmd/operator/app/thread_chi_test.go b/cmd/operator/app/thread_chi_test.go new file mode 100644 index 000000000..db993457d --- /dev/null +++ b/cmd/operator/app/thread_chi_test.go @@ -0,0 +1,92 @@ +package app + +import ( + "testing" + + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sLabels "k8s.io/apimachinery/pkg/labels" + + chiLabeler "github.com/altinity/clickhouse-operator/pkg/model/chi/tags/labeler" +) + +// The kube informer factory applies chopGeneratedObjectsLabelSelector() server-side. Handlers still apply +// the client-side Controller.isTrackedObject() check, which requires Labeler.IsCHOPGeneratedObject(). +// If the two ever disagree, the operator silently stops seeing objects it is supposed to reconcile, so +// pin down both the literal selector and its equivalence to the client-side predicate. + +func Test_chopGeneratedObjectsLabelSelector(t *testing.T) { + want := "clickhouse.altinity.com/app=chop" + if got := chopGeneratedObjectsLabelSelector(); got != want { + t.Errorf("chopGeneratedObjectsLabelSelector() = %q, want %q", got, want) + } +} + +func Test_chopGeneratedObjectsLabelSelectorMatchesIsCHOPGeneratedObject(t *testing.T) { + tests := []struct { + name string + labels map[string]string + want bool + }{ + { + name: "operator-generated object", + labels: map[string]string{"clickhouse.altinity.com/app": "chop"}, + want: true, + }, + { + name: "operator-generated object among unrelated labels", + labels: map[string]string{ + "clickhouse.altinity.com/app": "chop", + "clickhouse.altinity.com/chi": "some-chi", + "kubernetes.io/service-name": "some-svc", + "app.kubernetes.io/managed-by": "Helm", + }, + want: true, + }, + { + name: "no labels at all", + labels: nil, + want: false, + }, + { + name: "unrelated labels only", + labels: map[string]string{"app": "clickhouse"}, + want: false, + }, + { + name: "right key, wrong value", + labels: map[string]string{"clickhouse.altinity.com/app": "not-chop"}, + want: false, + }, + { + name: "bare app label is not enough", + labels: map[string]string{"app": "chop"}, + want: false, + }, + { + // CHK-generated objects carry the clickhouse-keeper API group key, and isTrackedObject() + // uses the CHI labeler, so they are not tracked by the CHI controller either way. + name: "keeper-generated object", + labels: map[string]string{"clickhouse-keeper.altinity.com/app": "chop"}, + want: false, + }, + } + + selector, err := k8sLabels.Parse(chopGeneratedObjectsLabelSelector()) + if err != nil { + t.Fatalf("unable to parse selector: %v", err) + } + labeler := chiLabeler.New(nil) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Server-side: what the informer factory asks the API server for + if got := selector.Matches(k8sLabels.Set(tt.labels)); got != tt.want { + t.Errorf("selector.Matches() = %v, want %v", got, tt.want) + } + // Client-side: what the event handlers gate on + if got := labeler.IsCHOPGeneratedObject(&meta.ObjectMeta{Labels: tt.labels}); got != tt.want { + t.Errorf("IsCHOPGeneratedObject() = %v, want %v", got, tt.want) + } + }) + } +}