From 5544681fa3d17d9f9f64ff232c2c84aa756b38cc Mon Sep 17 00:00:00 2001 From: Robusta Runner Date: Thu, 16 May 2024 15:16:19 +0300 Subject: [PATCH 1/3] add support for CRDs --- README.md | 10 +++++++- config/config.go | 11 +++++++- examples/conf/kubewatch.conf.crd.yaml | 4 +++ pkg/controller/controller.go | 36 +++++++++++++++++++++++++++ pkg/utils/k8sutil.go | 32 +++++++++++++++++++++++- 5 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 examples/conf/kubewatch.conf.crd.yaml diff --git a/README.md b/README.md index 1e3d663d..1f09957c 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,10 @@ resourcesToWatch: ingress: false coreevent: false event: true +customresources: + - group: monitoring.coreos.com + version: v1 + resource: prometheusrules slack: channel: '#YOUR_CHANNEL' token: 'xoxb-YOUR_TOKEN' @@ -129,7 +133,7 @@ Once the Pod is running, you will start seeing Kubernetes events in your configu ![slack](./docs/slack.png) -To modify what notifications you get, update the `kubewatch` ConfigMap and turn on and off (true/false) resources: +To modify what notifications you get, update the `kubewatch` ConfigMap and turn on and off (true/false) resources or configure any resource of your choosing with customresources (CRDs): ``` resource: @@ -151,6 +155,10 @@ resource: ingress: false coreevent: false event: true +customresources: + - group: monitoring.coreos.com + version: v1 + resource: prometheusrules ``` #### Working with RBAC diff --git a/config/config.go b/config/config.go index 9778c022..4afdd971 100755 --- a/config/config.go +++ b/config/config.go @@ -19,6 +19,7 @@ limitations under the License. package config import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -73,7 +74,12 @@ type Resource struct { CoreEvent bool `json:"coreevent"` } -// Config struct contains kubewatch configuration +type CRD struct { + Group string `json:"group"` + Version string `json:"version"` + Resource string `json:"resource"` +} + type Config struct { // Handlers know how to send notifications to specific services. Handler Handler `json:"handler"` @@ -83,6 +89,9 @@ type Config struct { // Resources to watch. Resource Resource `json:"resource"` + // CustomResources to Watch + CustomResources []CRD `json:"customresources"` + // For watching specific namespace, leave it empty for watching all. // this config is ignored when watching namespaces Namespace string `json:"namespace,omitempty"` diff --git a/examples/conf/kubewatch.conf.crd.yaml b/examples/conf/kubewatch.conf.crd.yaml new file mode 100644 index 00000000..7eb8ada3 --- /dev/null +++ b/examples/conf/kubewatch.conf.crd.yaml @@ -0,0 +1,4 @@ +customresources: + - group: monitoring.coreos.com + version: v1 + resource: prometheusrules \ No newline at end of file diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 987607b7..7b694052 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -19,6 +19,8 @@ package controller import ( "context" "fmt" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "os" "os/signal" "reflect" @@ -44,6 +46,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -89,11 +92,14 @@ func objName(obj interface{}) string { // Start prepares watchers and run their controllers, then waits for process termination signals func Start(conf *config.Config, eventHandler handlers.Handler) { var kubeClient kubernetes.Interface + var dynamicClient dynamic.Interface if _, err := rest.InClusterConfig(); err != nil { kubeClient = utils.GetClientOutOfCluster() + dynamicClient = utils.GetDynamicClientOutOfCluster() } else { kubeClient = utils.GetClient() + dynamicClient = utils.GetDynamicClient() } // User Configured Events @@ -542,6 +548,36 @@ func Start(conf *config.Config, eventHandler handlers.Handler) { go c.Run(stopCh) } + for _, crd := range conf.CustomResources { + informer := cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { + return dynamicClient.Resource(schema.GroupVersionResource{ + Group: crd.Group, + Version: crd.Version, + Resource: crd.Resource, + }).List(options) + }, + WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { + return dynamicClient.Resource(schema.GroupVersionResource{ + Group: crd.Group, + Version: crd.Version, + Resource: crd.Resource, + }).Watch(options) + }, + }, + &unstructured.Unstructured{}, + 0, //Skip resync + cache.Indexers{}, + ) + + c := newResourceController(kubeClient, eventHandler, informer, crd.Resource, fmt.Sprintf("%s/%s", crd.Group, crd.Version)) + stopCh := make(chan struct{}) + defer close(stopCh) + + go c.Run(stopCh) + } + sigterm := make(chan os.Signal, 1) signal.Notify(sigterm, syscall.SIGTERM) signal.Notify(sigterm, syscall.SIGINT) diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go index 78d9e006..faffba94 100644 --- a/pkg/utils/k8sutil.go +++ b/pkg/utils/k8sutil.go @@ -13,12 +13,27 @@ import ( events_v1 "k8s.io/api/events/v1" rbac_v1beta1 "k8s.io/api/rbac/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) -// GetClient returns a k8s clientset to the request from inside of cluster +// GetDynamicClient returns a k8s dynamic clientset to the request from inside of cluster +func GetDynamicClient() dynamic.Interface { + config, err := rest.InClusterConfig() + if err != nil { + logrus.Fatalf("Can not get kubernetes config: %v", err) + } + + clientset, err := dynamic.NewForConfig(config) + if err != nil { + logrus.Fatalf("Can not create dynamic kubernetes client: %v", err) + } + + return clientset +} + func GetClient() kubernetes.Interface { config, err := rest.InClusterConfig() if err != nil { @@ -56,6 +71,21 @@ func GetClientOutOfCluster() kubernetes.Interface { return clientset } +// GetDynamicClientOutOfCluster returns a k8s dynamic clientset to the request from outside of cluster +func GetDynamicClientOutOfCluster() dynamic.Interface { + config, err := buildOutOfClusterConfig() + if err != nil { + logrus.Fatalf("Can not get kubernetes config: %v", err) + } + + clientset, err := dynamic.NewForConfig(config) + if err != nil { + logrus.Fatalf("Can not get kubernetes config: %v", err) + } + + return clientset +} + // GetObjectMetaData returns metadata of a given k8s object func GetObjectMetaData(obj interface{}) (objectMeta meta_v1.ObjectMeta) { From 8111bb71bcf220df7f8ba42ce73a3837bbb73af0 Mon Sep 17 00:00:00 2001 From: Dima Chievtaiev Date: Wed, 14 Aug 2024 17:33:21 +0200 Subject: [PATCH 2/3] Fixed usage of ListWatch and WatchFunc for conf.CustomResources; removed unused fmt import from config/config.go --- config/config.go | 1 - pkg/controller/controller.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index 4afdd971..5037285e 100755 --- a/config/config.go +++ b/config/config.go @@ -19,7 +19,6 @@ limitations under the License. package config import ( - "fmt" "io/ioutil" "os" "path/filepath" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7b694052..a6e2c91d 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -556,14 +556,14 @@ func Start(conf *config.Config, eventHandler handlers.Handler) { Group: crd.Group, Version: crd.Version, Resource: crd.Resource, - }).List(options) + }).List(context.Background(), options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return dynamicClient.Resource(schema.GroupVersionResource{ Group: crd.Group, Version: crd.Version, Resource: crd.Resource, - }).Watch(options) + }).Watch(context.Background(), options) }, }, &unstructured.Unstructured{}, From ba14afe18de21cb4bd4142040b9e14510c1a94aa Mon Sep 17 00:00:00 2001 From: Dima Chievtaiev Date: Wed, 14 Aug 2024 17:43:17 +0200 Subject: [PATCH 3/3] Added customRules rbac in kubewatch/templates/clusterrole.yaml --- helm/kubewatch/templates/clusterrole.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helm/kubewatch/templates/clusterrole.yaml b/helm/kubewatch/templates/clusterrole.yaml index 2401fa41..f0d90abf 100644 --- a/helm/kubewatch/templates/clusterrole.yaml +++ b/helm/kubewatch/templates/clusterrole.yaml @@ -76,4 +76,9 @@ rules: - get - list - watch + {{- range .Values.rbac.customRules }} + - apiGroups: {{ toYaml .apiGroups | nindent 4 }} + resources: {{ toYaml .resources | nindent 4 }} + verbs: {{ toYaml .verbs | nindent 4 }} + {{- end }} {{- end -}}