From 3f84c22f3c5c025941392aef1652b512e576668c Mon Sep 17 00:00:00 2001
From: Jakob Steiner <jakob.steiner@glasskube.eu>
Date: Mon, 9 Oct 2023 16:09:57 +0200
Subject: [PATCH] feat: add webhook for status changes (#359)

---
 deploy/crd/matomos.glasskube.eu-v1.yml        |  3 +
 .../apps/common/database/HasReadyStatus.kt    |  8 ++
 .../operator/apps/gitea/GiteaReconciler.kt    | 30 +++---
 .../operator/apps/gitea/GiteaStatus.kt        |  6 +-
 .../operator/apps/gitlab/GitlabReconciler.kt  | 10 +-
 .../operator/apps/gitlab/GitlabStatus.kt      |  5 +-
 .../apps/glitchtip/GlitchtipReconciler.kt     | 10 +-
 .../apps/glitchtip/GlitchtipStatus.kt         |  6 +-
 .../apps/keycloak/KeycloakReconciler.kt       | 10 +-
 .../operator/apps/keycloak/KeycloakStatus.kt  |  6 +-
 .../glasskube/operator/apps/matomo/Matomo.kt  |  6 +-
 .../operator/apps/matomo/MatomoReconciler.kt  | 21 ++--
 .../apps/metabase/MetabaseReconciler.kt       |  8 +-
 .../operator/apps/metabase/MetabaseStatus.kt  |  6 +-
 .../apps/nextcloud/NextcloudReconciler.kt     | 10 +-
 .../apps/nextcloud/NextcloudStatus.kt         |  6 +-
 .../eu/glasskube/operator/apps/odoo/Odoo.kt   |  5 +-
 .../operator/apps/odoo/OdooReconciler.kt      |  7 +-
 .../operator/apps/plane/PlaneReconciler.kt    | 10 +-
 .../operator/apps/plane/PlaneStatus.kt        |  6 +-
 .../operator/apps/vault/VaultReconciler.kt    |  9 +-
 .../operator/apps/vault/VaultStatus.kt        |  6 +-
 .../glasskube/operator/boot/OperatorConfig.kt |  4 +
 .../operator/exception/WebhookException.kt    |  7 ++
 .../operator/generic/BaseReconciler.kt        | 22 +++++
 .../operator/webhook/WebhookPayload.kt        | 15 +++
 .../operator/webhook/WebhookService.kt        | 97 +++++++++++++++++++
 .../kotlin/eu/glasskube/utils/ObjectMapper.kt | 16 +++
 28 files changed, 290 insertions(+), 65 deletions(-)
 create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/apps/common/database/HasReadyStatus.kt
 create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/exception/WebhookException.kt
 create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/generic/BaseReconciler.kt
 create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/webhook/WebhookPayload.kt
 create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/webhook/WebhookService.kt
 create mode 100644 operator/src/main/kotlin/eu/glasskube/utils/ObjectMapper.kt

diff --git a/deploy/crd/matomos.glasskube.eu-v1.yml b/deploy/crd/matomos.glasskube.eu-v1.yml
index 466fc3c0..8cb36412 100644
--- a/deploy/crd/matomos.glasskube.eu-v1.yml
+++ b/deploy/crd/matomos.glasskube.eu-v1.yml
@@ -80,6 +80,9 @@ spec:
                 type: object
             type: object
           status:
+            properties:
+              readyReplicas:
+                type: integer
             type: object
         type: object
     served: true
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/common/database/HasReadyStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/database/HasReadyStatus.kt
new file mode 100644
index 00000000..d89068c4
--- /dev/null
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/database/HasReadyStatus.kt
@@ -0,0 +1,8 @@
+package eu.glasskube.operator.apps.common.database
+
+import com.fasterxml.jackson.annotation.JsonIgnore
+
+interface HasReadyStatus {
+    @get:JsonIgnore
+    val isReady: Boolean
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitea/GiteaReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitea/GiteaReconciler.kt
index 3ed9c18d..f8d83dcf 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitea/GiteaReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitea/GiteaReconciler.kt
@@ -18,7 +18,9 @@ import eu.glasskube.operator.apps.gitea.dependent.GiteaSSHService
 import eu.glasskube.operator.apps.gitea.dependent.GiteaSecret
 import eu.glasskube.operator.apps.gitea.dependent.GiteaServiceMonitor
 import eu.glasskube.operator.apps.gitea.dependent.GiteaVolume
+import eu.glasskube.operator.generic.BaseReconciler
 import eu.glasskube.operator.infra.postgres.PostgresCluster
+import eu.glasskube.operator.webhook.WebhookService
 import eu.glasskube.utils.logger
 import io.fabric8.kubernetes.api.model.ConfigMap
 import io.fabric8.kubernetes.api.model.Service
@@ -27,7 +29,6 @@ import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.api.reconciler.UpdateControl
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 
@@ -98,24 +99,21 @@ import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
         )
     ]
 )
-class GiteaReconciler : Reconciler<Gitea>, EventSourceInitializer<Gitea> {
+class GiteaReconciler(webhookService: WebhookService) :
+    BaseReconciler<Gitea>(webhookService), EventSourceInitializer<Gitea> {
 
-    override fun reconcile(resource: Gitea, context: Context<Gitea>): UpdateControl<Gitea> {
-        log.info("Reconciling ${resource.metadata.name}@${resource.metadata.namespace}")
+    override fun processReconciliation(resource: Gitea, context: Context<Gitea>): UpdateControl<Gitea> = with(context) {
+        val deployment: Deployment? by secondaryResource(GiteaDeployment.Discriminator())
+        val redisDeployment: Deployment? by secondaryResource(GiteaRedisDeployment.Discriminator())
+        val postgresCluster: PostgresCluster? by secondaryResource()
 
-        return with(context) {
-            val deployment: Deployment? by secondaryResource(GiteaDeployment.Discriminator())
-            val redisDeployment: Deployment? by secondaryResource(GiteaRedisDeployment.Discriminator())
-            val postgresCluster: PostgresCluster? by secondaryResource()
-
-            resource.patchOrUpdateStatus(
-                GiteaStatus(
-                    readyReplicas = deployment?.status?.readyReplicas ?: 0,
-                    redisReady = redisDeployment?.status?.readyReplicas?.let { it > 0 } ?: false,
-                    postgresReady = postgresCluster?.status?.instances?.let { it > 0 } ?: false
-                )
+        resource.patchOrUpdateStatus(
+            GiteaStatus(
+                readyReplicas = deployment?.status?.readyReplicas ?: 0,
+                redisReady = redisDeployment?.status?.readyReplicas?.let { it > 0 } ?: false,
+                postgresReady = postgresCluster?.status?.instances?.let { it > 0 } ?: false
             )
-        }
+        )
     }
 
     override fun prepareEventSources(context: EventSourceContext<Gitea>) = with(context) {
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitea/GiteaStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitea/GiteaStatus.kt
index 046da4cd..b56fa7f1 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitea/GiteaStatus.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitea/GiteaStatus.kt
@@ -1,7 +1,11 @@
 package eu.glasskube.operator.apps.gitea
 
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
+
 data class GiteaStatus(
     val readyReplicas: Int,
     val redisReady: Boolean,
     val postgresReady: Boolean
-)
+) : HasReadyStatus {
+    override val isReady get() = readyReplicas > 0
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabReconciler.kt
index 349c298f..82353e0e 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabReconciler.kt
@@ -17,7 +17,9 @@ import eu.glasskube.operator.apps.gitlab.dependent.GitlabService
 import eu.glasskube.operator.apps.gitlab.dependent.GitlabServiceMonitor
 import eu.glasskube.operator.apps.gitlab.dependent.GitlabVolume
 import eu.glasskube.operator.apps.gitlab.runner.GitlabRunner
+import eu.glasskube.operator.generic.BaseReconciler
 import eu.glasskube.operator.infra.postgres.PostgresCluster
+import eu.glasskube.operator.webhook.WebhookService
 import eu.glasskube.utils.logger
 import io.fabric8.kubernetes.api.model.Service
 import io.fabric8.kubernetes.api.model.apps.Deployment
@@ -26,7 +28,6 @@ import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 
 @ControllerConfiguration(
@@ -88,9 +89,10 @@ import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
         )
     ]
 )
-class GitlabReconciler : Reconciler<Gitlab>, EventSourceInitializer<Gitlab> {
-    override fun reconcile(resource: Gitlab, context: Context<Gitlab>) = with(context) {
-        log.info("Reconciling ${resource.metadata.name}@${resource.metadata.namespace}")
+class GitlabReconciler(webhookService: WebhookService) :
+    BaseReconciler<Gitlab>(webhookService), EventSourceInitializer<Gitlab> {
+
+    override fun processReconciliation(resource: Gitlab, context: Context<Gitlab>) = with(context) {
         resource.patchOrUpdateStatus(
             GitlabStatus(
                 getSecondaryResource<Deployment>().map { it.status?.readyReplicas ?: 0 }.orElse(0),
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabStatus.kt
index 9feb1f03..ab88b993 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabStatus.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabStatus.kt
@@ -1,9 +1,12 @@
 package eu.glasskube.operator.apps.gitlab
 
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
 import eu.glasskube.operator.apps.gitlab.runner.GitlabRunnerStatus
 
 data class GitlabStatus(
     val readyReplicas: Int,
     val postgresReady: Boolean,
     val runners: Map<String, GitlabRunnerStatus?> = emptyMap()
-)
+) : HasReadyStatus {
+    override val isReady get() = readyReplicas > 0
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/glitchtip/GlitchtipReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/glitchtip/GlitchtipReconciler.kt
index da36486c..1154a5a8 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/glitchtip/GlitchtipReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/glitchtip/GlitchtipReconciler.kt
@@ -16,9 +16,11 @@ import eu.glasskube.operator.apps.glitchtip.dependent.GlitchtipRedisService
 import eu.glasskube.operator.apps.glitchtip.dependent.GlitchtipSecret
 import eu.glasskube.operator.apps.glitchtip.dependent.GlitchtipVolume
 import eu.glasskube.operator.apps.glitchtip.dependent.GlitchtipWorkerDeployment
+import eu.glasskube.operator.generic.BaseReconciler
 import eu.glasskube.operator.generic.condition.isReady
 import eu.glasskube.operator.infra.postgres.PostgresCluster
 import eu.glasskube.operator.infra.postgres.isReady
+import eu.glasskube.operator.webhook.WebhookService
 import eu.glasskube.utils.logger
 import io.fabric8.kubernetes.api.model.Service
 import io.fabric8.kubernetes.api.model.apps.Deployment
@@ -26,7 +28,6 @@ import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 import kotlin.jvm.optionals.getOrDefault
 
@@ -88,9 +89,10 @@ import kotlin.jvm.optionals.getOrDefault
         )
     ]
 )
-class GlitchtipReconciler : Reconciler<Glitchtip>, EventSourceInitializer<Glitchtip> {
-    override fun reconcile(resource: Glitchtip, context: Context<Glitchtip>) = with(context) {
-        log.info("Reconciling ${resource.metadata.name}@${resource.metadata.namespace}")
+class GlitchtipReconciler(webhookService: WebhookService) :
+    BaseReconciler<Glitchtip>(webhookService), EventSourceInitializer<Glitchtip> {
+
+    override fun processReconciliation(resource: Glitchtip, context: Context<Glitchtip>) = with(context) {
         resource.patchOrUpdateStatus(
             GlitchtipStatus(
                 readyReplicas = getSecondaryResource(GlitchtipDeployment.Discriminator())
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/glitchtip/GlitchtipStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/glitchtip/GlitchtipStatus.kt
index 8738f457..8811993e 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/glitchtip/GlitchtipStatus.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/glitchtip/GlitchtipStatus.kt
@@ -1,8 +1,12 @@
 package eu.glasskube.operator.apps.glitchtip
 
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
+
 data class GlitchtipStatus(
     val readyReplicas: Int,
     val workerReadyReplicas: Int,
     val redisReady: Boolean,
     val postgresReady: Boolean
-)
+) : HasReadyStatus {
+    override val isReady get() = readyReplicas > 0
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/keycloak/KeycloakReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/keycloak/KeycloakReconciler.kt
index 89705892..339aecc2 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/keycloak/KeycloakReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/keycloak/KeycloakReconciler.kt
@@ -11,16 +11,16 @@ import eu.glasskube.operator.apps.keycloak.dependent.KeycloakPostgresBackup
 import eu.glasskube.operator.apps.keycloak.dependent.KeycloakPostgresBackupBucket
 import eu.glasskube.operator.apps.keycloak.dependent.KeycloakPostgresCluster
 import eu.glasskube.operator.apps.keycloak.dependent.KeycloakService
+import eu.glasskube.operator.generic.BaseReconciler
 import eu.glasskube.operator.infra.postgres.PostgresCluster
 import eu.glasskube.operator.infra.postgres.isReady
+import eu.glasskube.operator.webhook.WebhookService
 import io.fabric8.kubernetes.api.model.Service
 import io.fabric8.kubernetes.api.model.apps.Deployment
 import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
-import io.javaoperatorsdk.operator.api.reconciler.UpdateControl
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 import kotlin.jvm.optionals.getOrDefault
 
@@ -59,8 +59,10 @@ import kotlin.jvm.optionals.getOrDefault
         Dependent(type = KeycloakIngress::class, name = "KeycloakIngress")
     ]
 )
-class KeycloakReconciler : Reconciler<Keycloak>, EventSourceInitializer<Keycloak> {
-    override fun reconcile(resource: Keycloak, context: Context<Keycloak>): UpdateControl<Keycloak> = with(context) {
+class KeycloakReconciler(webhookService: WebhookService) :
+    BaseReconciler<Keycloak>(webhookService), EventSourceInitializer<Keycloak> {
+
+    override fun processReconciliation(resource: Keycloak, context: Context<Keycloak>) = with(context) {
         resource.patchOrUpdateStatus(
             KeycloakStatus(
                 getSecondaryResource<Deployment>().map { it.status?.readyReplicas ?: 0 }.getOrDefault(0),
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/keycloak/KeycloakStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/keycloak/KeycloakStatus.kt
index 57ecfc3e..68be044b 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/keycloak/KeycloakStatus.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/keycloak/KeycloakStatus.kt
@@ -1,6 +1,10 @@
 package eu.glasskube.operator.apps.keycloak
 
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
+
 data class KeycloakStatus(
     val readyInstances: Int,
     val postgresReady: Boolean
-)
+) : HasReadyStatus {
+    override val isReady get() = readyInstances > 0
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/matomo/Matomo.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/matomo/Matomo.kt
index 6320165d..16e88bfc 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/matomo/Matomo.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/matomo/Matomo.kt
@@ -1,6 +1,7 @@
 package eu.glasskube.operator.apps.matomo
 
 import eu.glasskube.operator.apps.common.database.HasDatabaseSpec
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
 import eu.glasskube.operator.apps.common.database.ResourceWithDatabaseSpec
 import eu.glasskube.operator.apps.common.database.mariadb.MariaDbDatabaseSpec
 import eu.glasskube.utils.resourceLabels
@@ -26,9 +27,8 @@ data class MatomoSpec(
     override val database: MariaDbDatabaseSpec = MariaDbDatabaseSpec()
 ) : HasDatabaseSpec<MariaDbDatabaseSpec>
 
-class MatomoStatus {
-    override fun equals(other: Any?) = this === other || javaClass == other?.javaClass
-    override fun hashCode() = javaClass.hashCode()
+data class MatomoStatus(val readyReplicas: Int) : HasReadyStatus {
+    override val isReady get() = readyReplicas > 0
 }
 
 @Group("glasskube.eu")
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/matomo/MatomoReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/matomo/MatomoReconciler.kt
index dbcadf52..6b8b7b6b 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/matomo/MatomoReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/matomo/MatomoReconciler.kt
@@ -3,6 +3,7 @@ package eu.glasskube.operator.apps.matomo
 import eu.glasskube.kubernetes.client.patchOrUpdateStatus
 import eu.glasskube.kubernetes.client.resources
 import eu.glasskube.operator.api.reconciler.HasRegistrationCondition
+import eu.glasskube.operator.api.reconciler.getSecondaryResource
 import eu.glasskube.operator.api.reconciler.informerEventSource
 import eu.glasskube.operator.apps.matomo.Matomo.Companion.APP_NAME
 import eu.glasskube.operator.apps.matomo.dependent.MatomoConfigMap
@@ -14,18 +15,20 @@ import eu.glasskube.operator.apps.matomo.dependent.MatomoIngress
 import eu.glasskube.operator.apps.matomo.dependent.MatomoMariaDB
 import eu.glasskube.operator.apps.matomo.dependent.MatomoService
 import eu.glasskube.operator.apps.matomo.dependent.MatomoVolume
+import eu.glasskube.operator.generic.BaseReconciler
+import eu.glasskube.operator.webhook.WebhookService
 import eu.glasskube.utils.logger
 import io.fabric8.kubernetes.api.model.PersistentVolumeClaim
 import io.fabric8.kubernetes.api.model.Secret
 import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition
+import io.fabric8.kubernetes.api.model.apps.Deployment
 import io.fabric8.kubernetes.client.KubernetesClient
 import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
-import io.javaoperatorsdk.operator.api.reconciler.UpdateControl
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
+import kotlin.jvm.optionals.getOrNull
 
 @ControllerConfiguration(
     dependents = [
@@ -54,8 +57,8 @@ import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
         Dependent(type = MatomoCronJob::class, name = "MatomoCronJob", dependsOn = ["MatomoDeployment"])
     ]
 )
-class MatomoReconciler(private val kubernetesClient: KubernetesClient) :
-    Reconciler<Matomo>, EventSourceInitializer<Matomo>, HasRegistrationCondition {
+class MatomoReconciler(private val kubernetesClient: KubernetesClient, webhookService: WebhookService) :
+    BaseReconciler<Matomo>(webhookService), EventSourceInitializer<Matomo>, HasRegistrationCondition {
 
     override val isRegistrationEnabled
         get() = kubernetesClient.resources<CustomResourceDefinition>()
@@ -65,15 +68,19 @@ class MatomoReconciler(private val kubernetesClient: KubernetesClient) :
     override val registrationConditionHint =
         "CRDs provided by the MariaDB Operator must be present on the cluster."
 
-    override fun reconcile(resource: Matomo, context: Context<Matomo>): UpdateControl<Matomo> {
-        context.getSecondaryResources(PersistentVolumeClaim::class.java)
+    override fun processReconciliation(resource: Matomo, context: Context<Matomo>) = with(context) {
+        getSecondaryResources(PersistentVolumeClaim::class.java)
             .filter { it.metadata.name.endsWith("-misc") }
             .forEach {
                 log.info("Deleting old persisted volume claim ${it.metadata.name}")
                 kubernetesClient.persistentVolumeClaims().resource(it).delete()
             }
 
-        return resource.patchOrUpdateStatus(MatomoStatus())
+        resource.patchOrUpdateStatus(
+            MatomoStatus(
+                readyReplicas = getSecondaryResource<Deployment>().getOrNull()?.status?.readyReplicas ?: 0
+            )
+        )
     }
 
     override fun prepareEventSources(context: EventSourceContext<Matomo>) = with(context) {
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/metabase/MetabaseReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/metabase/MetabaseReconciler.kt
index 734e2e93..8004bf55 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/metabase/MetabaseReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/metabase/MetabaseReconciler.kt
@@ -12,12 +12,13 @@ import eu.glasskube.operator.apps.metabase.dependent.MetabasePostgresBackup
 import eu.glasskube.operator.apps.metabase.dependent.MetabasePostgresCluster
 import eu.glasskube.operator.apps.metabase.dependent.MetabaseSecret
 import eu.glasskube.operator.apps.metabase.dependent.MetabaseServiceMonitor
+import eu.glasskube.operator.generic.BaseReconciler
 import eu.glasskube.operator.infra.postgres.PostgresCluster
+import eu.glasskube.operator.webhook.WebhookService
 import eu.glasskube.utils.logger
 import io.fabric8.kubernetes.api.model.apps.Deployment
 import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 
 @ControllerConfiguration(
@@ -64,10 +65,9 @@ import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
         )
     ]
 )
-class MetabaseReconciler : Reconciler<Metabase> {
+class MetabaseReconciler(webhookService: WebhookService) : BaseReconciler<Metabase>(webhookService) {
 
-    override fun reconcile(resource: Metabase, context: Context<Metabase>) = with(context) {
-        log.info("Reconciling ${resource.metadata.name}@${resource.metadata.namespace}")
+    override fun processReconciliation(resource: Metabase, context: Context<Metabase>) = with(context) {
         resource.patchOrUpdateStatus(
             MetabaseStatus(
                 getSecondaryResource<Deployment>().map { it.status?.readyReplicas ?: 0 }.orElse(0),
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/metabase/MetabaseStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/metabase/MetabaseStatus.kt
index fd733590..2e7ab73d 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/metabase/MetabaseStatus.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/metabase/MetabaseStatus.kt
@@ -1,6 +1,10 @@
 package eu.glasskube.operator.apps.metabase
 
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
+
 data class MetabaseStatus(
     val readyReplicas: Int,
     val postgresReady: Boolean
-)
+) : HasReadyStatus {
+    override val isReady get() = readyReplicas > 0
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudReconciler.kt
index 727f9bde..a1b6ca45 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudReconciler.kt
@@ -17,9 +17,11 @@ import eu.glasskube.operator.apps.nextcloud.dependent.NextcloudRedisDeployment
 import eu.glasskube.operator.apps.nextcloud.dependent.NextcloudRedisService
 import eu.glasskube.operator.apps.nextcloud.dependent.NextcloudService
 import eu.glasskube.operator.apps.nextcloud.dependent.NextcloudVolume
+import eu.glasskube.operator.generic.BaseReconciler
 import eu.glasskube.operator.generic.condition.isReady
 import eu.glasskube.operator.infra.postgres.PostgresCluster
 import eu.glasskube.operator.infra.postgres.isReady
+import eu.glasskube.operator.webhook.WebhookService
 import eu.glasskube.utils.logger
 import io.fabric8.kubernetes.api.model.Service
 import io.fabric8.kubernetes.api.model.apps.Deployment
@@ -27,7 +29,6 @@ import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 import kotlin.jvm.optionals.getOrDefault
 
@@ -88,9 +89,10 @@ import kotlin.jvm.optionals.getOrDefault
         )
     ]
 )
-class NextcloudReconciler : Reconciler<Nextcloud>, EventSourceInitializer<Nextcloud> {
-    override fun reconcile(resource: Nextcloud, context: Context<Nextcloud>) = with(context) {
-        log.info("Reconciling ${resource.metadata.name}@${resource.metadata.namespace}")
+class NextcloudReconciler(webhookService: WebhookService) :
+    BaseReconciler<Nextcloud>(webhookService), EventSourceInitializer<Nextcloud> {
+
+    override fun processReconciliation(resource: Nextcloud, context: Context<Nextcloud>) = with(context) {
         resource.patchOrUpdateStatus(
             NextcloudStatus(
                 readyReplicas = getSecondaryResource(NextcloudDeployment.Discriminator())
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudStatus.kt
index f5e604e7..7699f8df 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudStatus.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudStatus.kt
@@ -1,8 +1,12 @@
 package eu.glasskube.operator.apps.nextcloud
 
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
+
 data class NextcloudStatus(
     val readyReplicas: Int,
     val redisReady: Boolean,
     val postgresReady: Boolean,
     val officeReady: Boolean
-)
+) : HasReadyStatus {
+    override val isReady get() = readyReplicas > 0
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/odoo/Odoo.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/odoo/Odoo.kt
index aeeab805..508c2aca 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/odoo/Odoo.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/odoo/Odoo.kt
@@ -1,6 +1,7 @@
 package eu.glasskube.operator.apps.odoo
 
 import eu.glasskube.operator.apps.common.database.HasDatabaseSpec
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
 import eu.glasskube.operator.apps.common.database.ResourceWithDatabaseSpec
 import eu.glasskube.operator.apps.common.database.postgres.PostgresDatabaseSpec
 import eu.glasskube.operator.generic.dependent.postgres.PostgresNameMapper
@@ -28,7 +29,9 @@ data class OdooSpec(
 data class OdooStatus(
     val ready: Boolean = false,
     val demoEnabledOnInstall: Boolean? = null
-)
+) : HasReadyStatus {
+    override val isReady get() = ready
+}
 
 @Group("glasskube.eu")
 @Version("v1alpha1")
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/odoo/OdooReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/odoo/OdooReconciler.kt
index caea104a..3c9fd3b8 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/odoo/OdooReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/odoo/OdooReconciler.kt
@@ -11,9 +11,10 @@ import eu.glasskube.operator.apps.odoo.dependent.OdooPersistentVolumeClaim
 import eu.glasskube.operator.apps.odoo.dependent.OdooPostgresCluster
 import eu.glasskube.operator.apps.odoo.dependent.OdooPostgresScheduledBackup
 import eu.glasskube.operator.apps.odoo.dependent.OdooService
+import eu.glasskube.operator.generic.BaseReconciler
+import eu.glasskube.operator.webhook.WebhookService
 import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.api.reconciler.UpdateControl
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 
@@ -54,8 +55,8 @@ import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
         )
     ]
 )
-class OdooReconciler : Reconciler<Odoo> {
-    override fun reconcile(resource: Odoo, context: Context<Odoo>): UpdateControl<Odoo> {
+class OdooReconciler(webhookService: WebhookService) : BaseReconciler<Odoo>(webhookService) {
+    override fun processReconciliation(resource: Odoo, context: Context<Odoo>): UpdateControl<Odoo> {
         check(resource.status?.demoEnabledOnInstall != !resource.spec.demoEnabled) {
             "demoEnabled can not be altered after first reconciliation"
         }
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneReconciler.kt
index f9988a4a..aa19d7a7 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneReconciler.kt
@@ -23,9 +23,11 @@ import eu.glasskube.operator.apps.plane.dependent.PlaneSpaceDeployment
 import eu.glasskube.operator.apps.plane.dependent.PlaneSpaceService
 import eu.glasskube.operator.apps.plane.dependent.PlaneWorkerConfigMap
 import eu.glasskube.operator.apps.plane.dependent.PlaneWorkerDeployment
+import eu.glasskube.operator.generic.BaseReconciler
 import eu.glasskube.operator.generic.condition.isReady
 import eu.glasskube.operator.infra.postgres.PostgresCluster
 import eu.glasskube.operator.infra.postgres.isReady
+import eu.glasskube.operator.webhook.WebhookService
 import eu.glasskube.utils.logger
 import io.fabric8.kubernetes.api.model.ConfigMap
 import io.fabric8.kubernetes.api.model.Service
@@ -34,7 +36,6 @@ import io.javaoperatorsdk.operator.api.reconciler.Context
 import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 import io.javaoperatorsdk.operator.processing.event.source.EventSource
 import java.util.Optional
@@ -142,9 +143,10 @@ import kotlin.jvm.optionals.getOrDefault
         )
     ]
 )
-class PlaneReconciler : Reconciler<Plane>, EventSourceInitializer<Plane> {
-    override fun reconcile(resource: Plane, context: Context<Plane>) = with(context) {
-        log.info("Reconciling ${resource.metadata.name}@${resource.metadata.namespace}")
+class PlaneReconciler(webhookService: WebhookService) :
+    BaseReconciler<Plane>(webhookService), EventSourceInitializer<Plane> {
+
+    override fun processReconciliation(resource: Plane, context: Context<Plane>) = with(context) {
         resource.patchOrUpdateStatus(
             PlaneStatus(
                 frontend = getSecondaryResource(PlaneFrontendDeployment.Discriminator()).getComponentStatus(),
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneStatus.kt
index d32b5e21..81b85978 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneStatus.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneStatus.kt
@@ -1,5 +1,7 @@
 package eu.glasskube.operator.apps.plane
 
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
+
 data class PlaneStatus(
     val frontend: ComponentStatus?,
     val space: ComponentStatus?,
@@ -8,6 +10,8 @@ data class PlaneStatus(
     val worker: ComponentStatus?,
     val database: ComponentStatus?,
     val redis: ComponentStatus?
-) {
+) : HasReadyStatus {
     data class ComponentStatus(val ready: Boolean)
+
+    override val isReady get() = frontend?.ready == true && api?.ready == true
 }
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/vault/VaultReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/vault/VaultReconciler.kt
index 64bcc2c5..f894a71b 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/vault/VaultReconciler.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/vault/VaultReconciler.kt
@@ -16,8 +16,10 @@ import eu.glasskube.operator.apps.vault.dependent.VaultService
 import eu.glasskube.operator.apps.vault.dependent.VaultServiceAccount
 import eu.glasskube.operator.apps.vault.dependent.VaultServiceHeadless
 import eu.glasskube.operator.apps.vault.dependent.VaultStatefulSet
+import eu.glasskube.operator.generic.BaseReconciler
 import eu.glasskube.operator.infra.postgres.PostgresCluster
 import eu.glasskube.operator.infra.postgres.isReady
+import eu.glasskube.operator.webhook.WebhookService
 import io.fabric8.kubernetes.api.model.Service
 import io.fabric8.kubernetes.api.model.apps.StatefulSet
 import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding
@@ -26,7 +28,6 @@ import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext
 import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer
 import io.javaoperatorsdk.operator.api.reconciler.MaxReconciliationInterval
-import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent
 import io.javaoperatorsdk.operator.processing.event.source.informer.Mappers
 import java.util.concurrent.TimeUnit
@@ -84,8 +85,10 @@ import kotlin.jvm.optionals.getOrDefault
     ],
     maxReconciliationInterval = MaxReconciliationInterval(interval = 10, timeUnit = TimeUnit.SECONDS)
 )
-class VaultReconciler : Reconciler<Vault>, EventSourceInitializer<Vault> {
-    override fun reconcile(resource: Vault, context: Context<Vault>) = with(context) {
+class VaultReconciler(webhookService: WebhookService) :
+    BaseReconciler<Vault>(webhookService), EventSourceInitializer<Vault> {
+
+    override fun processReconciliation(resource: Vault, context: Context<Vault>) = with(context) {
         resource.patchOrUpdateStatus(
             VaultStatus(
                 getSecondaryResource<StatefulSet>().map { it.status?.readyReplicas ?: 0 }.getOrDefault(0),
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/vault/VaultStatus.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/vault/VaultStatus.kt
index 49112478..6cc51a29 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/apps/vault/VaultStatus.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/vault/VaultStatus.kt
@@ -1,6 +1,10 @@
 package eu.glasskube.operator.apps.vault
 
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
+
 data class VaultStatus(
     val readyReplicas: Int,
     val postgresReady: Boolean
-)
+) : HasReadyStatus {
+    override val isReady get() = readyReplicas > 0
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/boot/OperatorConfig.kt b/operator/src/main/kotlin/eu/glasskube/operator/boot/OperatorConfig.kt
index 506e9806..d5aded77 100644
--- a/operator/src/main/kotlin/eu/glasskube/operator/boot/OperatorConfig.kt
+++ b/operator/src/main/kotlin/eu/glasskube/operator/boot/OperatorConfig.kt
@@ -18,6 +18,7 @@ import io.javaoperatorsdk.operator.api.config.ConfigurationService
 import io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider
 import io.javaoperatorsdk.operator.api.reconciler.Constants
 import io.javaoperatorsdk.operator.api.reconciler.Reconciler
+import okhttp3.OkHttpClient
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import java.security.SecureRandom
@@ -61,6 +62,9 @@ class OperatorConfig {
     fun random(): Random =
         SecureRandom.getInstanceStrong()
 
+    @Bean
+    fun okHttpClient(): OkHttpClient = OkHttpClient()
+
     private fun <T : HasMetadata> Operator.registerForNamespaceOrCluster(reconciler: Reconciler<T>): RegisteredController<T> =
         register(reconciler) { it.settingNamespaceFromEnv() }
 
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/exception/WebhookException.kt b/operator/src/main/kotlin/eu/glasskube/operator/exception/WebhookException.kt
new file mode 100644
index 00000000..81e1276c
--- /dev/null
+++ b/operator/src/main/kotlin/eu/glasskube/operator/exception/WebhookException.kt
@@ -0,0 +1,7 @@
+package eu.glasskube.operator.exception
+
+class WebhookException : RuntimeException {
+    constructor() : super()
+    constructor(message: String) : super(message)
+    constructor(message: String, cause: Throwable) : super(message, cause)
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/generic/BaseReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/generic/BaseReconciler.kt
new file mode 100644
index 00000000..5ae7ffbf
--- /dev/null
+++ b/operator/src/main/kotlin/eu/glasskube/operator/generic/BaseReconciler.kt
@@ -0,0 +1,22 @@
+package eu.glasskube.operator.generic
+
+import eu.glasskube.kubernetes.api.model.loggingId
+import eu.glasskube.operator.webhook.WebhookService
+import eu.glasskube.utils.logger
+import io.fabric8.kubernetes.client.CustomResource
+import io.javaoperatorsdk.operator.api.reconciler.Context
+import io.javaoperatorsdk.operator.api.reconciler.Reconciler
+import io.javaoperatorsdk.operator.api.reconciler.UpdateControl
+
+abstract class BaseReconciler<P : CustomResource<*, *>>(private val webhookService: WebhookService) : Reconciler<P> {
+    abstract fun processReconciliation(resource: P, context: Context<P>): UpdateControl<P>
+
+    final override fun reconcile(resource: P, context: Context<P>): UpdateControl<P> {
+        log.debug("{} reconciling", resource.loggingId)
+        return processReconciliation(resource, context).let { webhookService.sendStatusWebhook(resource, it) }
+    }
+
+    companion object {
+        private val log = logger()
+    }
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/webhook/WebhookPayload.kt b/operator/src/main/kotlin/eu/glasskube/operator/webhook/WebhookPayload.kt
new file mode 100644
index 00000000..220860e2
--- /dev/null
+++ b/operator/src/main/kotlin/eu/glasskube/operator/webhook/WebhookPayload.kt
@@ -0,0 +1,15 @@
+package eu.glasskube.operator.webhook
+
+data class WebhookPayload(val status: Status) {
+    enum class Status {
+        READY, NOT_READY;
+
+        companion object {
+            fun from(isReady: Boolean) = if (isReady) READY else NOT_READY
+        }
+    }
+
+    companion object {
+        fun from(isReady: Boolean) = WebhookPayload(Status.from(isReady))
+    }
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/operator/webhook/WebhookService.kt b/operator/src/main/kotlin/eu/glasskube/operator/webhook/WebhookService.kt
new file mode 100644
index 00000000..355c916a
--- /dev/null
+++ b/operator/src/main/kotlin/eu/glasskube/operator/webhook/WebhookService.kt
@@ -0,0 +1,97 @@
+package eu.glasskube.operator.webhook
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import eu.glasskube.kubernetes.api.model.loggingId
+import eu.glasskube.operator.apps.common.database.HasReadyStatus
+import eu.glasskube.operator.exception.WebhookException
+import eu.glasskube.utils.logger
+import eu.glasskube.utils.responseBody
+import io.fabric8.kubernetes.api.model.HasMetadata
+import io.fabric8.kubernetes.client.CustomResource
+import io.javaoperatorsdk.operator.api.reconciler.UpdateControl
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.internal.closeQuietly
+import org.springframework.stereotype.Component
+import java.io.IOException
+import java.util.concurrent.TimeUnit
+
+@Component
+class WebhookService(
+    private val okHttpClient: OkHttpClient,
+    private val objectMapper: ObjectMapper
+) {
+    fun <P : CustomResource<*, *>> sendStatusWebhook(resource: P, updateControl: UpdateControl<P>): UpdateControl<P> {
+        val url = resource.metadata.annotations[WEBHOOK_ANNOTATION] ?: return updateControl
+        val status = resource.status
+
+        if (status !is HasReadyStatus) {
+            log.warn("{} webhook annotation found but status is not HasReadyStatus", resource.loggingId)
+            return updateControl
+        }
+
+        return if (updateControl.isUpdateStatus || WEBHOOK_ERROR_ANNOTATION in resource.metadata.annotations) {
+            log.debug("{} sending webhook", resource.loggingId)
+            doSendWebhookAndAnnotateResource(updateControl, resource, url, status.toWebhookPayload())
+        } else {
+            log.debug("{} webhook configured but status did not change", resource.loggingId)
+            updateControl
+        }
+    }
+
+    private fun HasReadyStatus.toWebhookPayload() = WebhookPayload.from(isReady)
+
+    private fun <P : HasMetadata> doSendWebhookAndAnnotateResource(
+        updateControl: UpdateControl<P>,
+        resource: P,
+        url: String,
+        payload: WebhookPayload
+    ): UpdateControl<P> = try {
+        doSendWebhook(resource, url, payload)
+        log.debug("{} webhook sent", resource.loggingId)
+        annotateAfterSuccess(updateControl, resource)
+    } catch (e: WebhookException) {
+        log.warn("${resource.loggingId} error sending webhook. scheduling for retry in 1 minute", e)
+        annotateAfterError(updateControl, resource).rescheduleAfter(1, TimeUnit.MINUTES)
+    }
+
+    private fun doSendWebhook(resource: HasMetadata, url: String, payload: WebhookPayload) {
+        val request = Request.Builder().url(url).post(objectMapper.responseBody(payload)).build()
+        try {
+            val response = okHttpClient.newCall(request).execute()
+            response.closeQuietly()
+            if (!response.isSuccessful) {
+                throw WebhookException("error response ${response.code} calling $url for resource ${resource.loggingId}")
+            }
+        } catch (e: IOException) {
+            throw WebhookException("failed to call $url for resource ${resource.loggingId}", e)
+        }
+    }
+
+    private fun <P : HasMetadata> annotateAfterError(updateControl: UpdateControl<P>, resource: P): UpdateControl<P> =
+        if (resource.metadata.annotations.put(WEBHOOK_ERROR_ANNOTATION, "true") == null) {
+            updateControl.alsoUpdateResource(resource)
+        } else {
+            updateControl
+        }
+
+    private fun <P : HasMetadata> annotateAfterSuccess(updateControl: UpdateControl<P>, resource: P): UpdateControl<P> =
+        if (resource.metadata.annotations.remove(WEBHOOK_ERROR_ANNOTATION) != null) {
+            updateControl.alsoUpdateResource(resource)
+        } else {
+            updateControl
+        }
+
+    private fun <P : HasMetadata> UpdateControl<P>.alsoUpdateResource(resource: P) = when {
+        isUpdateResource -> this
+        isPatchStatus -> UpdateControl.updateResourceAndPatchStatus(resource)
+        isUpdateStatus -> UpdateControl.updateResourceAndStatus(resource)
+        else -> UpdateControl.updateResource(resource)
+    }
+
+    companion object {
+        private val log = logger()
+        private const val WEBHOOK_ANNOTATION = "glasskube.eu/webhook"
+        private const val WEBHOOK_ERROR_ANNOTATION = "glasskube.eu/webhook-error"
+    }
+}
diff --git a/operator/src/main/kotlin/eu/glasskube/utils/ObjectMapper.kt b/operator/src/main/kotlin/eu/glasskube/utils/ObjectMapper.kt
new file mode 100644
index 00000000..f6ece5ad
--- /dev/null
+++ b/operator/src/main/kotlin/eu/glasskube/utils/ObjectMapper.kt
@@ -0,0 +1,16 @@
+package eu.glasskube.utils
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.RequestBody
+import okio.BufferedSink
+
+private val MEDIA_TYPE_JSON = "application/json".toMediaType()
+
+fun ObjectMapper.responseBody(value: Any?): RequestBody =
+    object : RequestBody() {
+        override fun contentType() = MEDIA_TYPE_JSON
+        override fun writeTo(sink: BufferedSink) {
+            writeValue(sink.outputStream(), value)
+        }
+    }