Skip to content
This repository has been archived by the owner on Aug 19, 2024. It is now read-only.

Commit

Permalink
Add error messages to backstage CR status (#149)
Browse files Browse the repository at this point in the history 
* Add error messages to backstage CR status

* Add error messages to backstage CR status

* Simplify backstage CR status conditions

* Group constants

* Adding error message checking

* Rename synced status to deployed
  • Loading branch information
@jianrongzhang89
jianrongzhang89 authored Jan 24, 2024
1 parent bc9d3aa commit 5f2b8df
Show file tree
Hide file tree
Showing 9 changed files with 100 additions and 114 deletions.
21 changes: 12 additions & 9 deletions api/v1alpha1/backstage_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Constants for status conditions
const (
RuntimeConditionRunning string = "RuntimeRunning"
RuntimeConditionSynced string = "RuntimeSyncedWithConfig"
RouteSynced string = "RouteSynced"
LocalDbSynced string = "LocalDbSynced"
SyncOK string = "SyncOK"
SyncFailed string = "SyncFailed"
Deleted string = "Deleted"
EnvPostGresImage string = "RELATED_IMAGE_postgresql"
EnvBackstageImage string = "RELATED_IMAGE_backstage"
// TODO: RuntimeConditionRunning string = "RuntimeRunning"
ConditionDeployed string = "Deployed"
DeployOK string = "DeployOK"
DeployFailed string = "DeployFailed"
DeployInProgress string = "DeployInProgress"
)

// Constants for image placeholders
const (
EnvPostGresImage string = "RELATED_IMAGE_postgresql"
EnvBackstageImage string = "RELATED_IMAGE_backstage"
)

// BackstageSpec defines the desired state of Backstage
Expand Down
2 changes: 1 addition & 1 deletion bundle/manifests/backstage-operator.clusterserviceversion.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ metadata:
}
]
capabilities: Basic Install
createdAt: "2024-01-12T14:31:03Z"
createdAt: "2024-01-19T01:12:25Z"
operators.operatorframework.io/builder: operator-sdk-v1.33.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
name: backstage-operator.v0.0.1
Expand Down
62 changes: 22 additions & 40 deletions controllers/backstage_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,13 +103,17 @@ func (r *BackstageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
defer func(bs *bs.Backstage) {
if err := r.Client.Status().Update(ctx, bs); err != nil {
if errors.IsConflict(err) {
lg.V(1).Info("Backstage object modified, retry syncing status", "Backstage Object", bs)
lg.V(1).Info("Backstage object modified, retry reconciliation", "Backstage Object", bs)
return
}
lg.Error(err, "Error updating the Backstage resource status", "Backstage Object", bs)
}
}(&backstage)

if len(backstage.Status.Conditions) == 0 {
setStatusCondition(&backstage, bs.ConditionDeployed, v1.ConditionFalse, bs.DeployInProgress, "Deployment process started")
}

if pointer.BoolDeref(backstage.Spec.Database.EnableLocalDb, true) {

/* We use default strogeclass currently, and no PV is needed in that case.
Expand All @@ -119,33 +123,29 @@ func (r *BackstageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
*/

err := r.reconcileLocalDbStatefulSet(ctx, backstage, req.Namespace)
err := r.reconcileLocalDbStatefulSet(ctx, &backstage, req.Namespace)
if err != nil {
setStatusCondition(&backstage, bs.LocalDbSynced, v1.ConditionFalse, bs.SyncFailed, fmt.Sprintf("failed to sync Database StatefulSet:%s", err.Error()))
return ctrl.Result{}, fmt.Errorf("failed to sync Database StatefulSet: %w", err)
return ctrl.Result{}, err
}

err = r.reconcileLocalDbServices(ctx, backstage, req.Namespace)
err = r.reconcileLocalDbServices(ctx, &backstage, req.Namespace)
if err != nil {
setStatusCondition(&backstage, bs.LocalDbSynced, v1.ConditionFalse, bs.SyncFailed, fmt.Sprintf("failed to sync Database Services:%s", err.Error()))
return ctrl.Result{}, fmt.Errorf("failed to sync Database Service: %w", err)
return ctrl.Result{}, err
}
setStatusCondition(&backstage, bs.LocalDbSynced, v1.ConditionTrue, bs.SyncOK, "")
} else { // Clean up the deployed local db resources if any
if err := r.cleanupLocalDbResources(ctx, backstage); err != nil {
setStatusCondition(&backstage, bs.LocalDbSynced, v1.ConditionFalse, bs.SyncFailed, fmt.Sprintf("failed to delete Database Services:%s", err.Error()))
setStatusCondition(&backstage, bs.ConditionDeployed, v1.ConditionFalse, bs.DeployFailed, fmt.Sprintf("failed to delete Database Services:%s", err.Error()))
return ctrl.Result{}, fmt.Errorf("failed to delete Database Service: %w", err)
}
setStatusCondition(&backstage, bs.LocalDbSynced, v1.ConditionTrue, bs.Deleted, "")
}

err := r.reconcileBackstageDeployment(ctx, backstage, req.Namespace)
err := r.reconcileBackstageDeployment(ctx, &backstage, req.Namespace)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to reconcile Backstage Deployment: %w", err)
return ctrl.Result{}, err
}

if err := r.reconcileBackstageService(ctx, backstage, req.Namespace); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to reconcile Backstage Service: %w", err)
if err := r.reconcileBackstageService(ctx, &backstage, req.Namespace); err != nil {
return ctrl.Result{}, err
}

if r.IsOpenShift {
Expand All @@ -154,10 +154,7 @@ func (r *BackstageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
}

//TODO: it is just a placeholder for the time
r.setRunningStatus(ctx, &backstage, req.Namespace)
r.setSyncStatus(&backstage)

setStatusCondition(&backstage, bs.ConditionDeployed, v1.ConditionTrue, bs.DeployOK, "")
return ctrl.Result{}, nil
}

Expand Down Expand Up @@ -219,7 +216,8 @@ func defFile(key string) string {
return filepath.Join(os.Getenv("LOCALBIN"), "default-config", key)
}

// sets the RuntimeRunning condition
/* TODO
sets the RuntimeRunning condition
func (r *BackstageReconciler) setRunningStatus(ctx context.Context, backstage *bs.Backstage, ns string) {
meta.SetStatusCondition(&backstage.Status.Conditions, v1.Condition{
Expand All @@ -230,27 +228,7 @@ func (r *BackstageReconciler) setRunningStatus(ctx context.Context, backstage *b
Message: "Runtime in unknown status",
})
}

// sets the RuntimeSyncedWithConfig condition
func (r *BackstageReconciler) setSyncStatus(backstage *bs.Backstage) {

status := v1.ConditionUnknown
reason := "Unknown"
message := "Sync in unknown status"
if r.OwnsRuntime {
status = v1.ConditionTrue
reason = "Synced"
message = "Backstage syncs runtime"
}

meta.SetStatusCondition(&backstage.Status.Conditions, v1.Condition{
Type: bs.RuntimeConditionSynced,
Status: status,
LastTransitionTime: v1.Time{},
Reason: reason,
Message: message,
})
}
*/

// sets status condition
func setStatusCondition(backstage *bs.Backstage, condType string, status v1.ConditionStatus, reason, msg string) {
Expand Down Expand Up @@ -337,3 +315,7 @@ func (r *BackstageReconciler) SetupWithManager(mgr ctrl.Manager, log logr.Logger

return builder.Complete(r)
}

func retryReconciliation(err error) error {
return fmt.Errorf("reconciliation retry needed: %v", err)
}
55 changes: 25 additions & 30 deletions controllers/backstage_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,20 @@ var _ = Describe("Backstage controller", func() {
var backstage bsv1alpha1.Backstage
err := k8sClient.Get(ctx, types.NamespacedName{Name: backstageName, Namespace: ns}, &backstage)
g.Expect(err).NotTo(HaveOccurred())
//TODO the status is under construction
g.Expect(isSynced(backstage)).To(BeTrue())
g.Expect(isDeployed(backstage)).To(BeTrue())
}, time.Minute, time.Second).Should(Succeed())
}

verifyBackstageInstanceError := func(ctx context.Context, errMsg string) {
Eventually(func(g Gomega) {
var backstage bsv1alpha1.Backstage
err := k8sClient.Get(ctx, types.NamespacedName{Name: backstageName, Namespace: ns}, &backstage)
g.Expect(err).NotTo(HaveOccurred())
cond := meta.FindStatusCondition(backstage.Status.Conditions, bsv1alpha1.ConditionDeployed)
g.Expect(cond).NotTo(BeNil())
g.Expect(cond.Status).To(Equal(metav1.ConditionFalse))
g.Expect(cond.Reason).To(Equal(bsv1alpha1.DeployFailed))
g.Expect(cond.Message).To(ContainSubstring(errMsg))
}, time.Minute, time.Second).Should(Succeed())
}

Expand Down Expand Up @@ -371,14 +383,6 @@ var _ = Describe("Backstage controller", func() {
By("Checking the latest Status added to the Backstage instance")
verifyBackstageInstance(ctx)

By("Checking the localDb Sync Status in the Backstage instance")
Eventually(func(g Gomega) {
var backstage bsv1alpha1.Backstage
err := k8sClient.Get(ctx, types.NamespacedName{Name: backstageName, Namespace: ns}, &backstage)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(isLocalDbDeployed(backstage)).To(BeTrue())
}, time.Minute, time.Second).Should(Succeed())

By("Checking the localdb statefulset has been created")
Eventually(func(g Gomega) {
err := k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("backstage-psql-%s", backstageName), Namespace: ns}, &appsv1.StatefulSet{})
Expand Down Expand Up @@ -417,14 +421,6 @@ var _ = Describe("Backstage controller", func() {
})
Expect(err).To(Not(HaveOccurred()))

By("Checking the localDb Sync Status has been updated in the Backstage instance")
Eventually(func(g Gomega) {
var backstage bsv1alpha1.Backstage
err := k8sClient.Get(ctx, types.NamespacedName{Name: backstageName, Namespace: ns}, &backstage)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(isLocalDbDeployed(backstage)).To(BeFalse())
}, time.Minute, time.Second).Should(Succeed())

By("Checking that the local db statefulset has been deleted")
Eventually(func(g Gomega) {
err := k8sClient.Get(ctx,
Expand Down Expand Up @@ -604,13 +600,13 @@ spec:
Context("App Configs", func() {
When("referencing non-existing ConfigMap as app-config", func() {
var backstage *bsv1alpha1.Backstage

const cmName = "a-non-existing-cm"
BeforeEach(func() {
backstage = buildBackstageCR(bsv1alpha1.BackstageSpec{
Application: &bsv1alpha1.Application{
AppConfig: &bsv1alpha1.AppConfig{
ConfigMaps: []bsv1alpha1.ObjectKeyRef{
{Name: "a-non-existing-cm"},
{Name: cmName},
},
},
},
Expand All @@ -631,6 +627,9 @@ spec:
NamespacedName: types.NamespacedName{Name: backstageName, Namespace: ns},
})
Expect(err).To(HaveOccurred())
errStr := fmt.Sprintf("failed to add volume mounts to Backstage deployment, reason: configmaps \"%s\" not found", cmName)
Expect(err.Error()).Should(ContainSubstring(errStr))
verifyBackstageInstanceError(ctx, errStr)

By("Not creating a Backstage Deployment")
Consistently(func() error {
Expand Down Expand Up @@ -855,13 +854,13 @@ plugins: []
kind := kind
When(fmt.Sprintf("referencing non-existing %s as extra-file", kind), func() {
var backstage *bsv1alpha1.Backstage
name := "a-non-existing-" + strings.ToLower(kind)

BeforeEach(func() {
var (
cmExtraFiles []bsv1alpha1.ObjectKeyRef
secExtraFiles []bsv1alpha1.ObjectKeyRef
)
name := "a-non-existing-" + strings.ToLower(kind)
switch kind {
case "ConfigMap":
cmExtraFiles = append(cmExtraFiles, bsv1alpha1.ObjectKeyRef{Name: name})
Expand Down Expand Up @@ -892,6 +891,9 @@ plugins: []
NamespacedName: types.NamespacedName{Name: backstageName, Namespace: ns},
})
Expect(err).To(HaveOccurred())
errStr := fmt.Sprintf("failed to add volume mounts to Backstage deployment, reason: %ss \"%s\" not found", strings.ToLower(kind), name)
Expect(err.Error()).Should(ContainSubstring(errStr))
verifyBackstageInstanceError(ctx, errStr)

By("Not creating a Backstage Deployment")
Consistently(func() error {
Expand Down Expand Up @@ -1525,15 +1527,8 @@ func findElementsByPredicate[T any](l []T, predicate func(t T) bool) (result []T
return result
}

func isLocalDbDeployed(backstage bsv1alpha1.Backstage) bool {
if cond := meta.FindStatusCondition(backstage.Status.Conditions, bsv1alpha1.LocalDbSynced); cond != nil {
return cond.Status == metav1.ConditionTrue && cond.Reason == bsv1alpha1.SyncOK
}
return false
}

func isSynced(backstage bsv1alpha1.Backstage) bool {
if cond := meta.FindStatusCondition(backstage.Status.Conditions, bsv1alpha1.RuntimeConditionSynced); cond != nil {
func isDeployed(backstage bsv1alpha1.Backstage) bool {
if cond := meta.FindStatusCondition(backstage.Status.Conditions, bsv1alpha1.ConditionDeployed); cond != nil {
return cond.Status == metav1.ConditionTrue
}
return false
Expand Down
12 changes: 7 additions & 5 deletions controllers/backstage_deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,16 +146,18 @@ func visitContainers(podTemplateSpec *v1.PodTemplateSpec, visitor ContainerVisit
}
}

func (r *BackstageReconciler) reconcileBackstageDeployment(ctx context.Context, backstage bs.Backstage, ns string) error {
func (r *BackstageReconciler) reconcileBackstageDeployment(ctx context.Context, backstage *bs.Backstage, ns string) error {
deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{
Name: getDefaultObjName(backstage),
Name: getDefaultObjName(*backstage),
Namespace: ns},
}
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, r.deploymentObjectMutFun(ctx, deployment, backstage, ns)); err != nil {
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, r.deploymentObjectMutFun(ctx, deployment, *backstage, ns)); err != nil {
if errors.IsConflict(err) {
return fmt.Errorf("retry sync needed: %v", err)
return retryReconciliation(err)
}
return err
msg := fmt.Sprintf("failed to deploy Backstage Deployment: %s", err)
setStatusCondition(backstage, bs.ConditionDeployed, metav1.ConditionFalse, bs.DeployFailed, msg)
return fmt.Errorf(msg)
}
return nil
}
Expand Down
17 changes: 9 additions & 8 deletions controllers/backstage_route.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,21 +38,22 @@ func (r *BackstageReconciler) reconcileBackstageRoute(ctx context.Context, backs
}

if !shouldCreateRoute(*backstage) {
deleted, err := r.cleanupResource(ctx, route, *backstage)
if err == nil && deleted {
setStatusCondition(backstage, bs.RouteSynced, metav1.ConditionTrue, bs.Deleted, "")
_, err := r.cleanupResource(ctx, route, *backstage)
if err != nil {
setStatusCondition(backstage, bs.ConditionDeployed, metav1.ConditionFalse, bs.DeployFailed, fmt.Sprintf("failed to delete route: %s", err))
return err
}
return err
return nil
}

if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, route, r.routeObjectMutFun(ctx, route, *backstage, ns)); err != nil {
if errors.IsConflict(err) {
return fmt.Errorf("retry sync needed: %v", err)
return retryReconciliation(err)
}
setStatusCondition(backstage, bs.RouteSynced, metav1.ConditionFalse, bs.SyncFailed, fmt.Sprintf("Error:%s", err.Error()))
return err
msg := fmt.Sprintf("failed to deploy Backstage Route: %s", err)
setStatusCondition(backstage, bs.ConditionDeployed, metav1.ConditionFalse, bs.DeployFailed, msg)
return fmt.Errorf(msg)
}
setStatusCondition(backstage, bs.RouteSynced, metav1.ConditionTrue, bs.SyncOK, fmt.Sprintf("Route host:%s", route.Spec.Host))
return nil
}

Expand Down
11 changes: 6 additions & 5 deletions controllers/backstage_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,20 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func (r *BackstageReconciler) reconcileBackstageService(ctx context.Context, backstage bs.Backstage, ns string) error {
func (r *BackstageReconciler) reconcileBackstageService(ctx context.Context, backstage *bs.Backstage, ns string) error {
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: getDefaultObjName(backstage),
Name: getDefaultObjName(*backstage),
Namespace: ns,
},
}
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, r.serviceObjectMutFun(ctx, service, backstage,
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, r.serviceObjectMutFun(ctx, service, *backstage,
backstage.Spec.RawRuntimeConfig.BackstageConfigName, "service.yaml", service.Name, service.Name)); err != nil {
if errors.IsConflict(err) {
return fmt.Errorf("retry sync needed: %v", err)
return retryReconciliation(err)
}
return err
msg := fmt.Sprintf("failed to deploy Backstage Service: %s", err)
setStatusCondition(backstage, bs.ConditionDeployed, metav1.ConditionFalse, bs.DeployFailed, msg)
}
return nil
}
Expand Down
13 changes: 7 additions & 6 deletions controllers/local_db_services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ import (
//
// `
// )
func (r *BackstageReconciler) reconcileLocalDbServices(ctx context.Context, backstage bs.Backstage, ns string) error {
name := getDefaultDbObjName(backstage)
func (r *BackstageReconciler) reconcileLocalDbServices(ctx context.Context, backstage *bs.Backstage, ns string) error {
name := getDefaultDbObjName(*backstage)
err := r.reconcilePsqlService(ctx, backstage, name, name, "db-service.yaml", ns)
if err != nil {
return err
Expand All @@ -72,18 +72,19 @@ func (r *BackstageReconciler) reconcileLocalDbServices(ctx context.Context, back

}

func (r *BackstageReconciler) reconcilePsqlService(ctx context.Context, backstage bs.Backstage, serviceName, label, configKey, ns string) error {
func (r *BackstageReconciler) reconcilePsqlService(ctx context.Context, backstage *bs.Backstage, serviceName, label, configKey, ns string) error {
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: serviceName,
Namespace: ns,
},
}
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, r.serviceObjectMutFun(ctx, service, backstage, backstage.Spec.RawRuntimeConfig.LocalDbConfigName, configKey, serviceName, label)); err != nil {
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, r.serviceObjectMutFun(ctx, service, *backstage, backstage.Spec.RawRuntimeConfig.LocalDbConfigName, configKey, serviceName, label)); err != nil {
if errors.IsConflict(err) {
return fmt.Errorf("retry sync needed: %v", err)
return retryReconciliation(err)
}
return err
msg := fmt.Sprintf("failed to deploy database service: %s", err)
setStatusCondition(backstage, bs.ConditionDeployed, metav1.ConditionFalse, bs.DeployFailed, msg)
}
return nil
}
Loading

0 comments on commit 5f2b8df

Please sign in to comment.