Added a workaround for postgres upgrade failures (#1396) (#1398)

If Postgres is not upgraded to postgres-15 and is still using postgres-12 image, continue to use postgres12 image, but set the noobaa CR as rejected. This is to prevent noobaa-db going into CLBO Signed-off-by: Danny Zaken <[email protected]> (cherry picked from commit 992d39b) Co-authored-by: Danny Zaken <[email protected]>
noobaa · Jul 26, 2024 · e5bedcd · e5bedcd
1 parent c9b7fdb
commit e5bedcd
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 10 deletions.
diff --git a/Makefile b/Makefile
@@ -156,7 +156,7 @@ gen-olm: gen
 
 gen-odf-package: cli
 	rm -rf $(MANIFESTS)
-	MANIFESTS="$(MANIFESTS)" CSV_NAME="$(csv-name)" SKIP_RANGE="$(skip-range)" REPLACES="$(replaces)" CORE_IMAGE="$(core-image)" DB_IMAGE="$(db-image)" OPERATOR_IMAGE="$(operator-image)" COSI_SIDECAR_IMAGE="$(cosi-sidecar-image)" OBC_CRD="$(obc-crd)" build/gen-odf-package.sh
+	MANIFESTS="$(MANIFESTS)" CSV_NAME="$(csv-name)" SKIP_RANGE="$(skip-range)" REPLACES="$(replaces)" CORE_IMAGE="$(core-image)" DB_IMAGE="$(db-image)" OPERATOR_IMAGE="$(operator-image)" COSI_SIDECAR_IMAGE="$(cosi-sidecar-image)" OBC_CRD="$(obc-crd)" PSQL_12_IMAGE="$(psql-12-image)" build/gen-odf-package.sh
 	@echo "✅ gen-odf-package"
 .PHONY: gen-odf-package
 

diff --git a/build/gen-odf-package.sh b/build/gen-odf-package.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [ "${MANIFESTS}" == "" ] || [ "${CSV_NAME}" == "" ] || [ "${CORE_IMAGE}" == "" ] || [ "${DB_IMAGE}" == "" ] || [ "${OPERATOR_IMAGE}" == "" ] || [ "${COSI_SIDECAR_IMAGE}" == "" ]
+if [ "${MANIFESTS}" == "" ] || [ "${CSV_NAME}" == "" ] || [ "${CORE_IMAGE}" == "" ] || [ "${PSQL_12_IMAGE}" == "" ] || [ "${DB_IMAGE}" == "" ] || [ "${OPERATOR_IMAGE}" == "" ] || [ "${COSI_SIDECAR_IMAGE}" == "" ]
 then
   echo "gen-odf-package.sh: not all required envs were supplied"
   exit 1
@@ -16,6 +16,7 @@ echo "--obc-crd=${OBC_CRD}"
 --replaces "${REPLACES}" \
 --noobaa-image ${CORE_IMAGE} \
 --db-image ${DB_IMAGE} \
+--psql-12-image ${PSQL_12_IMAGE} \
 --operator-image ${OPERATOR_IMAGE} \
 --cosi-sidecar-image ${COSI_SIDECAR_IMAGE} \
 --obc-crd=${OBC_CRD} 
@@ -34,6 +35,8 @@ cat >> ${temp_csv} << EOF
     name: noobaa-core
   - image: ${DB_IMAGE}
     name: noobaa-db
+  - image: ${PSQL_12_IMAGE}
+    name: noobaa-psql-12
   - image: ${OPERATOR_IMAGE}
     name: noobaa-operator
 EOF

diff --git a/pkg/olm/olm.go b/pkg/olm/olm.go
@@ -297,6 +297,10 @@ func GenerateCSV(opConf *operator.Conf, csvParams *generateCSVParams) *operv1.Cl
 					Name:  "NOOBAA_DB_IMAGE",
 					Value: options.DBImage,
 				},
+				corev1.EnvVar{
+					Name:  "NOOBAA_PSQL_12_IMAGE",
+					Value: options.Psql12Image,
+				},
 				corev1.EnvVar{
 					Name:  "ENABLE_NOOBAA_ADMISSION",
 					Value: "true",

diff --git a/pkg/options/options.go b/pkg/options/options.go
@@ -74,6 +74,10 @@ var NooBaaImage = ContainerImage
 // it can be overridden for testing or different registry locations.
 var DBImage = "quay.io/sclorg/postgresql-15-c9s"
 
+// Psql12Image is the default postgres12 db image url
+// currently it can not be overridden.
+var Psql12Image = "centos/postgresql-12-centos7"
+
 // DBType is the default db image type
 // it can be overridden for testing or different types.
 var DBType = "postgres"
@@ -214,6 +218,10 @@ func init() {
 		&DBImage, "db-image",
 		DBImage, "The database container image",
 	)
+	FlagSet.StringVar(
+		&Psql12Image, "psql-12-image",
+		Psql12Image, "The database old container image",
+	)
 	FlagSet.StringVar(
 		&CosiSideCarImage, "cosi-sidecar-image",
 		CosiSideCarImage, "The cosi side car container image",

diff --git a/pkg/system/phase2_creating.go b/pkg/system/phase2_creating.go
@@ -247,7 +247,7 @@ func (r *Reconciler) SetDesiredNooBaaDB() error {
 	for i := range podSpec.Containers {
 		c := &podSpec.Containers[i]
 		if c.Name == "db" {
-			c.Image = GetDesiredDBImage(r.NooBaa)
+			c.Image = GetDesiredDBImage(r.NooBaa, c.Image)
 			if r.NooBaa.Spec.DBResources != nil {
 				c.Resources = *r.NooBaa.Spec.DBResources
 			}

diff --git a/pkg/system/reconciler.go b/pkg/system/reconciler.go
@@ -421,12 +421,22 @@ func (r *Reconciler) Reconcile() (reconcile.Result, error) {
 			log.Warnf("⏳ Temporary Error: %s", err)
 		}
 	} else {
-		r.SetPhase(
-			nbv1.SystemPhaseReady,
-			"SystemPhaseReady",
-			"noobaa operator completed reconcile - system is ready",
-		)
-		log.Infof("✅ Done")
+		// Postgres upgrade failure workaround
+		// if the system reconciliation has no other error, but the postgres image is still postresql-12 image, set the status to Rejected
+		if IsPostgresql12Image(r.NooBaaPostgresDB.Spec.Template.Spec.Containers[0].Image) {
+			r.SetPhase(nbv1.SystemPhaseRejected,
+				"PostgresImageVersion",
+				"Noobaa is using Postgresql-12 which indicates a failure to upgrade to Postgresql-15. Please contact support.")
+			log.Errorf("❌ Postgres image version is set to postgresql-12. Indicates a failure to upgrade to Postgresql-15. Please contact support.")
+		} else {
+			r.SetPhase(
+				nbv1.SystemPhaseReady,
+				"SystemPhaseReady",
+				"noobaa operator completed reconcile - system is ready",
+			)
+			log.Infof("✅ Done")
+		}
+
 	}
 
 	err = r.UpdateStatus()

diff --git a/pkg/system/system.go b/pkg/system/system.go
@@ -1116,7 +1116,19 @@ func Connect(isExternal bool) (*Client, error) {
 }
 
 // GetDesiredDBImage returns the desired DB image according to spec or env or default (in options)
-func GetDesiredDBImage(sys *nbv1.NooBaa) string {
+func GetDesiredDBImage(sys *nbv1.NooBaa, currentImage string) string {
+	// Postgres upgrade failure workaround
+	// if the current Postgres image is a postgresql-12 image, use NOOBAA_PSQL12_IMAGE. otherwise use GetdesiredDBImage
+	if IsPostgresql12Image(currentImage) {
+		psql12Image, ok := os.LookupEnv("NOOBAA_PSQL_12_IMAGE")
+		util.Logger().Warnf("The current Postgres image is a postgresql-12 image. using (%s)", psql12Image)
+		if !ok {
+			psql12Image = currentImage
+			util.Logger().Warnf("NOOBAA_PSQL_12_IMAGE is not set. using the current image %s", currentImage)
+		}
+		return psql12Image
+	}
+
 	if sys.Spec.DBImage != nil {
 		return *sys.Spec.DBImage
 	}
@@ -1128,6 +1140,11 @@ func GetDesiredDBImage(sys *nbv1.NooBaa) string {
 	return options.DBImage
 }
 
+// IsPostgresql12Image checks if the image is a postgresql-12 image
+func IsPostgresql12Image(image string) bool {
+	return strings.Contains(image, "postgresql-12")
+}
+
 // CheckSystem checks the state of the system and initializes its status fields
 func CheckSystem(sys *nbv1.NooBaa) bool {
 	log := util.Logger()