fix: update pvc failed as old pvc watchdog have not been deleting com…

…pleted (#1338)

pkg/server/biz/pvc/pvc.go

@@ -68,7 +68,8 @@ func CreatePVC(tenantName, storageClass, size string, namespace string, client *
 		PVC:       pvcName,
 	})
 	if err != nil {
-		log.Warningf("Launch PVC usage watcher for %s/%s error: %v", nsname, pvcName, err)
+		log.Errorf("Launch PVC usage watcher for %s/%s error: %v", nsname, pvcName, err)
+		return err
 	}
 
 	return nil
@@ -97,6 +98,34 @@ func DeletePVC(tenantName, namespace string, client *kubernetes.Clientset) error
 	return nil
 }
 
+// ConfirmPVCDeleted confirms whether the PVC and PVC watchdog have been deleted.
+// Only return nil represents the PVC and PVC watchdog have been deleted.
+func ConfirmPVCDeleted(tenantName, namespace string, client *kubernetes.Clientset) error {
+	pvcName := common.TenantPVC(tenantName)
+	nsname := common.TenantNamespace(tenantName)
+	if namespace != "" {
+		nsname = namespace
+	}
+
+	_, err := usage.GetPVCUsageWatcher(client, nsname)
+	if err == nil {
+		return fmt.Errorf("PVC watchdog for tenant %s has not been deleted", tenantName)
+	}
+	if !errors.IsNotFound(err) {
+		return err
+	}
+
+	_, err = client.CoreV1().PersistentVolumeClaims(nsname).Get(pvcName, meta_v1.GetOptions{})
+	if err == nil {
+		return fmt.Errorf("PVC for tenant %s has not been deleted", tenantName)
+	}
+	if !errors.IsNotFound(err) {
+		return err
+	}
+
+	return nil
+}
+
 // UpdatePVC delete the old pvc and recreate another one, so the data of the pvc will lost.
 func UpdatePVC(tenantName, storageClass, size string, namespace string, client *kubernetes.Clientset) error {
 	// Can not update pvc when there are workflows running.
@@ -130,12 +159,21 @@ func UpdatePVC(tenantName, storageClass, size string, namespace string, client *
 	}
 
 	backoff := wait.Backoff{
-		Steps:    18,
-		Duration: 10 * time.Second,
+		Steps:    12,
+		Duration: 5 * time.Second,
 		Factor:   1.0,
 		Jitter:   0.1,
 	}
 
+	// Wait for the PVC and PVC watchdog deleting completed
+	err = retry.OnError(backoff, func() error {
+		return ConfirmPVCDeleted(tenantName, namespace, client)
+	})
+	if err != nil {
+		return err
+	}
+
+	// Create a new PVC
 	err = retry.OnError(backoff, func() error {
 		return CreatePVC(tenantName, storageClass, size, namespace, client)
 	})

pkg/server/biz/usage/watcher.go

@@ -159,3 +159,8 @@ func DeletePVCUsageWatcher(client kubernetes.Interface, namespace string) error
 
 	return nil
 }
+
+// GetPVCUsageWatcher gets the pvc watch dog deployment.
+func GetPVCUsageWatcher(client kubernetes.Interface, namespace string) (*v1beta1.Deployment, error) {
+	return client.ExtensionsV1beta1().Deployments(namespace).Get(PVCWatcherName, metav1.GetOptions{})
+}