Sync persistent volumes before statefulsets. (zalando#293)

alexeyklyukin · web-flow · commit 88d6a7be3ff8 · 2018-05-18T12:01:43.000+02:00
Avoid the condition of waiting for the pod that cannot start
PostgreSQL because it ran out of disk space.
diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
@@ -50,6 +50,18 @@ func (c *Cluster) Sync(newSpec *spec.Postgresql) (err error) {
 		return
 	}
 
+	// potentially enlarge volumes before changing the statefulset. By doing that
+	// in this order we make sure the operator is not stuck waiting for a pod that
+	// cannot start because it ran out of disk space.
+	// TODO: handle the case of the cluster that is downsized and enlarged again
+	// (there will be a volume from the old pod for which we can't act before the
+	//  the statefulset modification is concluded)
+	c.logger.Debugf("syncing persistent volumes")
+	if err = c.syncVolumes(); err != nil {
+		err = fmt.Errorf("could not sync persistent volumes: %v", err)
+		return
+	}
+
 	c.logger.Debugf("syncing statefulsets")
 	if err = c.syncStatefulSet(); err != nil {
 		if !k8sutil.ResourceAlreadyExists(err) {
@@ -72,12 +84,6 @@ func (c *Cluster) Sync(newSpec *spec.Postgresql) (err error) {
 		}
 	}
 
-	c.logger.Debugf("syncing persistent volumes")
-	if err = c.syncVolumes(); err != nil {
-		err = fmt.Errorf("could not sync persistent volumes: %v", err)
-		return
-	}
-
 	c.logger.Debug("syncing pod disruption budgets")
 	if err = c.syncPodDisruptionBudget(false); err != nil {
 		err = fmt.Errorf("could not sync pod disruption budget: %v", err)
