Skip to content

Commit 88735a7

Browse files
yanchenko-igorFxKu
yanchenko-igor
and
FxKu
authored
Resize volume by changing pvc size if enabled in config. (zalando#958)
* Try to resize pvc if resizing pv has failed * added config option to switch between storage resize strategies * changes according to requests * Update pkg/controller/operator_config.go Co-authored-by: Felix Kunde <felix-kunde@gmx.de> * enable_storage_resize documented added examples to the default configuration and helm value files * enable_storage_resize renamed to volume_resize_mode, off by default * volume_resize_mode renamed to storage_resize_mode * Update pkg/apis/acid.zalan.do/v1/crds.go * pkg/cluster/volumes.go updated * Update docs/reference/operator_parameters.md * Update manifests/postgresql-operator-default-configuration.yaml * Update pkg/controller/operator_config.go * Update pkg/util/config/config.go * Update charts/postgres-operator/values-crd.yaml * Update charts/postgres-operator/values.yaml * Update docs/reference/operator_parameters.md * added logging if no changes required Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
1 parent 6869c2c commit 88735a7

File tree

12 files changed

+125
-11
lines changed

12 files changed

+125
-11
lines changed

‎charts/postgres-operator/values-crd.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ configKubernetes:
124124

125125
# whether the Spilo container should run in privileged mode
126126
spilo_privileged: false
127+
# storage resize strategy, available options are: ebs, pvc, off
128+
storage_resize_mode: ebs
127129
# operator watches for postgres objects in the given namespace
128130
watched_namespace: "*" # listen to all namespaces
129131

‎charts/postgres-operator/values.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ configKubernetes:
115115

116116
# whether the Spilo container should run in privileged mode
117117
spilo_privileged: "false"
118+
# storage resize strategy, available options are: ebs, pvc, off
119+
storage_resize_mode: ebs
118120
# operator watches for postgres objects in the given namespace
119121
watched_namespace: "*" # listen to all namespaces
120122

‎docs/reference/operator_parameters.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,12 @@ configuration they are grouped under the `kubernetes` key.
333333
of stateful sets of PG clusters. The default is `ordered_ready`, the second
334334
possible value is `parallel`.
335335

336+
* **storage_resize_mode**
337+
defines how operator handels the difference between requested volume size and
338+
actual size. Available options are: ebs - tries to resize EBS volume, pvc -
339+
changes PVC definition, off - disables resize of the volumes. Default is "ebs".
340+
When using OpenShift please use one of the other available options.
341+
336342
## Kubernetes resource requests
337343

338344
This group allows you to configure resource requests for the Postgres pods.

‎manifests/configmap.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ data:
9797
# set_memory_request_to_limit: "false"
9898
# spilo_fsgroup: 103
9999
spilo_privileged: "false"
100+
# storage_resize_mode: "off"
100101
super_username: postgres
101102
# team_admin_role: "admin"
102103
# team_api_role_configuration: "log_statement:all"

‎manifests/operatorconfiguration.crd.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,12 @@ spec:
168168
type: integer
169169
spilo_privileged:
170170
type: boolean
171+
storage_resize_mode:
172+
type: string
173+
enum:
174+
- "ebs"
175+
- "pvc"
176+
- "off"
171177
toleration:
172178
type: object
173179
additionalProperties:

‎manifests/postgresql-operator-default-configuration.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ configuration:
5959
secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}"
6060
# spilo_fsgroup: 103
6161
spilo_privileged: false
62+
storage_resize_mode: ebs
6263
# toleration: {}
6364
# watched_namespace: ""
6465
postgres_pod_resources:

‎pkg/apis/acid.zalan.do/v1/crds.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -980,6 +980,20 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation
980980
"spilo_privileged": {
981981
Type: "boolean",
982982
},
983+
"storage_resize_mode": {
984+
Type: "string",
985+
Enum: []apiextv1beta1.JSON{
986+
{
987+
Raw: []byte(`"ebs"`),
988+
},
989+
{
990+
Raw: []byte(`"pvc"`),
991+
},
992+
{
993+
Raw: []byte(`"off"`),
994+
},
995+
},
996+
},
983997
"toleration": {
984998
Type: "object",
985999
AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{

‎pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ type KubernetesMetaConfiguration struct {
5353
WatchedNamespace string `json:"watched_namespace,omitempty"`
5454
PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"`
5555
EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"`
56+
StorageResizeMode string `json:"storage_resize_mode,omitempty"`
5657
EnableInitContainers *bool `json:"enable_init_containers,omitempty"`
5758
EnableSidecars *bool `json:"enable_sidecars,omitempty"`
5859
SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"`

‎pkg/cluster/sync.go

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,26 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
5757
return err
5858
}
5959

60-
// potentially enlarge volumes before changing the statefulset. By doing that
61-
// in this order we make sure the operator is not stuck waiting for a pod that
62-
// cannot start because it ran out of disk space.
63-
// TODO: handle the case of the cluster that is downsized and enlarged again
64-
// (there will be a volume from the old pod for which we can't act before the
65-
// the statefulset modification is concluded)
66-
c.logger.Debugf("syncing persistent volumes")
67-
if err = c.syncVolumes(); err != nil {
68-
err = fmt.Errorf("could not sync persistent volumes: %v", err)
69-
return err
60+
if c.OpConfig.StorageResizeMode == "pvc" {
61+
c.logger.Debugf("syncing persistent volume claims")
62+
if err = c.syncVolumeClaims(); err != nil {
63+
err = fmt.Errorf("could not sync persistent volume claims: %v", err)
64+
return err
65+
}
66+
} else if c.OpConfig.StorageResizeMode == "ebs" {
67+
// potentially enlarge volumes before changing the statefulset. By doing that
68+
// in this order we make sure the operator is not stuck waiting for a pod that
69+
// cannot start because it ran out of disk space.
70+
// TODO: handle the case of the cluster that is downsized and enlarged again
71+
// (there will be a volume from the old pod for which we can't act before the
72+
// the statefulset modification is concluded)
73+
c.logger.Debugf("syncing persistent volumes")
74+
if err = c.syncVolumes(); err != nil {
75+
err = fmt.Errorf("could not sync persistent volumes: %v", err)
76+
return err
77+
}
78+
} else {
79+
c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.")
7080
}
7181

7282
if err = c.enforceMinResourceLimits(&c.Spec); err != nil {
@@ -571,6 +581,27 @@ func (c *Cluster) syncRoles() (err error) {
571581
return nil
572582
}
573583

584+
// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset.
585+
func (c *Cluster) syncVolumeClaims() error {
586+
c.setProcessName("syncing volume claims")
587+
588+
act, err := c.volumeClaimsNeedResizing(c.Spec.Volume)
589+
if err != nil {
590+
return fmt.Errorf("could not compare size of the volume claims: %v", err)
591+
}
592+
if !act {
593+
c.logger.Infof("volume claims don't require changes")
594+
return nil
595+
}
596+
if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil {
597+
return fmt.Errorf("could not sync volume claims: %v", err)
598+
}
599+
600+
c.logger.Infof("volume claims have been synced successfully")
601+
602+
return nil
603+
}
604+
574605
// syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset.
575606
func (c *Cluster) syncVolumes() error {
576607
c.setProcessName("syncing volumes")

‎pkg/cluster/volumes.go

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,35 @@ func (c *Cluster) deletePersistentVolumeClaims() error {
5252
return nil
5353
}
5454

55+
func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error {
56+
c.logger.Debugln("resizing PVCs")
57+
pvcs, err := c.listPersistentVolumeClaims()
58+
if err != nil {
59+
return err
60+
}
61+
newQuantity, err := resource.ParseQuantity(newVolume.Size)
62+
if err != nil {
63+
return fmt.Errorf("could not parse volume size: %v", err)
64+
}
65+
_, newSize, err := c.listVolumesWithManifestSize(newVolume)
66+
for _, pvc := range pvcs {
67+
volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
68+
if volumeSize >= newSize {
69+
if volumeSize > newSize {
70+
c.logger.Warningf("cannot shrink persistent volume")
71+
}
72+
continue
73+
}
74+
pvc.Spec.Resources.Requests[v1.ResourceStorage] = newQuantity
75+
c.logger.Debugf("updating persistent volume claim definition for volume %q", pvc.Name)
76+
if _, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}); err != nil {
77+
return fmt.Errorf("could not update persistent volume claim: %q", err)
78+
}
79+
c.logger.Debugf("successfully updated persistent volume claim %q", pvc.Name)
80+
}
81+
return nil
82+
}
83+
5584
func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
5685
result := make([]*v1.PersistentVolume, 0)
5786

@@ -150,7 +179,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
150179
c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
151180
}
152181
if !compatible {
153-
c.logger.Warningf("volume %q is incompatible with all available resizing providers", pv.Name)
182+
c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name)
154183
totalIncompatible++
155184
}
156185
}
@@ -160,6 +189,25 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu
160189
return nil
161190
}
162191

192+
func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error) {
193+
newSize, err := resource.ParseQuantity(newVolume.Size)
194+
manifestSize := quantityToGigabyte(newSize)
195+
if err != nil {
196+
return false, fmt.Errorf("could not parse volume size from the manifest: %v", err)
197+
}
198+
pvcs, err := c.listPersistentVolumeClaims()
199+
if err != nil {
200+
return false, fmt.Errorf("could not receive persistent volume claims: %v", err)
201+
}
202+
for _, pvc := range pvcs {
203+
currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage])
204+
if currentSize != manifestSize {
205+
return true, nil
206+
}
207+
}
208+
return false, nil
209+
}
210+
163211
func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) {
164212
vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume)
165213
if err != nil {

‎pkg/controller/operator_config.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
6565
result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace
6666
result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat
6767
result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True())
68+
result.StorageResizeMode = util.Coalesce(fromCRD.Kubernetes.StorageResizeMode, "ebs")
6869
result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True())
6970
result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True())
7071
result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate

‎pkg/util/config/config.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ type Config struct {
142142
CustomPodAnnotations map[string]string `name:"custom_pod_annotations"`
143143
EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"`
144144
PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"`
145+
StorageResizeMode string `name:"storage_resize_mode" default:"ebs"`
145146
// deprecated and kept for backward compatibility
146147
EnableLoadBalancer *bool `name:"enable_load_balancer"`
147148
MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`

0 commit comments

Comments
 (0)