Skip to content

Commit 93bfed3

Browse files
Shinzusdudoladov
Shinzu
authored and
sdudoladov
committed
Add secret mount to operator (zalando#535)
* add secret mount to operator
1 parent 0ed92ed commit 93bfed3

File tree

10 files changed

+143
-6
lines changed

10 files changed

+143
-6
lines changed

‎charts/postgres-operator/values.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ configAwsOrGcp:
6969
# kube_iam_role: ""
7070
# log_s3_bucket: ""
7171
# wal_s3_bucket: ""
72+
# additional_secret_mount: "some-secret-name"
73+
# additional_secret_mount_path: "/some/dir"
7274

7375
configLogicalBackup:
7476
logical_backup_schedule: "30 00 * * *"

‎docs/administrator.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,3 +333,19 @@ The operator can manage k8s cron jobs to run logical backups of Postgres cluster
333333
4. You may use your own image by overwriting the relevant field in the operator configuration. Any such image must ensure the logical backup is able to finish [in presence of pod restarts](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures) and [simultaneous invocations](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-job-limitations) of the backup cron job.
334334

335335
5. For that feature to work, your RBAC policy must enable operations on the `cronjobs` resource from the `batch` API group for the operator service account. See [example RBAC](../manifests/operator-service-account-rbac.yaml)
336+
337+
## Access to cloud resources from clusters in non cloud environment
338+
339+
To access cloud resources like S3 from a cluster in a bare metal setup you can use
340+
`additional_secret_mount` and `additional_secret_mount_path` config parameters.
341+
With this you can provision cloud credentials to the containers in the pods of the StatefulSet.
342+
This works this way that it mounts a volume from the given secret in the pod and this can
343+
then accessed in the container over the configured mount path. Via [Custum Pod Environment Variables](#custom-pod-environment-variables)
344+
you can then point the different cloud sdk's (aws, google etc.) to this mounted secret.
345+
With this credentials the cloud sdk can then access cloud resources to upload logs etc.
346+
347+
A secret can be pre provisioned in different ways:
348+
349+
* Generic secret created via `kubectl create secret generic some-cloud-creds --from-file=some-cloud-credentials-file.json`
350+
351+
* Automaticly provisioned via a Controller like [kube-aws-iam-controller](https://github.com/mikkeloscar/kube-aws-iam-controller). This controller would then also rotate the credentials. Please visit the documention for more information.

‎docs/reference/operator_parameters.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,12 @@ yet officially supported.
407407
* **aws_region**
408408
AWS region used to store ESB volumes. The default is `eu-central-1`.
409409

410+
* **additional_secret_mount**
411+
Additional Secret (aws or gcp credentials) to mount in the pod. The default is empty.
412+
413+
* **additional_secret_mount_path**
414+
Path to mount the above Secret in the filesystem of the container(s). The default is empty.
415+
410416
## Debugging the operator
411417

412418
Options to aid debugging of the operator itself. Grouped under the `debug` key.

‎manifests/configmap.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ data:
3333
# https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees
3434
# inherited_labels: ""
3535
aws_region: eu-central-1
36+
# additional_secret_mount: "some-secret-name"
37+
# additional_secret_mount_path: "/some/dir"
3638
db_hosted_zone: db.example.com
3739
master_dns_name_format: '{cluster}.{team}.staging.{hostedzone}'
3840
replica_dns_name_format: '{cluster}-repl.{team}.staging.{hostedzone}'

‎manifests/postgresql-operator-default-configuration.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ configuration:
6767
# log_s3_bucket: ""
6868
# kube_iam_role: ""
6969
aws_region: eu-central-1
70+
# additional_secret_mount: "some-secret-name"
71+
# additional_secret_mount_path: "/some/dir"
7072
debug:
7173
debug_logging: true
7274
enable_database_access: true

‎pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,12 @@ type LoadBalancerConfiguration struct {
101101
// AWSGCPConfiguration defines the configuration for AWS
102102
// TODO complete Google Cloud Platform (GCP) configuration
103103
type AWSGCPConfiguration struct {
104-
WALES3Bucket string `json:"wal_s3_bucket,omitempty"`
105-
AWSRegion string `json:"aws_region,omitempty"`
106-
LogS3Bucket string `json:"log_s3_bucket,omitempty"`
107-
KubeIAMRole string `json:"kube_iam_role,omitempty"`
104+
WALES3Bucket string `json:"wal_s3_bucket,omitempty"`
105+
AWSRegion string `json:"aws_region,omitempty"`
106+
LogS3Bucket string `json:"log_s3_bucket,omitempty"`
107+
KubeIAMRole string `json:"kube_iam_role,omitempty"`
108+
AdditionalSecretMount string `json:"additional_secret_mount,omitempty"`
109+
AdditionalSecretMountPath string `json:"additional_secret_mount_path" default:"/meta/credentials"`
108110
}
109111

110112
// OperatorDebugConfiguration defines options for the debug mode

‎pkg/cluster/k8sres.go

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,7 @@ func generateContainer(
384384
VolumeMounts: volumeMounts,
385385
Env: envVars,
386386
SecurityContext: &v1.SecurityContext{
387-
Privileged: &privilegedMode,
387+
Privileged: &privilegedMode,
388388
ReadOnlyRootFilesystem: &falseBool,
389389
},
390390
}
@@ -445,6 +445,8 @@ func generatePodTemplate(
445445
shmVolume bool,
446446
podAntiAffinity bool,
447447
podAntiAffinityTopologyKey string,
448+
additionalSecretMount string,
449+
additionalSecretMountPath string,
448450
) (*v1.PodTemplateSpec, error) {
449451

450452
terminateGracePeriodSeconds := terminateGracePeriod
@@ -479,6 +481,10 @@ func generatePodTemplate(
479481
podSpec.PriorityClassName = priorityClassName
480482
}
481483

484+
if additionalSecretMount != "" {
485+
addSecretVolume(&podSpec, additionalSecretMount, additionalSecretMountPath)
486+
}
487+
482488
template := v1.PodTemplateSpec{
483489
ObjectMeta: metav1.ObjectMeta{
484490
Labels: labels,
@@ -864,7 +870,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*v1beta1.State
864870
effectivePodPriorityClassName,
865871
mountShmVolumeNeeded(c.OpConfig, spec),
866872
c.OpConfig.EnablePodAntiAffinity,
867-
c.OpConfig.PodAntiAffinityTopologyKey); err != nil {
873+
c.OpConfig.PodAntiAffinityTopologyKey,
874+
c.OpConfig.AdditionalSecretMount,
875+
c.OpConfig.AdditionalSecretMountPath); err != nil {
868876
return nil, fmt.Errorf("could not generate pod template: %v", err)
869877
}
870878

@@ -1013,6 +1021,28 @@ func addShmVolume(podSpec *v1.PodSpec) {
10131021
podSpec.Volumes = volumes
10141022
}
10151023

1024+
func addSecretVolume(podSpec *v1.PodSpec, additionalSecretMount string, additionalSecretMountPath string) {
1025+
volumes := append(podSpec.Volumes, v1.Volume{
1026+
Name: additionalSecretMount,
1027+
VolumeSource: v1.VolumeSource{
1028+
Secret: &v1.SecretVolumeSource{
1029+
SecretName: additionalSecretMount,
1030+
},
1031+
},
1032+
})
1033+
1034+
for i := range podSpec.Containers {
1035+
mounts := append(podSpec.Containers[i].VolumeMounts,
1036+
v1.VolumeMount{
1037+
Name: additionalSecretMount,
1038+
MountPath: additionalSecretMountPath,
1039+
})
1040+
podSpec.Containers[i].VolumeMounts = mounts
1041+
}
1042+
1043+
podSpec.Volumes = volumes
1044+
}
1045+
10161046
func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) {
10171047

10181048
var storageClassName *string
@@ -1395,6 +1425,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) {
13951425
"",
13961426
false,
13971427
false,
1428+
"",
1429+
"",
13981430
""); err != nil {
13991431
return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err)
14001432
}

‎pkg/cluster/k8sres_test.go

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,3 +389,74 @@ func TestCloneEnv(t *testing.T) {
389389
}
390390
}
391391
}
392+
393+
func TestSecretVolume(t *testing.T) {
394+
testName := "TestSecretVolume"
395+
tests := []struct {
396+
subTest string
397+
podSpec *v1.PodSpec
398+
secretPos int
399+
}{
400+
{
401+
subTest: "empty PodSpec",
402+
podSpec: &v1.PodSpec{
403+
Volumes: []v1.Volume{},
404+
Containers: []v1.Container{
405+
{
406+
VolumeMounts: []v1.VolumeMount{},
407+
},
408+
},
409+
},
410+
secretPos: 0,
411+
},
412+
{
413+
subTest: "non empty PodSpec",
414+
podSpec: &v1.PodSpec{
415+
Volumes: []v1.Volume{{}},
416+
Containers: []v1.Container{
417+
{
418+
VolumeMounts: []v1.VolumeMount{
419+
{
420+
Name: "data",
421+
ReadOnly: false,
422+
MountPath: "/data",
423+
},
424+
},
425+
},
426+
},
427+
},
428+
secretPos: 1,
429+
},
430+
}
431+
for _, tt := range tests {
432+
additionalSecretMount := "aws-iam-s3-role"
433+
additionalSecretMountPath := "/meta/credentials"
434+
435+
numMounts := len(tt.podSpec.Containers[0].VolumeMounts)
436+
437+
addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath)
438+
439+
volumeName := tt.podSpec.Volumes[tt.secretPos].Name
440+
441+
if volumeName != additionalSecretMount {
442+
t.Errorf("%s %s: Expected volume %s was not created, have %s instead",
443+
testName, tt.subTest, additionalSecretMount, volumeName)
444+
}
445+
446+
for i := range tt.podSpec.Containers {
447+
volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name
448+
449+
if volumeMountName != additionalSecretMount {
450+
t.Errorf("%s %s: Expected mount %s was not created, have %s instead",
451+
testName, tt.subTest, additionalSecretMount, volumeMountName)
452+
}
453+
}
454+
455+
numMountsCheck := len(tt.podSpec.Containers[0].VolumeMounts)
456+
457+
if numMountsCheck != numMounts+1 {
458+
t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v",
459+
numMountsCheck, numMounts+1)
460+
}
461+
}
462+
}

‎pkg/controller/operator_config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
8686
result.AWSRegion = fromCRD.AWSGCP.AWSRegion
8787
result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket
8888
result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole
89+
result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount
90+
result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath
8991

9092
result.DebugLogging = fromCRD.OperatorDebug.DebugLogging
9193
result.EnableDBAccess = fromCRD.OperatorDebug.EnableDBAccess

‎pkg/util/config/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ type Config struct {
9898
WALES3Bucket string `name:"wal_s3_bucket"`
9999
LogS3Bucket string `name:"log_s3_bucket"`
100100
KubeIAMRole string `name:"kube_iam_role"`
101+
AdditionalSecretMount string `name:"additional_secret_mount"`
102+
AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"`
101103
DebugLogging bool `name:"debug_logging" default:"true"`
102104
EnableDBAccess bool `name:"enable_database_access" default:"true"`
103105
EnableTeamsAPI bool `name:"enable_teams_api" default:"true"`

0 commit comments

Comments
 (0)