Skip to content

Commit 276c3bf

Browse files
agneumagneum
committed
Merge branch '152-environment' into 'master'
fix: prepare environment for Database Lab retrieval (#152) * create if not exists PostgreSQL directory(`global.dataDir`) and set permissions * pass environment variables from OS with Database Lab instance to physical and logical job containers * describe `customTool` for physical restore commands * bump Go version * bump golangci-lint version See merge request postgres-ai/database-lab!156
2 parents c0eb017 + e8af85b commit 276c3bf

File tree

11 files changed

+62
-31
lines changed

11 files changed

+62
-31
lines changed

‎.gitlab-ci.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
image: golang:1.13
1+
image: golang:1.14
22

33
stages:
44
- test
@@ -38,7 +38,7 @@ lint:
3838
when: manual
3939

4040
build-binary-alpine:
41-
image: golang:1.13-alpine
41+
image: golang:1.14-alpine
4242
stage: build-binary
4343
only:
4444
refs:

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ all: clean build
2929

3030
# Install the linter to $GOPATH/bin which is expected to be in $PATH
3131
install-lint:
32-
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.22.2
32+
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0
3333

3434
run-lint:
3535
golangci-lint run

‎configs/config.example.physical_generic.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,13 +147,17 @@ retrieval:
147147
PGHOST: "source.hostname"
148148

149149
customTool:
150-
# To use pg_basebackup, specify environment variables in "env".
150+
# To use pg_basebackup, specify environment variables in "envs".
151151
# Do not edit PostgreSQL data directory (-D).
152152
# Note that command chains are not supported here; if you need to use a more
153153
# complicated snippet, create a shell script, use --mount (-v) option
154154
# when starting a container with Database Lab and use path to it here.
155+
# Write your data to dataDir defined in "global.config"
155156
command: "pg_basebackup -X stream -D /var/lib/dblab/data"
156157

158+
# PostgreSQL "restore_command" configuration option.
159+
restore_command: ""
160+
157161
physical-snapshot:
158162
options:
159163
# Promote PGDATA after data fetching.

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module gitlab.com/postgres-ai/database-lab
22

3-
go 1.13
3+
go 1.14
44

55
require (
66
github.com/AlekSi/pointer v1.1.0

‎pkg/config/config.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"io/ioutil"
1111
"os/user"
1212

13-
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
13+
retConfig "gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
1414
"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
1515
"gitlab.com/postgres-ai/database-lab/pkg/services/platform"
1616
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
@@ -28,7 +28,7 @@ type Config struct {
2828
Cloning cloning.Config `yaml:"cloning"`
2929
Platform platform.Config `yaml:"platform"`
3030
Global Global `yaml:"global"`
31-
Retrieval config.Config `yaml:"retrieval"`
31+
Retrieval retConfig.Config `yaml:"retrieval"`
3232
}
3333

3434
// Global contains global Database Lab configurations.

‎pkg/retrieval/engine/postgres/logical/dump.go

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -400,16 +400,10 @@ func (d *DumpJob) getContainerNetworkMode() container.NetworkMode {
400400
}
401401

402402
func (d *DumpJob) getExecEnvironmentVariables() []string {
403-
execEnvs := d.dumper.GetCmdEnvVariables()
403+
execEnvs := append(os.Environ(), d.dumper.GetCmdEnvVariables()...)
404404

405-
pgPassword := d.config.db.Password
406-
407-
if pgPassword == "" && os.Getenv("PGPASSWORD") != "" {
408-
pgPassword = os.Getenv("PGPASSWORD")
409-
}
410-
411-
if pgPassword != "" {
412-
execEnvs = append(execEnvs, "PGPASSWORD="+pgPassword)
405+
if d.config.db.Password != "" && os.Getenv("PGPASSWORD") == "" {
406+
execEnvs = append(execEnvs, "PGPASSWORD="+d.config.db.Password)
413407
}
414408

415409
return execEnvs

‎pkg/retrieval/engine/postgres/logical/restore.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ package logical
88
import (
99
"context"
1010
"fmt"
11+
"os"
1112
"strconv"
1213
"strings"
1314

@@ -203,10 +204,10 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
203204
func (r *RestoreJob) buildContainerConfig(password string) *container.Config {
204205
return &container.Config{
205206
Labels: map[string]string{"label": tools.DBLabControlLabel},
206-
Env: []string{
207+
Env: append(os.Environ(), []string{
207208
"PGDATA=" + r.globalCfg.DataDir,
208209
"POSTGRES_PASSWORD=" + password,
209-
},
210+
}...),
210211
Image: r.RestoreOptions.DockerImage,
211212
Healthcheck: health.GetConfig(),
212213
}

‎pkg/retrieval/engine/postgres/physical/custom.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ type custom struct {
2020
}
2121

2222
type customOptions struct {
23-
RestoreCommand string `yaml:"command"`
24-
RefreshCommand string `yaml:"refresh_command"`
23+
Command string `yaml:"command"`
24+
RestoreCommand string `yaml:"restore_command"`
2525
}
2626

2727
func newCustomTool(options customOptions) *custom {
@@ -42,15 +42,15 @@ func (c *custom) GetMounts() []mount.Mount {
4242

4343
// GetRestoreCommand returns a custom command to restore data.
4444
func (c *custom) GetRestoreCommand() string {
45-
return c.options.RestoreCommand
45+
return c.options.Command
4646
}
4747

4848
// GetRecoveryConfig returns a recovery config to restore data.
4949
func (c *custom) GetRecoveryConfig() []byte {
5050
buffer := bytes.Buffer{}
5151

5252
buffer.WriteString("standby_mode = 'on'\n")
53-
buffer.WriteString(fmt.Sprintf("restore_command = '%s'\n", c.options.RefreshCommand))
53+
buffer.WriteString(fmt.Sprintf("restore_command = '%s'\n", c.options.RestoreCommand))
5454

5555
return buffer.Bytes()
5656
}

‎pkg/retrieval/engine/postgres/physical/physical.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
171171
return errors.Wrapf(err, "failed to start container: %v", contID)
172172
}
173173

174-
log.Msg("Running restore command")
174+
log.Msg("Running restore command: ", r.restorer.GetRestoreCommand())
175175

176176
if err := tools.ExecCommand(ctx, r.dockerClient, contID, types.ExecConfig{
177177
Cmd: []string{"bash", "-c", r.restorer.GetRestoreCommand()},
@@ -352,11 +352,16 @@ func (r *RestoreJob) runSyncInstance(ctx context.Context) error {
352352
}
353353

354354
func (r *RestoreJob) getEnvironmentVariables(password string) []string {
355-
envVariables := append([]string{
355+
// Pass Database Lab environment variables.
356+
envVariables := append(os.Environ(), []string{
356357
"POSTGRES_PASSWORD=" + password,
357358
"PGDATA=" + r.globalCfg.DataDir,
358-
}, r.restorer.GetEnvVariables()...)
359+
}...)
359360

361+
// Add restore-specific environment variables.
362+
envVariables = append(envVariables, r.restorer.GetEnvVariables()...)
363+
364+
// Add user-defined environment variables.
360365
for env, value := range r.Envs {
361366
envVariables = append(envVariables, fmt.Sprintf("%s=%s", env, value))
362367
}

‎pkg/retrieval/retrieval.go

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ package retrieval
77

88
import (
99
"context"
10+
"os"
11+
"path/filepath"
1012

1113
"github.com/docker/docker/client"
1214
"github.com/pkg/errors"
@@ -22,7 +24,8 @@ import (
2224

2325
// Retrieval describes a data retrieval.
2426
type Retrieval struct {
25-
config *config.Config
27+
cfg *config.Config
28+
globalCfg *dblabCfg.Global
2629
retrievalRunner components.JobBuilder
2730
cloneManager thinclones.Manager
2831
jobs []components.JobRunner
@@ -36,14 +39,19 @@ func New(cfg *dblabCfg.Config, dockerCLI *client.Client, cloneManager thinclones
3639
}
3740

3841
return &Retrieval{
39-
config: &cfg.Retrieval,
42+
cfg: &cfg.Retrieval,
43+
globalCfg: &cfg.Global,
4044
retrievalRunner: retrievalRunner,
4145
cloneManager: cloneManager,
4246
}, nil
4347
}
4448

4549
// Run start retrieving process.
4650
func (r *Retrieval) Run(ctx context.Context) error {
51+
if len(r.cfg.Jobs) == 0 {
52+
return nil
53+
}
54+
4755
if err := r.parseJobs(); err != nil {
4856
return errors.Wrap(err, "failed to parse retrieval jobs")
4957
}
@@ -52,6 +60,10 @@ func (r *Retrieval) Run(ctx context.Context) error {
5260
return errors.Wrap(err, "invalid data retrieval configuration")
5361
}
5462

63+
if err := r.prepareEnvironment(); err != nil {
64+
return errors.Wrap(err, "failed to prepare retrieval environment")
65+
}
66+
5567
for _, j := range r.jobs {
5668
if err := j.Run(ctx); err != nil {
5769
return err
@@ -63,8 +75,8 @@ func (r *Retrieval) Run(ctx context.Context) error {
6375

6476
// parseJobs processes configuration to define data retrieval jobs.
6577
func (r *Retrieval) parseJobs() error {
66-
for _, jobName := range r.config.Jobs {
67-
jobConfig, ok := r.config.JobsSpec[jobName]
78+
for _, jobName := range r.cfg.Jobs {
79+
jobConfig, ok := r.cfg.JobsSpec[jobName]
6880
if !ok {
6981
return errors.Errorf("Job %q not found", jobName)
7082
}
@@ -103,3 +115,18 @@ func (r *Retrieval) validate() error {
103115

104116
return nil
105117
}
118+
119+
func (r *Retrieval) prepareEnvironment() error {
120+
if err := os.MkdirAll(r.globalCfg.DataDir, 0700); err != nil {
121+
return err
122+
}
123+
124+
return filepath.Walk(r.globalCfg.DataDir, func(name string, info os.FileInfo, err error) error {
125+
if err == nil {
126+
// PGDATA dir permissions must be 0700 to avoid errors.
127+
err = os.Chmod(name, 0700)
128+
}
129+
130+
return err
131+
})
132+
}

‎pkg/srv/server.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,12 @@ func (s *Server) Run() error {
9393

9494
// Show Swagger UI on index page.
9595
if err := attachAPI(r); err != nil {
96-
log.Err(fmt.Sprintf("Cannot load API description."))
96+
log.Err("Cannot load API description.")
9797
}
9898

9999
// Show Swagger UI on index page.
100100
if err := attachSwaggerUI(r); err != nil {
101-
log.Err(fmt.Sprintf("Cannot start Swagger UI."))
101+
log.Err("Cannot start Swagger UI.")
102102
}
103103

104104
// Show not found error for all other possible routes.

0 commit comments

Comments
 (0)