Skip to content

Commit c2dfc27

Browse files
agneumagneum
committed
Merge branch '134-reload-configuration' into 'master'
feat: reload configuration without downtime (#134): * reload Database Lab configuration without downtime * remove legacy options * highlight configuration errors * reload snapshot and retention schedulers Closes #134 See merge request postgres-ai/database-lab!200
2 parents 7252a3c + 0b4431b commit c2dfc27

File tree

17 files changed

+275
-136
lines changed

17 files changed

+275
-136
lines changed

‎cmd/database-lab/main.go

Lines changed: 58 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@
1111
package main
1212

1313
import (
14-
"bytes"
1514
"context"
15+
"os"
16+
"os/signal"
17+
"syscall"
1618

1719
"github.com/docker/docker/client"
18-
"github.com/jessevdk/go-flags"
1920
"github.com/pkg/errors"
2021
"github.com/rs/xid"
2122

@@ -31,57 +32,18 @@ import (
3132
"gitlab.com/postgres-ai/database-lab/version"
3233
)
3334

34-
var opts struct {
35-
VerificationToken string `short:"t" long:"token" description:"API verification token" env:"VERIFICATION_TOKEN"`
36-
37-
MountDir string `long:"mount-dir" description:"clones data mount directory" env:"MOUNT_DIR"`
38-
UnixSocketDir string `long:"sockets-dir" description:"unix sockets directory for secure connection to clones" env:"UNIX_SOCKET_DIR"`
39-
DockerImage string `long:"docker-image" description:"clones Docker image" env:"DOCKER_IMAGE"`
40-
41-
ShowHelp func() error `long:"help" description:"Show this help message"`
42-
}
43-
4435
func main() {
4536
log.Msg("Database Lab version: ", version.GetVersion())
4637

47-
// Load CLI options.
48-
if _, err := parseArgs(); err != nil {
49-
if flags.WroteHelp(err) {
50-
return
51-
}
38+
instanceID := xid.New().String()
5239

53-
log.Fatal("Args parse error:", err)
54-
}
40+
log.Msg("Database Lab Instance ID:", instanceID)
5541

56-
cfg, err := config.LoadConfig("config.yml")
42+
cfg, err := loadConfiguration(instanceID)
5743
if err != nil {
5844
log.Fatalf(errors.WithMessage(err, "failed to parse config"))
5945
}
6046

61-
log.DEBUG = cfg.Global.Debug
62-
log.Dbg("Config loaded", cfg)
63-
64-
// TODO(anatoly): Annotate envs in configs. Use different lib for flags/configs?
65-
if len(opts.MountDir) > 0 {
66-
cfg.Provision.Options.ClonesMountDir = opts.MountDir
67-
}
68-
69-
if len(opts.UnixSocketDir) > 0 {
70-
cfg.Provision.Options.UnixSocketDir = opts.UnixSocketDir
71-
}
72-
73-
if len(opts.DockerImage) > 0 {
74-
cfg.Provision.Options.DockerImage = opts.DockerImage
75-
}
76-
77-
if cfg.Provision.Options.ClonesMountDir != "" {
78-
cfg.Global.ClonesMountDir = cfg.Provision.Options.ClonesMountDir
79-
}
80-
81-
cfg.Global.InstanceID = xid.New().String()
82-
83-
log.Msg("Database Lab Instance ID", cfg.Global.InstanceID)
84-
8547
ctx, cancel := context.WithCancel(context.Background())
8648
defer cancel()
8749

@@ -121,40 +83,73 @@ func main() {
12183
log.Fatalf(errors.WithMessage(err, "failed to create a new platform service"))
12284
}
12385

124-
if len(opts.VerificationToken) > 0 {
125-
cfg.Server.VerificationToken = opts.VerificationToken
126-
}
127-
12886
observerCfg := &observer.Config{
12987
CloneDir: cfg.Provision.Options.ClonesMountDir,
13088
DataSubDir: cfg.Global.DataSubDir,
13189
SocketDir: cfg.Provision.Options.UnixSocketDir,
13290
}
13391

134-
// Start the Database Lab.
13592
server := srv.NewServer(&cfg.Server, observerCfg, cloningSvc, platformSvc, dockerCLI)
93+
94+
c := make(chan os.Signal, 1)
95+
signal.Notify(c, syscall.SIGHUP)
96+
97+
go func() {
98+
for range c {
99+
log.Msg("Reloading configuration")
100+
101+
if err := reloadConfig(instanceID, provisionSvc, retrievalSvc, cloningSvc, platformSvc, server); err != nil {
102+
log.Err("Failed to reload configuration", err)
103+
}
104+
105+
log.Msg("Configuration has been reloaded")
106+
}
107+
}()
108+
109+
// Start the Database Lab.
136110
if err = server.Run(); err != nil {
137111
log.Fatalf(err)
138112
}
139113
}
140114

141-
func parseArgs() ([]string, error) {
142-
var parser = flags.NewParser(&opts, flags.Default & ^flags.HelpFlag)
115+
func loadConfiguration(instanceID string) (*config.Config, error) {
116+
cfg, err := config.LoadConfig("config.yml")
117+
if err != nil {
118+
return nil, errors.Wrap(err, "failed to parse config")
119+
}
143120

144-
// jessevdk/go-flags lib doesn't allow to use short flag -h because
145-
// it's binded to usage help. We need to hack it a bit to use -h
146-
// for as a hostname option.
147-
// See https://github.com/jessevdk/go-flags/issues/240
148-
opts.ShowHelp = func() error {
149-
var b bytes.Buffer
121+
log.DEBUG = cfg.Global.Debug
122+
log.Dbg("Config loaded", cfg)
150123

151-
parser.WriteHelp(&b)
124+
if cfg.Provision.Options.ClonesMountDir != "" {
125+
cfg.Global.ClonesMountDir = cfg.Provision.Options.ClonesMountDir
126+
}
152127

153-
return &flags.Error{
154-
Type: flags.ErrHelp,
155-
Message: b.String(),
156-
}
128+
cfg.Global.InstanceID = instanceID
129+
130+
return cfg, nil
131+
}
132+
133+
func reloadConfig(instanceID string, provisionSvc provision.Provision, retrievalSvc *retrieval.Retrieval, cloningSvc cloning.Cloning,
134+
platformSvc *platform.Service, server *srv.Server) error {
135+
cfg, err := loadConfiguration(instanceID)
136+
if err != nil {
137+
return err
138+
}
139+
140+
if err := provision.IsValidConfig(cfg.Provision); err != nil {
141+
return err
157142
}
158143

159-
return parser.Parse()
144+
if err := retrieval.IsValidConfig(cfg); err != nil {
145+
return err
146+
}
147+
148+
provisionSvc.Reload(cfg.Provision)
149+
retrievalSvc.Reload(cfg)
150+
cloningSvc.Reload(cfg.Cloning)
151+
platformSvc.Reload(cfg.Platform)
152+
server.Reload(cfg.Server)
153+
154+
return nil
160155
}

‎pkg/config/config.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
package config
77

88
import (
9-
"fmt"
109
"io/ioutil"
1110
"os/user"
1211
"path"
@@ -87,7 +86,7 @@ func LoadConfig(name string) (*Config, error) {
8786

8887
cfg := &Config{}
8988
if err := yaml.Unmarshal(b, cfg); err != nil {
90-
return nil, fmt.Errorf("error parsing %s config", name)
89+
return nil, errors.WithMessagef(err, "error parsing %s config", name)
9190
}
9291

9392
if err := cfg.setUpProvisionParams(); err != nil {

‎pkg/retrieval/components/components.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ type JobRunner interface {
2222
// Name returns a job name.
2323
Name() string
2424

25+
// Reload reloads job configuration.
26+
Reload(cfg map[string]interface{}) error
27+
2528
// Run starts a job.
2629
Run(ctx context.Context) error
2730
}

‎pkg/retrieval/engine/postgres/logical/dump.go

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -120,16 +120,10 @@ func NewDumpJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Gl
120120
},
121121
}
122122

123-
if err := options.Unmarshal(cfg.Options, &dumpJob.DumpOptions); err != nil {
124-
return nil, errors.Wrap(err, "failed to unmarshal configuration options")
123+
if err := dumpJob.Reload(cfg.Options); err != nil {
124+
return nil, errors.Wrap(err, "failed to load job config")
125125
}
126126

127-
if err := dumpJob.validate(); err != nil {
128-
return nil, errors.Wrap(err, "invalid logical dump job")
129-
}
130-
131-
dumpJob.setDefaults()
132-
133127
if err := dumpJob.setupDumper(); err != nil {
134128
return nil, errors.Wrap(err, "failed to set up a dump helper")
135129
}
@@ -195,6 +189,21 @@ func (d *DumpJob) Name() string {
195189
return d.name
196190
}
197191

192+
// Reload reloads job configuration.
193+
func (d *DumpJob) Reload(cfg map[string]interface{}) (err error) {
194+
if err := options.Unmarshal(cfg, &d.DumpOptions); err != nil {
195+
return errors.Wrap(err, "failed to unmarshal configuration options")
196+
}
197+
198+
if err := d.validate(); err != nil {
199+
return errors.Wrap(err, "invalid logical dump job")
200+
}
201+
202+
d.setDefaults()
203+
204+
return nil
205+
}
206+
198207
// Run starts the job.
199208
func (d *DumpJob) Run(ctx context.Context) (err error) {
200209
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", d.Name(), d.DumpOptions))

‎pkg/retrieval/engine/postgres/logical/restore.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func NewJob(cfg config.JobConfig, docker *client.Client, globalCfg *dblabCfg.Glo
7373
dbMark: &dbmarker.Config{DataType: dbmarker.LogicalDataType},
7474
}
7575

76-
if err := options.Unmarshal(cfg.Options, &restoreJob.RestoreOptions); err != nil {
76+
if err := restoreJob.Reload(cfg.Options); err != nil {
7777
return nil, errors.Wrap(err, "failed to unmarshal configuration options")
7878
}
7979

@@ -98,6 +98,17 @@ func (r *RestoreJob) Name() string {
9898
return r.name
9999
}
100100

101+
// Reload reloads job configuration.
102+
func (r *RestoreJob) Reload(cfg map[string]interface{}) (err error) {
103+
if err := options.Unmarshal(cfg, &r.RestoreOptions); err != nil {
104+
return errors.Wrap(err, "failed to unmarshal configuration options")
105+
}
106+
107+
r.setDefaults()
108+
109+
return nil
110+
}
111+
101112
// Run starts the job.
102113
func (r *RestoreJob) Run(ctx context.Context) (err error) {
103114
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", r.Name(), r.RestoreOptions))

‎pkg/retrieval/engine/postgres/physical/physical.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ func NewJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global
9595
dbMarker: marker,
9696
}
9797

98-
if err := options.Unmarshal(cfg.Options, &physicalJob.CopyOptions); err != nil {
98+
if err := physicalJob.Reload(cfg.Options); err != nil {
9999
return nil, errors.Wrap(err, "failed to unmarshal configuration options")
100100
}
101101

@@ -131,6 +131,11 @@ func (r *RestoreJob) Name() string {
131131
return r.name
132132
}
133133

134+
// Reload reloads job configuration.
135+
func (r *RestoreJob) Reload(cfg map[string]interface{}) (err error) {
136+
return options.Unmarshal(cfg, &r.CopyOptions)
137+
}
138+
134139
// Run starts the job.
135140
func (r *RestoreJob) Run(ctx context.Context) (err error) {
136141
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", r.Name(), r.CopyOptions))

‎pkg/retrieval/engine/postgres/snapshot/logical.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func NewLogicalInitialJob(cfg config.JobConfig, cloneManager thinclones.Manager,
5151
dbMarker: marker,
5252
}
5353

54-
if err := options.Unmarshal(cfg.Options, &li.options); err != nil {
54+
if err := li.Reload(cfg.Options); err != nil {
5555
return nil, errors.Wrap(err, "failed to unmarshal configuration options")
5656
}
5757

@@ -63,6 +63,11 @@ func (s *LogicalInitial) Name() string {
6363
return s.name
6464
}
6565

66+
// Reload reloads job configuration.
67+
func (s *LogicalInitial) Reload(cfg map[string]interface{}) (err error) {
68+
return options.Unmarshal(cfg, &s.options)
69+
}
70+
6671
// Run starts the job.
6772
func (s *LogicalInitial) Run(_ context.Context) error {
6873
if s.options.PreprocessingScript != "" {

0 commit comments

Comments
 (0)