Skip to content

Commit a400fb7

Browse files
agneumagneum
committed
feat: automated data retrieval - physical and logical restoring (#93)
1 parent 3fa22fc commit a400fb7

File tree

30 files changed

+1445
-18
lines changed

30 files changed

+1445
-18
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@
88
/deploy/
99

1010
/configs/config.yml
11+
/configs/retrieval.yml

‎cmd/database-lab/main.go

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ import (
1414
"bytes"
1515
"context"
1616

17+
"github.com/docker/docker/client"
1718
"github.com/jessevdk/go-flags"
1819
"github.com/pkg/errors"
1920

2021
"gitlab.com/postgres-ai/database-lab/pkg/config"
2122
"gitlab.com/postgres-ai/database-lab/pkg/log"
23+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval"
2224
"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
2325
"gitlab.com/postgres-ai/database-lab/pkg/services/platform"
2426
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
@@ -36,8 +38,6 @@ var opts struct {
3638
}
3739

3840
func main() {
39-
ctx := context.Background()
40-
4141
// Load CLI options.
4242
if _, err := parseArgs(); err != nil {
4343
if flags.WroteHelp(err) {
@@ -68,24 +68,41 @@ func main() {
6868
cfg.Provision.ModeLocal.DockerImage = opts.DockerImage
6969
}
7070

71-
provisionSvc, err := provision.NewProvision(ctx, cfg.Provision)
71+
dockerCLI, err := client.NewClientWithOpts(client.FromEnv)
72+
if err != nil {
73+
log.Fatal("Failed to create a Docker client:", err)
74+
}
75+
76+
ctx, cancel := context.WithCancel(context.Background())
77+
defer cancel()
78+
79+
// Create a new retrieval service to prepare a data directory and start snapshotting.
80+
retrievalSvc, err := retrieval.New(cfg, dockerCLI)
81+
if err != nil {
82+
log.Fatal("Failed to build a retrieval service:", err)
83+
}
84+
85+
if err := retrievalSvc.Run(ctx); err != nil {
86+
log.Fatal("Failed to run the data retrieval service:", err)
87+
}
88+
89+
// Create a cloning service to provision new clones.
90+
provisionSvc, err := provision.New(ctx, cfg.Provision)
7291
if err != nil {
7392
log.Fatalf(errors.WithMessage(err, `error in "provision" config`))
7493
}
7594

76-
cloningSvc, err := cloning.NewCloning(&cfg.Cloning, provisionSvc)
95+
cloningSvc, err := cloning.New(&cfg.Cloning, provisionSvc)
7796
if err != nil {
7897
log.Fatalf(errors.WithMessage(err, "failed to init a new cloning service"))
7998
}
8099

81-
ctx, cancel := context.WithCancel(context.Background())
82-
defer cancel()
83-
84100
if err = cloningSvc.Run(ctx); err != nil {
85101
log.Fatalf(err)
86102
}
87103

88-
platformSvc := platform.NewService(cfg.Platform)
104+
// Create a platform service to verify Platform tokens.
105+
platformSvc := platform.New(cfg.Platform)
89106
if err := platformSvc.Init(ctx); err != nil {
90107
log.Fatalf(errors.WithMessage(err, "failed to create a new platform service"))
91108
}
@@ -94,6 +111,7 @@ func main() {
94111
cfg.Server.VerificationToken = opts.VerificationToken
95112
}
96113

114+
// Start the Database Lab.
97115
server := srv.NewServer(&cfg.Server, cloningSvc, platformSvc)
98116
if err = server.Run(); err != nil {
99117
log.Fatalf(err)

‎configs/config.sample.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ server:
1111
# HTTP server port. By default: 2345.
1212
port: 2345
1313

14+
global:
15+
engine: postgres
16+
dataDir: /var/lib/dblab
17+
dockerImage: "postgres:12-alpine"
18+
1419
# Postgres.ai Platform integration.
1520
platform:
1621
# API URL.
@@ -82,4 +87,30 @@ cloning:
8287
# 0 - disable automatic deletion.
8388
maxIdleMinutes: 120
8489

90+
retrieval:
91+
stages:
92+
- initialize
93+
94+
spec:
95+
initialize:
96+
jobs:
97+
# - name: logical-restore
98+
# options:
99+
# dumpFile: /tmp/db.dump
100+
# forceInit: false
101+
# dbName: test
102+
# partial:
103+
# tables:
104+
# - test
105+
- name: physical-restore
106+
options:
107+
tool: walg
108+
dockerImage: "postgresai/sync-instance:12"
109+
envs:
110+
WALG_GS_PREFIX: "gs://{BUCKET}/{SCOPE}"
111+
walg:
112+
storage: gcs
113+
backupName: LATEST
114+
credentialsFile: /tmp/sa.json # optional
115+
85116
debug: true

‎go.mod

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,20 @@ go 1.13
44

55
require (
66
github.com/AlekSi/pointer v1.1.0
7+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
78
github.com/Microsoft/go-winio v0.4.14 // indirect
9+
github.com/containerd/containerd v1.3.4 // indirect
810
github.com/docker/distribution v2.7.1+incompatible // indirect
911
github.com/docker/docker v1.13.1
1012
github.com/docker/go-connections v0.4.0 // indirect
1113
github.com/docker/go-units v0.4.0 // indirect
14+
github.com/gogo/protobuf v1.3.1 // indirect
1215
github.com/gorilla/mux v1.7.3
1316
github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4
1417
github.com/lib/pq v1.3.0
18+
github.com/morikuni/aec v1.0.0 // indirect
1519
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
20+
github.com/opencontainers/image-spec v1.0.1 // indirect
1621
github.com/pkg/errors v0.8.1
1722
github.com/rs/xid v1.2.1
1823
github.com/sergi/go-diff v1.1.0
@@ -22,5 +27,9 @@ require (
2227
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9
2328
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
2429
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect
30+
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
31+
google.golang.org/grpc v1.30.0 // indirect
2532
gopkg.in/yaml.v2 v2.2.7
2633
)
34+
35+
replace github.com/docker/docker v1.13.1 => github.com/docker/engine v17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible

‎go.sum

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,50 @@
1+
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
12
github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI=
23
github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE=
4+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
5+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
36
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
47
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
58
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
69
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
10+
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
11+
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
12+
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
13+
github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI=
14+
github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
715
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
816
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
917
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1018
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
1119
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1220
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
1321
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
14-
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
15-
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
22+
github.com/docker/engine v17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible h1:5uC2w1yhWUPlduFVWY/W3wiluc85Cv8nxIKWPSFk/WY=
23+
github.com/docker/engine v17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
1624
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
1725
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
1826
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
1927
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
28+
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
29+
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
30+
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
31+
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
32+
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
33+
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
34+
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
35+
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
36+
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
37+
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
38+
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
39+
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
40+
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
41+
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
2042
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
2143
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
2244
github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4 h1:xKkUL6QBojwguhKKetf1SocCAKqc6W7S/mGm9xEGllo=
2345
github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
46+
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
47+
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
2448
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
2549
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
2650
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -30,12 +54,17 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
3054
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
3155
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
3256
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
57+
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
58+
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
3359
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
3460
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
61+
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
62+
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
3563
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
3664
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
3765
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
3866
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
67+
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
3968
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
4069
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
4170
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
@@ -57,9 +86,22 @@ github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2
5786
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
5887
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM=
5988
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
89+
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
90+
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
91+
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
92+
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
93+
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
94+
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
95+
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
96+
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
6097
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
6198
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
6299
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
100+
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
101+
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
102+
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
103+
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
104+
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
63105
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
64106
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
65107
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -69,11 +111,35 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qd
69111
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
70112
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
71113
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
114+
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
72115
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
116+
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
117+
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
118+
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
119+
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
120+
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
121+
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
122+
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
123+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
124+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
125+
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
126+
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
127+
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
128+
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
129+
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
130+
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
131+
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
132+
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
133+
google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
134+
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
73135
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
74136
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
75137
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
76138
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
77139
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
78140
gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
79141
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
142+
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
143+
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
144+
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
145+
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

‎pkg/config/config.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"io/ioutil"
1111
"os/user"
1212

13+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
1314
"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
1415
"gitlab.com/postgres-ai/database-lab/pkg/services/platform"
1516
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
@@ -26,9 +27,18 @@ type Config struct {
2627
Provision provision.Config `yaml:"provision"`
2728
Cloning cloning.Config `yaml:"cloning"`
2829
Platform platform.Config `yaml:"platform"`
30+
Global Global `yaml:"global"`
31+
Retrieval config.Config `yaml:"retrieval"`
2932
Debug bool `yaml:"debug"`
3033
}
3134

35+
// Global contains global Database Lab configurations.
36+
type Global struct {
37+
Engine string `yaml:"engine"`
38+
DataDir string `yaml:"dataDir"`
39+
DockerImage string `yaml:"dockerImage"`
40+
}
41+
3242
// LoadConfig instances a new Config by configuration filename.
3343
func LoadConfig(name string) (*Config, error) {
3444
path, err := util.GetConfigPath(name)

‎pkg/retrieval/components/components.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
2020 © Postgres.ai
3+
*/
4+
5+
// Package components provides the key components of data retrieval.
6+
package components
7+
8+
import (
9+
"context"
10+
11+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
12+
)
13+
14+
// StageBuilder builds a StageRunner.
15+
type StageBuilder interface {
16+
BuildStageRunner(string) (StageRunner, error)
17+
}
18+
19+
// JobRunner performs a job.
20+
type JobRunner interface {
21+
// Name returns a job name.
22+
Name() string
23+
24+
// Run starts a job.
25+
Run(ctx context.Context) error
26+
}
27+
28+
// StageRunner declares stage content and performs stage jobs.
29+
type StageRunner interface {
30+
// AddJob applies jobs to the current stage.
31+
AddJob(JobRunner)
32+
33+
// BuildJob builds stage jobs.
34+
BuildJob(config.JobConfig) (JobRunner, error)
35+
36+
// Run starts stage jobs.
37+
Run(ctx context.Context) error
38+
}

‎pkg/retrieval/config/config.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
2020 © Postgres.ai
3+
*/
4+
5+
// Package config contains configuration options of the data retrieval.
6+
package config
7+
8+
// Config describes of data retrieval stages.
9+
type Config struct {
10+
Stages []string `yaml:"stages,flow"`
11+
StageSpec map[string]Spec `yaml:"spec"`
12+
}
13+
14+
// Spec describes a retrieval job.
15+
type Spec struct {
16+
Jobs []JobConfig `yaml:"jobs,flow"`
17+
}
18+
19+
// JobConfig contains details about a job.
20+
type JobConfig struct {
21+
Name string `yaml:"name"`
22+
Options map[string]interface{} `yaml:"options"`
23+
}

0 commit comments

Comments
 (0)